{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
\nDigit Recogniser | Dimension Reduction PCA, t-SNE, UMAP\n
","metadata":{}},{"cell_type":"markdown","source":"\"tree\"","metadata":{}},{"cell_type":"markdown","source":"
\nTable of Contents:\n
\n\n\n\n* [1. Import Data](#1)\n* [2. Digit Plots](#2)\n* [3. PCA](#3)\n* [4. Dimension Reduction](#4)\n* [5. PCA + t-SNE](#5)\n* [6. PCA + UMAP](#6)\n* [7. My First Artificial Neural networks (ANN) Model](#7)\n* [8. Logistic Regression vs Perceptron](#8)\n* [9. Building an Image Classifier Using Keras Simple Sequential Class](#9)\n* [10.Building an Image Classifier Using Keras Simple Functional Class](#10)\n* [11. TensorBoard on Kaggle](#11)","metadata":{}},{"cell_type":"code","source":"#computation import \nimport pandas as pd\nimport numpy as np\nimport os\nimport sklearn.metrics as metrics\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import accuracy_score\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom scipy import stats\n\n#plot imports\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport matplotlib as mpl\n\n#management import\nimport gc\nimport time\n\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n1. Import Data\n
\n\n* [Jump Top](#0)\n","metadata":{}},{"cell_type":"code","source":"%time\ndir = '../input/digit-recognizer/'\nz = '.csv'\ntrain = pd.read_csv(dir+'train'+z)\ntest = pd.read_csv(dir+'test'+z)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ENV SETUP OF THE NOTEBOOK\nTARGET = 'label'\nFOLD = 5\nSEED = 42","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train Shape: ',train.shape)\ntrain.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Test Shape: ',test.shape)\ntest.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#seeing for missing values\nprint(f'''Train Null Count: {train.isnull().sum().sum()}\nTest Null Count: {test.isnull().sum().sum()}''')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train[TARGET]\ntrain.drop(TARGET,axis=1,inplace=True)\n\n#check if all columns in train is in test\nassert train.columns.to_list() == test.columns.to_list()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n2. Digit Plots\n
\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"code","source":"#function to plot multiple digits\ndef plot_digits(instances, images_per_row=5, **options):\n size = 28\n images_per_row = min(len(instances), images_per_row)\n images = [instance.reshape(size,size) for instance in instances]\n n_rows = (len(instances) - 1) // images_per_row + 1\n row_images = []\n n_empty = n_rows * images_per_row - len(instances)\n images.append(np.zeros((size, size * n_empty)))\n for row in range(n_rows):\n rimages = images[row * images_per_row : (row + 1) * images_per_row]\n row_images.append(np.concatenate(rimages, axis=1))\n image = np.concatenate(row_images, axis=0)\n plt.imshow(image, cmap = mpl.cm.binary, **options)\n plt.axis(\"off\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#single digit plot\nsome_digit = train.iloc[0].values\nsome_digit_image = some_digit.reshape(28, 28)\nplt.imshow(some_digit_image, cmap=mpl.cm.binary)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#multiple digit plot\nexample_images = train.iloc[:100].values\nplot_digits(example_images, images_per_row=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plot the count of labels in the training dataset\n_x = y.value_counts().index\n_y = y.value_counts().values\n\nsns.barplot(_x,_y)\n\ndel _x, _y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nNote: Upvote is Free!!!\n
","metadata":{}},{"cell_type":"markdown","source":"
\n

Exercise

Train a Random Forest classifier on the dataset and time how long it takes, then evaluate the resulting model on the test set.\n
","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nX_train, X_test, y_train, y_test = train_test_split(train,y,test_size=0.2,\n random_state=SEED)\n\nstart_time = time.time()\nrf = RandomForestClassifier(n_estimators=100, random_state=SEED)\nrf.fit(X_train,y_train)\n\nexecution_time = time.time() - start_time\n\nprint(f\"Training took {execution_time:.2f}s\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat = rf.predict(X_test)\nround(accuracy_score(y_test,y_hat)*100,2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)\nstart_time = time.time()\nfor fold,(train_indx,test_indx) in enumerate(cv.split(train,y)):\n X_train, X_test = train.iloc[train_indx].reset_index(drop=True),train.iloc[test_indx].reset_index(drop=True)\n y_train, y_test = y.iloc[train_indx].reset_index(drop=True),y.iloc[test_indx].reset_index(drop=True)\n \n rf = RandomForestClassifier(n_estimators=100,random_state=fold)\n rf.fit(X_train,y_train)\n \n y_hat = rf.predict(X_test)\n print(f'Fold {fold+1}: Accuracy Score: {round(accuracy_score(y_test,y_hat)*100,2)}%')\n print('='*30)\nexecution_time = time.time() - start_time\n\ndel X_train, X_test, y_train, y_test, rf, cv, y_hat\ngc.collect()\n\nprint(f\"5 Folds Took: {execution_time:.2f}s\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n3. PCA\n
\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"markdown","source":"
\n

Exercise

Next, use PCA to reduce the dataset's dimensionality, with an explained variance ratio of 95%.\n
","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\nX_train, X_test, y_train, y_test = train_test_split(train,y,test_size=0.2,\n random_state=SEED)\n\npca = PCA(n_components=0.95)\nX_train_reduced = pca.fit_transform(X_train)\nprint('Shape of train-set After PCA:',X_train_reduced.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\nThe following code compresses the MNIST dataset down to 154 dimensions, then\nuses the inverse_transform() method to decompress it back to 784 dimensions:\n'''\n\nX_recovered = pca.inverse_transform(X_train_reduced)\n\n#single digit plot\nsome_digit = X_recovered[0]\nsome_digit_image = some_digit.reshape(28, 28)\nplt.imshow(some_digit_image, cmap=mpl.cm.binary)\nplt.axis(\"off\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Exercise

Train a new Random Forest classifier on the reduced dataset and see how long it takes. Was training much faster?\n
","metadata":{}},{"cell_type":"code","source":"start_time = time.time()\nrf = RandomForestClassifier(n_estimators=100, random_state=SEED)\nrf.fit(X_train_reduced,y_train)\n\nexecution_time = time.time() - start_time\n\nprint(f\"Training took {execution_time:.2f}s\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Oh no!

Training is actually more than twice slower now! How can that be? Well, as we saw in this chapter, dimensionality reduction does not always lead to faster training time: it depends on the dataset, the model and the training algorithm.\n
\n\n","metadata":{}},{"cell_type":"markdown","source":"
\n

Exercise

Next evaluate the classifier on the test set: how does it compare to the previous classifier?\n
","metadata":{}},{"cell_type":"code","source":"X_test_reduced = pca.transform(X_test)\n\ny_hat = rf.predict(X_test_reduced)\nround(accuracy_score(y_test,y_hat)*100,2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Poor

It is common for performance to drop slightly when reducing dimensionality, because we do lose some useful signal in the process. However, the performance drop is rather severe in this case. So PCA really did not help: it slowed down training and reduced performance. :(\n
","metadata":{}},{"cell_type":"markdown","source":"
\nLet's see if it helps when using softmax regression:\n
","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlog_clf = LogisticRegression(multi_class=\"multinomial\", solver=\"saga\", random_state=SEED)\n\nstart_time = time.time()\nlog_clf.fit(X_train, y_train)\nend_time = time.time()\n\nex_time = end_time - start_time\nprint(f\"Training took {ex_time:.2f}s\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = log_clf.predict(X_test)\nround(accuracy_score(y_test, y_pred),2)*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Okay

so softmax regression takes much longer to train on this dataset than the random forest classifier, plus it performs worse on the test set. But that's not what we are interested in right now, we want to see how much PCA can help softmax regression. Let's train the softmax regression model using the reduced dataset:\n
\n","metadata":{}},{"cell_type":"code","source":"log_clf2 = LogisticRegression(multi_class=\"multinomial\", solver=\"saga\", random_state=SEED)\n\nstart_time = time.time()\nlog_clf2.fit(X_train_reduced, y_train)\nend_time = time.time()\n\nex_time = end_time - start_time\nprint(f\"Training took {ex_time:.2f}s\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = log_clf.predict(X_test)\nround(accuracy_score(y_test, y_pred),2)*100","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

Nice!

\nReducing dimensionality led to over 2× speedup. :)\nA very slight drop in performance, which might be a reasonable price to pay for a 2× speedup, depending on the application.\nSo there you have it: PCA can give you a formidable speedup... but not always!\n
","metadata":{}},{"cell_type":"markdown","source":"
\n4. Dimension Reduction\n
\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"markdown","source":"
\nUse t-SNE to reduce the MNIST dataset down to two dimensions and plot the result using Matplotlib. You can use a scatterplot using 10 different colors to represent each image's target class. \nDimensionality reduction on the full 42,000 images takes a very long time, so let's only do this on a random subset of 10,000 images:","metadata":{}},{"cell_type":"code","source":"np.random.seed(SEED) #setting up the seed\n\nm = 10000\nidx = np.random.permutation(42000)[:m]\n\nX = train.iloc[idx,:]\ny = y[idx]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.manifold import TSNE\nstart_time = time.time()\ntsne = TSNE(n_components=2, random_state=SEED)\nX_reduced = tsne.fit_transform(X)\nend_time = time.time()\n\nprint(end_time - start_time)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(13,10))\nplt.scatter(X_reduced[:, 0], X_reduced[:, 1],s=12, c=y, cmap=\"tab10\")\nplt.axis('off')\nplt.colorbar()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#function to plot scatter plot with digits on it.\n\nfrom sklearn.preprocessing import MinMaxScaler\nfrom matplotlib.offsetbox import AnnotationBbox, OffsetImage\n\ndef plot_digits(X, y, min_distance=0.05, images=None, figsize=(13, 10)):\n # Let's scale the input features so that they range from 0 to 1\n X_normalized = MinMaxScaler().fit_transform(X)\n # Now we create the list of coordinates of the digits plotted so far.\n # We pretend that one is already plotted far away at the start, to\n # avoid `if` statements in the loop below\n neighbors = np.array([[10., 10.]])\n # The rest should be self-explanatory\n plt.figure(figsize=figsize)\n cmap = mpl.cm.get_cmap(\"jet\")\n digits = np.unique(y)\n for digit in digits:\n plt.scatter(X_normalized[y == digit, 0], X_normalized[y == digit, 1],s=12, c=[cmap(digit / 9)])\n plt.axis(\"off\")\n ax = plt.gcf().gca() # get current axes in current figure\n for index, image_coord in enumerate(X_normalized):\n closest_distance = np.linalg.norm(neighbors - image_coord, axis=1).min()\n if closest_distance > min_distance:\n neighbors = np.r_[neighbors, [image_coord]]\n if images is None:\n plt.text(image_coord[0], image_coord[1], str(int(y[index])),\n color=cmap(y[index] / 9), fontdict={\"weight\": \"bold\", \"size\": 16})\n else:\n image = images[index].reshape(28, 28)\n imagebox = AnnotationBbox(OffsetImage(image, cmap=\"binary\"), image_coord)\n ax.add_artist(imagebox)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_digits(X_reduced, y, images=X.values, figsize=(22, 22))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nstart_time = time.time()\ntsne = PCA(n_components=2, random_state=SEED)\nX_reduced = tsne.fit_transform(X)\nend_time = time.time()\n\nprint(f'{round(end_time - start_time,2)}s')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_digits(X_reduced, y, images=X.values, figsize=(22, 22))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.manifold import LocallyLinearEmbedding\nstart_time = time.time()\ntsne = LocallyLinearEmbedding(n_components=2, random_state=SEED)\nX_reduced = tsne.fit_transform(X)\nend_time = time.time()\n\nprint(f'{round(end_time - start_time,2)}s')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_digits(X_reduced, y, images=X.values, figsize=(22, 22))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nstart_time = time.time()\ntsne = LinearDiscriminantAnalysis(n_components=2)\nX_reduced = tsne.fit_transform(X,y)\nend_time = time.time()\n\nprint(f'{round(end_time - start_time,2)}s')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_digits(X_reduced, y, images=X.values, figsize=(22, 22))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import umap\nstart_time = time.time()\ntsne = umap.UMAP(n_components=2, random_state=SEED)\nX_reduced = tsne.fit_transform(X)\nend_time = time.time()\n\nprint(f'{round(end_time - start_time,2)}s')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\npca_tsne = Pipeline([\n (\"pca\", PCA(n_components=0.95, random_state=42)),\n (\"tsne\", TSNE(n_components=2, random_state=42)),\n])\nstart_time = time.time()\nX_pca_tsne_reduced = pca_tsne.fit_transform(X)\nend_time = time.time()\nprint(\"PCA+t-SNE took {:.1f}s.\".format(end_time - start_time))\nplot_digits(X_pca_tsne_reduced, y.values)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_digits(X_pca_tsne_reduced, y, images=X.values, figsize=(22, 22))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n5. PCA + t-SNE\n
\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"code","source":"plot_digits(X_pca_tsne_reduced, y, images=X.values, figsize=(22, 22))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n6. PCA + UMAP\n
\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"code","source":"pca_umap = Pipeline([\n (\"pca\", PCA(n_components=0.95, random_state=SEED)),\n (\"umap\", umap.UMAP(n_components=2, random_state=SEED)),\n])\nstart_time = time.time()\nX_pca_umap_reduced = pca_umap.fit_transform(X)\nend_time = time.time()\nprint(\"PCA+UMAP took {:.1f}s.\".format(end_time - start_time))\nplot_digits(X_pca_umap_reduced, y.values)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_digits(X_pca_umap_reduced, y, images=X.values, figsize=(22, 22))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n7. My First Artificial Neural networks (ANN) Model\n
\n\n# ANNs\nANNs are at the very core of Deep Learning. They are versatile, powerful, and scala‐\nble, making them ideal to tackle large and highly complex Machine Learning tasks\nsuch as classifying billions of images (e.g., Google Images), powering speech recogni‐\ntion services (e.g., Apple’s Siri), recommending the best videos to watch to hundreds\nof millions of users every day (e.g., YouTube), or learning to beat the world champion\nat the game of Go (DeepMind’s AlphaGo).\n\n# Perceptron\n\nThe Perceptron is one of the simplest ANN architectures, invented in 1957 by Frank\nRosenblatt. It is based on a slightly different artificial neuron (see Figure below) called\na threshold logic unit (TLU), or sometimes a linear threshold unit (LTU). The inputs\nand output are numbers (instead of binary on/off values), and each input connection\nis associated with a weight. The TLU computes a weighted sum of its inputs (z = w1 x1 + w2 x2 ... + wn xn = x⊺ w), then applies a step function to that sum and outputs the\nresult: hw(x) = step(z), where z = x⊺ w.\n\n\n\n> Threshold logic unit: an artificial neuron which computes a weighted sum\nof its inputs then applies a step function\n\n# Step Function\n\n\n\nPerceptrons are trained using a variant of this rule that takes\ninto account the error made by the network when it makes a prediction\nPerceptron learning rule reinforces connections that help reduce the error. More\nspecifically, the Perceptron is fed one training instance at a time, and for each\ninstance it makes its predictions. For every output neuron that produced a wrong\nprediction, it reinforces the connection weights from the inputs that would have con‐\ntributed to the correct prediction.\n\n\nThe decision boundary of each output neuron is linear, so Perceptrons are incapable\nof learning complex patterns (just like Logistic Regression classifiers). However, if the\ntraining instances are linearly separable, Rosenblatt demonstrated that this algorithm\nwould converge to a solution.This is called the [**Perceptron convergence theorem**](http://web.mit.edu/course/other/i2course/www/vision_and_learning/perceptron_notes.pdf).\n\n* [Jump Top](#0)\n","metadata":{}},{"cell_type":"code","source":"_X = train[:5000]\n_y = y[:5000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n8. Logistic Regression Vs Perceptron\n
\n\nLogistic Regression perform better than perceptron. see this document for more:\n[Connections between Perceptron and Logistic Regression (and SVM)](https://www.cs.utexas.edu/~gdurrett/courses/fa2021/perc-lr-connections.pdf)\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.linear_model import Perceptron, LogisticRegression\n\n#starified kfold\ncv = StratifiedKFold(n_splits=5,shuffle=True,random_state=SEED)\nstart_time = time.time()\nfor fold,(train_indx,test_indx) in enumerate(cv.split(_X,_y)):\n X_train, X_test = _X.iloc[train_indx].reset_index(drop=True),_X.iloc[test_indx].reset_index(drop=True)\n y_train, y_test = _y.iloc[train_indx].reset_index(drop=True),_y.iloc[test_indx].reset_index(drop=True)\n \n #perceptron and logistic regression comparision\n per = Perceptron(tol=1e-3, random_state=SEED)\n logi = LogisticRegression(solver='saga',random_state=SEED,multi_class='multinomial')\n \n per.fit(X=X_train,y=y_train)\n logi.fit(X=X_train,y=y_train)\n \n y_hat = per.predict(X_test)\n y_hat_lr = logi.predict(X_test)\n \n print(f'Fold {fold+1}:Perceptron Accuracy Score: {round(accuracy_score(y_test,y_hat)*100,2)}%')\n print(f'Fold {fold+1}:Logistic Regression Accuracy Score: {round(accuracy_score(y_test,y_hat_lr)*100,2)}%')\n\n print('='*30)\nexecution_time = time.time() - start_time\n\ndel X_train, X_test, y_train, y_test, cv, y_hat, y_hat_lr, _X, _y\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n9. Building an Image Classifier Using Keras Simple Sequential Class\n
\n\n> First let's import TensorFlow and Keras.\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"markdown","source":"## **Using TensorBoard for Visualization**","metadata":{}},{"cell_type":"code","source":"import os\n\nroot_dir = os.path.join(os.curdir,'my_logs')\n\ndef get_run_log():\n import time\n run_id = time.strftime('run_%Y_%m_%d-%H_%M_%S')\n return os.path.join(root_dir,run_id)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\ntf.__version__, keras.__version__","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.loc[0].min(), train.loc[0].max() #we need to scale down the greyscale value to 0-1 (normalize)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir = '../input/digit-recognizer/'\nz = '.csv'\ntrain = pd.read_csv(dir+'train'+z)\ntest = pd.read_csv(dir+'test'+z)\n\ny = train[TARGET]\ntrain.drop(TARGET,axis=1,inplace=True)\n\n#normalizing the values between 0 and 1\nX_valid, X_train = train[:5000] / 255., train[5000:] / 255.\ny_valid, y_train = y[:5000], y[5000:]\nX_test = test / 255.","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_train.iloc[0,:].values.reshape(28,28), cmap=\"binary\")\nprint(y_train.iloc[0])\nplt.axis('off')\nplt.show() #ploting the first row of all the columns to plot the digit","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clearing previous states and seeding\nkeras.backend.clear_session() #Resets all state generated by Keras.\nnp.random.seed(42) #seed for reuseablity\ntf.random.set_seed(42)\n#initializing model\nmodel = keras.models.Sequential()\nmodel.add(keras.layers.Dense(300, activation=\"relu\",input_shape=(784,))) #first layer with 300 neurons and we also mentioned the input shape\nmodel.add(keras.layers.Dense(100, activation=\"relu\")) #secound layer with relu activation \nmodel.add(keras.layers.Dense(10, activation=\"softmax\")) #output layer with softmax function as this is the multiclass classification problem","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.layers #we can see all the layers as list of object (only if mentioned the input shape above)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary() #it shows all the information about the model like layer name, input/output shape etc. for more info please refer tensorflow documents","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras.utils.plot_model(model, \"simple_keras_sequential.png\", show_shapes=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hidden1 = model.layers[1]\nhidden1.name","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.get_layer(hidden1.name) is hidden1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights, biases = hidden1.get_weights()\nweights","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"biases","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weights.shape, biases.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#compiling model (mentioning the loss and metrics for the model)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n optimizer=\"sgd\",\n metrics=[\"accuracy\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using earlystopping callback to use best model and avoid overfitting \ncallback = keras.callbacks.EarlyStopping(patience=10,restore_best_weights=True)\n\n#tensorboard\nrun_log = get_run_log()\ntensorboard_cb = keras.callbacks.TensorBoard(run_log)\n\nhistory = model.fit(X_train, y_train, epochs=100,validation_data=(X_valid, y_valid), callbacks= [callback,tensorboard_cb])","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.params","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(history.epoch)\nhistory.history.keys()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history).plot(figsize=(8, 5))\nplt.grid(True)\nplt.gca().set_ylim(0, 1)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(X_valid, y_valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\n\npreds = model.predict(test)\nnp.argmax(preds, axis=-1)\n\nsample_submission.Label = np.argmax(preds, axis=-1)\nsample_submission.to_csv('keras_Squesntial_model.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using Earlystopping callback on simple sequential class","metadata":{}},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5,random_state=SEED, shuffle=True)\n\nprediction = {}\nprediction['sequential'] = []\n\nfor fold, (train_indx,valid_indx) in enumerate(skf.split(train,y)):\n X_train, X_valid = train.loc[train_indx,:].reset_index(drop=True) /255, train.loc[valid_indx,:].reset_index(drop=True) /255\n y_train, y_valid = y.loc[train_indx].reset_index(drop=True), y.loc[valid_indx].reset_index(drop=True)\n \n #clearing previous states and seeding\n keras.backend.clear_session() #Resets all state generated by Keras.\n np.random.seed(42) #seed for reuseablity\n tf.random.set_seed(42)\n\n #initializing model\n model = keras.models.Sequential()\n model.add(keras.layers.Dense(300, activation=\"relu\",input_shape=(784,))) #first layer with 300 neurons and we also mentioned the input shape\n model.add(keras.layers.Dense(100, activation=\"relu\")) #secound layer with relu activation \n model.add(keras.layers.Dense(10, activation=\"softmax\")) #output layer with softmax function as this is the multiclass classification problem\n\n #compiling model (mentioning the loss and metrics for the model)\n model.compile(loss=\"sparse_categorical_crossentropy\",\n optimizer=\"sgd\",\n metrics=[\"accuracy\"])\n \n #using earlystopping callback to use best model and avoid overfitting \n callback = keras.callbacks.EarlyStopping(patience=10,restore_best_weights=True)\n history = model.fit(X_train, y_train, epochs=100,\n validation_data=(X_valid, y_valid), callbacks=[callback])\n \n #ploting the model history for every fold\n pd.DataFrame(history.history).plot(figsize=(8, 5))\n plt.grid(True)\n plt.gca().set_ylim(0, 1)\n plt.show()\n \n #make prediction on the test set\n preds = model.predict_proba(test)\n \n #save the prediction of every fold in dictionary\n prediction['sequential'].append(preds)\n \n #cleaning up memory\n del X_train, y_train, X_valid, y_valid, history, model, preds\n gc.collect()","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#taking mean of each columns and then converting to class label\nsample_submission.Label = np.argmax(np.mean(prediction['sequential'],axis=0),axis=1)\nsample_submission.to_csv('Sequentialkeras_5fold.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n10. Building an Image Classifier Using Keras Simple Functional Class\n
\n\n> The Keras functional API is a way to create models that are more flexible than the tf.keras.Sequential API. The functional API can handle models with non-linear topology, shared layers, and even multiple inputs or outputs.\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"code","source":"dir = '../input/digit-recognizer/'\nz = '.csv'\ntrain = pd.read_csv(dir+'train'+z)\ntest = pd.read_csv(dir+'test'+z)\n\ny = train[TARGET]\ntrain.drop(TARGET,axis=1,inplace=True)\n\n#normalizing the values between 0 and 1\nX_valid, X_train = train[:5000] / 255., train[5000:] / 255.\ny_valid, y_train = y[:5000], y[5000:]\nX_test = test / 255.","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#clearing previous states and seeding\nkeras.backend.clear_session() #Resets all state generated by Keras.\nnp.random.seed(42) #seed for reuseablity\ntf.random.set_seed(42)\n#initializing model\n\ninput_ = keras.layers.Input(shape=(784,))\nhidden1 = keras.layers.Dense(300, activation=\"relu\")(input_) #first layer with 300 neurons and we also mentioned the input shape\nhidden2 = keras.layers.Dense(100, activation=\"relu\")(hidden1) #secound layer with relu activation \nconcat = keras.layers.Concatenate()([input_, hidden2])\noutput_ = keras.layers.Dense(10, activation=\"softmax\")(concat) #output layer with softmax function as this is the multiclass classification problem\n\nmodel = keras.Model(inputs=[input_], outputs=[output_])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keras.utils.plot_model(model, \"simple_keras_functional.png\", show_shapes=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#compiling model (mentioning the loss and metrics for the model)\nmodel.compile(loss=\"sparse_categorical_crossentropy\",\n optimizer=\"sgd\",\n metrics=[\"accuracy\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using earlystopping callback to use best model and avoid overfitting \ncallback = keras.callbacks.EarlyStopping(patience=10,restore_best_weights=True)\n\n\n#tensorboard\nrun_log = get_run_log()\ntensorboard_cb = keras.callbacks.TensorBoard(run_log)\n\nhistory = model.fit(X_train, y_train, epochs=100,validation_data=(X_valid, y_valid), callbacks= [callback,tensorboard_cb])","metadata":{"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history).plot(figsize=(8, 5))\nplt.grid(True)\nplt.gca().set_ylim(0, 1)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.evaluate(X_valid, y_valid)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\n\npreds = model.predict(test)\nnp.argmax(preds, axis=-1)\n\nsample_submission.Label = np.argmax(preds, axis=-1)\nsample_submission.to_csv('keras_Functional_model.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"skf = StratifiedKFold(n_splits=5,random_state=SEED, shuffle=True)\n\nprediction = {}\nprediction['Functional'] = []\n\nfor fold, (train_indx,valid_indx) in enumerate(skf.split(train,y)):\n X_train, X_valid = train.loc[train_indx,:].reset_index(drop=True) /255, train.loc[valid_indx,:].reset_index(drop=True) /255\n y_train, y_valid = y.loc[train_indx].reset_index(drop=True), y.loc[valid_indx].reset_index(drop=True)\n \n #clearing previous states and seeding\n keras.backend.clear_session() #Resets all state generated by Keras.\n np.random.seed(42) #seed for reuseablity\n tf.random.set_seed(42)\n \n #initializing model\n input_ = keras.layers.Input(shape=(784,))\n hidden1 = keras.layers.Dense(300, activation=\"relu\")(input_) #first layer with 300 neurons and we also mentioned the input shape\n hidden2 = keras.layers.Dense(100, activation=\"relu\")(hidden1) #secound layer with relu activation \n concat = keras.layers.Concatenate()([input_, hidden2])\n output_ = keras.layers.Dense(10, activation=\"softmax\")(concat) #output layer with softmax function as this is the multiclass classification problem\n\n model = keras.Model(inputs=[input_], outputs=[output_])\n \n #compiling model (mentioning the loss and metrics for the model)\n model.compile(loss=\"sparse_categorical_crossentropy\",\n optimizer=\"sgd\",\n metrics=[\"accuracy\"])\n \n #using earlystopping callback to use best model and avoid overfitting \n callback = keras.callbacks.EarlyStopping(patience=10,restore_best_weights=True)\n history = model.fit(X_train, y_train, epochs=100,\n validation_data=(X_valid, y_valid), callbacks=[callback])\n \n #ploting the model history for every fold\n pd.DataFrame(history.history).plot(figsize=(8, 5))\n plt.grid(True)\n plt.gca().set_ylim(0, 1)\n plt.show()\n \n #make prediction on the test set\n preds = model.predict(test)\n \n #save the prediction of every fold in dictionary\n prediction['Functional'].append(preds)\n \n #cleaning up memory\n del X_train, y_train, X_valid, y_valid, history, model, preds\n gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#taking mode of each columns and then converting to class label\nsample_submission.Label = np.argmax(stats.mode(prediction['Functional'],axis=0)[0][0],axis=1)\nsample_submission.to_csv('Functionalkeras_5fold.csv',index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n11. TensorBoard on Kaggle\n
\n\n> Kaggle New Feature: [How to use TensorBoard in Kaggle Kernel](https://www.kaggle.com/product-feedback/89671)\n\n\n> [TensorBoard on Kaggle](https://www.kaggle.com/shivam1600/tensorboard-on-kaggle)\n\n\n* [Jump Top](#0)\n\n\n","metadata":{}},{"cell_type":"markdown","source":"## **Start Tensorboard**","metadata":{}},{"cell_type":"code","source":"# Download Ngrok to tunnel the tensorboard port to an external port\n!wget https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip\n!unzip ngrok-stable-linux-amd64.zip\n\n# Run tensorboard as well as Ngrox (for tunneling as non-blocking processes)\nimport os\nimport multiprocessing\n\n\npool = multiprocessing.Pool(processes = 10)\nresults_of_processes = [pool.apply_async(os.system, args=(cmd, ), callback = None )\n for cmd in [\n f\"tensorboard --logdir ./my_logs --host 0.0.0.0 --port 6006 &\",\n \"./ngrok http 6006 &\"\n ]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Get the url to access tensorboad**\nNote: This block of code might throw an error while running all blocks at time. So run blocks one by one till the block below","metadata":{}},{"cell_type":"code","source":"! curl -s http://localhost:4040/api/tunnels | python3 -c \\\n \"import sys, json; print(json.load(sys.stdin)['tunnels'][0]['public_url'])\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}