{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nimport matplotlib.pylab as plt\nimport sklearn as sklearn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import cross_val_predict\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import roc_curve\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n#MLP hyperparameters.\noutput_shape = 10","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-10-09T16:52:24.865466Z","iopub.execute_input":"2021-10-09T16:52:24.865797Z","iopub.status.idle":"2021-10-09T16:52:30.880805Z","shell.execute_reply.started":"2021-10-09T16:52:24.865715Z","shell.execute_reply":"2021-10-09T16:52:30.880133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_roc_curve(fpr, tpr, label=None): \n    plt.plot(fpr, tpr, linewidth=2, label=label) \n    plt.plot([0, 1], [0, 1], 'k--') \n    ","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:30.882113Z","iopub.execute_input":"2021-10-09T16:52:30.883012Z","iopub.status.idle":"2021-10-09T16:52:30.888747Z","shell.execute_reply.started":"2021-10-09T16:52:30.882974Z","shell.execute_reply":"2021-10-09T16:52:30.887822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(dirname+'/'+filenames[1])\ntest = pd.read_csv(dirname+'/'+filenames[2])","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:30.89005Z","iopub.execute_input":"2021-10-09T16:52:30.890362Z","iopub.status.idle":"2021-10-09T16:52:36.501845Z","shell.execute_reply.started":"2021-10-09T16:52:30.890323Z","shell.execute_reply":"2021-10-09T16:52:36.501213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Prepare data. \ny = train['label']\nX = train.values[:,1:]/255.0\n\nX_train, X_val, y_train, y_val = train_test_split(X,y, test_size=0.2, shuffle=True, random_state=42)\n\nX_test = test.values/255.0","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:36.503656Z","iopub.execute_input":"2021-10-09T16:52:36.504459Z","iopub.status.idle":"2021-10-09T16:52:37.155339Z","shell.execute_reply.started":"2021-10-09T16:52:36.50442Z","shell.execute_reply":"2021-10-09T16:52:37.154394Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SGD\n#from sklearn.linear_model import SGDClassifier\n#SGD = SGDClassifier(random_state=42)\n#SGD.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.15646Z","iopub.execute_input":"2021-10-09T16:52:37.157211Z","iopub.status.idle":"2021-10-09T16:52:37.161072Z","shell.execute_reply.started":"2021-10-09T16:52:37.157178Z","shell.execute_reply":"2021-10-09T16:52:37.160293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#How well does the model performs?\n#SGD_scores = cross_val_score(SGD, X_val, y_val, scoring=\"accuracy\", cv=3)\n#print (SGD_scores)\n#SGD_xval = cross_val_predict(SGD, X_val, y_val, cv=3)\n#print (SGD_xval)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.162394Z","iopub.execute_input":"2021-10-09T16:52:37.163118Z","iopub.status.idle":"2021-10-09T16:52:37.173347Z","shell.execute_reply.started":"2021-10-09T16:52:37.16308Z","shell.execute_reply":"2021-10-09T16:52:37.172584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SGD_conf_mx = confusion_matrix(y_val, SGD_xval)\n#np.fill_diagonal(SGD_conf_mx, 0) \n#plt.matshow(SGD_conf_mx, cmap=plt.cm.gray) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.174542Z","iopub.execute_input":"2021-10-09T16:52:37.175321Z","iopub.status.idle":"2021-10-09T16:52:37.190026Z","shell.execute_reply.started":"2021-10-09T16:52:37.175276Z","shell.execute_reply":"2021-10-09T16:52:37.189086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Decision Tree\n#from sklearn.tree import DecisionTreeClassifier\n\n#dtree = DecisionTreeClassifier(max_depth=10000)\n#dtree.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.191488Z","iopub.execute_input":"2021-10-09T16:52:37.192003Z","iopub.status.idle":"2021-10-09T16:52:37.20186Z","shell.execute_reply.started":"2021-10-09T16:52:37.191955Z","shell.execute_reply":"2021-10-09T16:52:37.201011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#How well does the model performs?\n#dtree_scores = cross_val_score(dtree, X_val, y_val, scoring=\"accuracy\", cv=3)\n#print (dtree_scores)\n#dtree_xval = cross_val_predict(dtree, X_val, y_val, cv=3)\n#print (dtree_xval)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.203188Z","iopub.execute_input":"2021-10-09T16:52:37.204015Z","iopub.status.idle":"2021-10-09T16:52:37.212879Z","shell.execute_reply.started":"2021-10-09T16:52:37.203966Z","shell.execute_reply":"2021-10-09T16:52:37.21219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dtree_conf_mx = confusion_matrix(y_val, dtree_xval)\n#np.fill_diagonal(dtree_conf_mx, 0) \n#plt.matshow(dtree_conf_mx, cmap=plt.cm.gray) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.215771Z","iopub.execute_input":"2021-10-09T16:52:37.216781Z","iopub.status.idle":"2021-10-09T16:52:37.224379Z","shell.execute_reply.started":"2021-10-09T16:52:37.216706Z","shell.execute_reply":"2021-10-09T16:52:37.223279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.ensemble import RandomForestClassifier\n\n#RF = RandomForestClassifier(max_depth=10000)\n#RF.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.226014Z","iopub.execute_input":"2021-10-09T16:52:37.226469Z","iopub.status.idle":"2021-10-09T16:52:37.233342Z","shell.execute_reply.started":"2021-10-09T16:52:37.226433Z","shell.execute_reply":"2021-10-09T16:52:37.23229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#How well does the model performs?\n#RF_scores = cross_val_score(RF, X_val, y_val, scoring=\"accuracy\", cv=3)\n#print (RF_scores)\n#RF_xval = cross_val_predict(RF, X_val, y_val, cv=3)\n#print (RF_xval)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.234502Z","iopub.execute_input":"2021-10-09T16:52:37.234791Z","iopub.status.idle":"2021-10-09T16:52:37.24407Z","shell.execute_reply.started":"2021-10-09T16:52:37.23476Z","shell.execute_reply":"2021-10-09T16:52:37.243404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#RF_conf_mx = confusion_matrix(y_val, RF_xval)\n#np.fill_diagonal(RF_conf_mx, 0) \n#plt.matshow(RF_conf_mx, cmap=plt.cm.gray) \n#plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.244958Z","iopub.execute_input":"2021-10-09T16:52:37.245634Z","iopub.status.idle":"2021-10-09T16:52:37.254557Z","shell.execute_reply.started":"2021-10-09T16:52:37.245598Z","shell.execute_reply":"2021-10-09T16:52:37.253797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_1 = tf.keras.models.Sequential([\n    tf.keras.layers.Input(shape=(28*28)),\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dense(output_shape, activation='sigmoid')  \n])\n\n#lossfn = tf.keras.losses.SparseCategoricalCrossentropy()\n#optfn = tf.keras.optimizers.Adam()\n\n#model_1.compile(loss=lossfn,\n#                optimizer=optfn,\n#                metrics=['accuracy'])\n\n#history = model_1.fit(X_train, y_train,\n#            epochs = 10,\n#            validation_data=(X_val, y_val))\n\n#plt.figure()\n#plt.plot(history.history['loss'], label='loss')\n#plt.plot(history.history['val_loss'], label='Val loss')\n#plt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.255582Z","iopub.execute_input":"2021-10-09T16:52:37.256615Z","iopub.status.idle":"2021-10-09T16:52:37.353073Z","shell.execute_reply.started":"2021-10-09T16:52:37.256569Z","shell.execute_reply":"2021-10-09T16:52:37.352193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_2 = tf.keras.models.Sequential([\n    tf.keras.layers.Input(shape=(28*28)),\n    tf.keras.layers.Dense(128, activation='relu'),    \n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dense(output_shape, activation='sigmoid')  \n])\n\n#lossfn = tf.keras.losses.SparseCategoricalCrossentropy()\n#optfn = tf.keras.optimizers.Adam()\n\n#model_2.compile(loss=lossfn,\n#                optimizer=optfn,\n#                metrics=['accuracy'])\n\n#history = model_2.fit(X_train, y_train,\n#            epochs = 10,\n#            validation_data=(X_val, y_val))\n\n#plt.figure()\n#plt.plot(history.history['loss'], label='loss')\n#plt.plot(history.history['val_loss'], label='Val loss')\n#plt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.354758Z","iopub.execute_input":"2021-10-09T16:52:37.355259Z","iopub.status.idle":"2021-10-09T16:52:37.393648Z","shell.execute_reply.started":"2021-10-09T16:52:37.355206Z","shell.execute_reply":"2021-10-09T16:52:37.392774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Construct a convolutional network. I made some guesses to the number of filters and kernel size based on other notebooks. ","metadata":{}},{"cell_type":"code","source":"model_3 = tf.keras.models.Sequential([\n    tf.keras.layers.Input(shape=(28*28)),\n    tf.keras.layers.Reshape((28,28,1)),\n    tf.keras.layers.Conv2D(32,5,strides=1, use_bias=False, activation='relu'),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Conv2D(64,5,strides=1, use_bias=False, activation='relu'),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Dense(256, activation='relu', use_bias=False),\n    tf.keras.layers.Dropout(0.2),\n    tf.keras.layers.Dense(128, activation='relu', use_bias=False),\n    tf.keras.layers.Dense(64, activation='relu', use_bias=False),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.Dense(output_shape, activation='softmax')  \n])\n\n#lossfn = tf.keras.losses.SparseCategoricalCrossentropy()\n#optfn = tf.keras.optimizers.Adam()\n\n#model_3.compile(loss=lossfn,\n#                optimizer=optfn,\n#                metrics=['accuracy'])\n\n#history = model_3.fit(X_train, y_train,\n#            epochs = 20,\n#            validation_data=(X_val, y_val))\n\n#plt.figure()\n#plt.plot(history.history['loss'], label='loss')\n#plt.plot(history.history['val_loss'], label='Val loss')\n#plt.legend()","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.394933Z","iopub.execute_input":"2021-10-09T16:52:37.395244Z","iopub.status.idle":"2021-10-09T16:52:37.509085Z","shell.execute_reply.started":"2021-10-09T16:52:37.395205Z","shell.execute_reply":"2021-10-09T16:52:37.507996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"I got a 99.246% accuracy for the network above. \nAdd a slight random rotation just after the reshape to introduce more variability in the input. ","metadata":{}},{"cell_type":"markdown","source":"Try augmenting the incoming data to help the model generalize more. Use RandomRotation and RandomZoom. Remember, some of these digits are already slightly rotated. ","metadata":{}},{"cell_type":"code","source":"preprocessing_test = tf.keras.models.Sequential([\n    tf.keras.layers.Input(shape=(28*28)),\n    tf.keras.layers.Reshape((28,28,1)),\n    tf.keras.layers.experimental.preprocessing.RandomRotation(0.05),\n    tf.keras.layers.experimental.preprocessing.RandomZoom(0.2)])\n\nsample = preprocessing_test(X[0:1])\n\nplt.figure()\nplt.imshow(sample[0])","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:37.510663Z","iopub.execute_input":"2021-10-09T16:52:37.510993Z","iopub.status.idle":"2021-10-09T16:52:38.082725Z","shell.execute_reply.started":"2021-10-09T16:52:37.510951Z","shell.execute_reply":"2021-10-09T16:52:38.081884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_4 = tf.keras.models.Sequential([\n    tf.keras.layers.Input(shape=(28*28)),\n    tf.keras.layers.Reshape((28,28,1)),\n    tf.keras.layers.experimental.preprocessing.RandomRotation(0.05),\n    tf.keras.layers.experimental.preprocessing.RandomRotation(0.2),\n    \n    tf.keras.layers.Conv2D(32,5,strides=1, activation='relu', padding='same'),    \n    tf.keras.layers.Conv2D(32,5,strides=1, activation='relu', padding='same'),  \n    tf.keras.layers.MaxPool2D(pool_size=(2,2)),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Conv2D(64,3,activation='relu', padding='same'),    \n    tf.keras.layers.Conv2D(64,3, activation='relu', padding='same'),  \n    tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=(2,2)),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Flatten(),   \n    tf.keras.layers.Dense(256, activation='relu'),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(output_shape, activation='softmax')  \n])\n\nmodel_4.summary()\nfrom tensorflow.keras.callbacks import History, ReduceLROnPlateau\nh = History()\nrlr = ReduceLROnPlateau(monitor='val_acc', \n                        patience=3, \n                        verbose=1, \n                        factor=0.5, \n                        min_lr=0.00001)\n\nlossfn = tf.keras.losses.SparseCategoricalCrossentropy()\n# Define the optimizer\noptimizer = tf.keras.optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n\nmodel_4.compile(loss=lossfn,\n                optimizer=optimizer,\n                metrics=['accuracy'])\n\nhistory = model_4.fit(X, y,\n            epochs = 30,\n            shuffle=True,\n            batch_size = 86,\n            callbacks=[rlr])\n            #validation_data=(X_val, y_val))\n\nplt.figure()\nplt.plot(history.history['loss'], label='loss')","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:55.543138Z","iopub.execute_input":"2021-10-09T16:52:55.544Z","iopub.status.idle":"2021-10-09T16:55:35.410739Z","shell.execute_reply.started":"2021-10-09T16:52:55.543961Z","shell.execute_reply":"2021-10-09T16:55:35.409427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SUBMIT TO KAGGLE\nresults = model_4.predict(X_test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\n\nmy_submission = pd.DataFrame({'ImageId': np.arange(1,28001), 'Label': results})\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-09T16:52:38.636775Z","iopub.status.idle":"2021-10-09T16:52:38.637136Z","shell.execute_reply.started":"2021-10-09T16:52:38.636949Z","shell.execute_reply":"2021-10-09T16:52:38.636971Z"},"trusted":true},"execution_count":null,"outputs":[]}]}