{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas, tensorflow, numpy\npandas.set_option('display.max_rows', 10)\npandas.set_option('display.max_columns', None)\nfrom sklearn.preprocessing import StandardScaler\nfrom keras import models, layers, regularizers\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.callbacks import EarlyStopping\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom sklearn.model_selection import StratifiedKFold\nfrom statistics import mean\nfrom keras_tuner import RandomSearch, Hyperband\nfrom sklearn.metrics import confusion_matrix","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-25T16:18:17.875978Z","iopub.execute_input":"2022-05-25T16:18:17.876526Z","iopub.status.idle":"2022-05-25T16:18:17.884295Z","shell.execute_reply.started":"2022-05-25T16:18:17.876489Z","shell.execute_reply":"2022-05-25T16:18:17.882834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy.random import seed\nseed(42)\ntensorflow.random.set_seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:18:17.893486Z","iopub.execute_input":"2022-05-25T16:18:17.893983Z","iopub.status.idle":"2022-05-25T16:18:17.901478Z","shell.execute_reply.started":"2022-05-25T16:18:17.893816Z","shell.execute_reply":"2022-05-25T16:18:17.900232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pandas.read_csv('../input/digit-recognizer/train.csv')\ntest = pandas.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:18:17.904780Z","iopub.execute_input":"2022-05-25T16:18:17.905543Z","iopub.status.idle":"2022-05-25T16:18:23.220575Z","shell.execute_reply.started":"2022-05-25T16:18:17.905487Z","shell.execute_reply":"2022-05-25T16:18:23.219433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- quick look at the datasets","metadata":{}},{"cell_type":"code","source":"print(f'Missing values in train : {train.isna().sum().sum()}')\nprint(train.info())\nprint()\nprint(f'Missing values in test : {test.isna().sum().sum()}')\nprint(test.info())","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:18:23.222660Z","iopub.execute_input":"2022-05-25T16:18:23.222969Z","iopub.status.idle":"2022-05-25T16:18:23.409146Z","shell.execute_reply.started":"2022-05-25T16:18:23.222936Z","shell.execute_reply":"2022-05-25T16:18:23.407818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- no missing values in our data\n- need to ensure we are working with a balanced dataset (else model tends to get biased towards majority class)\n- check distribution of `target` in train (should be uniform)","metadata":{}},{"cell_type":"code","source":"fig = px.histogram(train, x=\"label\", color=\"label\")\nfig.update_layout(bargap=0.5)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:18:23.410558Z","iopub.execute_input":"2022-05-25T16:18:23.410834Z","iopub.status.idle":"2022-05-25T16:18:24.112294Z","shell.execute_reply.started":"2022-05-25T16:18:23.410801Z","shell.execute_reply":"2022-05-25T16:18:24.111263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- seems almost uniform\n- let us have a peek at the training data","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:18:24.114714Z","iopub.execute_input":"2022-05-25T16:18:24.115641Z","iopub.status.idle":"2022-05-25T16:18:24.413764Z","shell.execute_reply.started":"2022-05-25T16:18:24.115581Z","shell.execute_reply":"2022-05-25T16:18:24.413053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:18:24.415170Z","iopub.execute_input":"2022-05-25T16:18:24.415786Z","iopub.status.idle":"2022-05-25T16:18:24.701774Z","shell.execute_reply.started":"2022-05-25T16:18:24.415750Z","shell.execute_reply":"2022-05-25T16:18:24.700707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- `test` does not contain the `target` column\n- pixel vals range between 0 & 255\n- we perform col standardization from pixel0 to pixel783","metadata":{}},{"cell_type":"code","source":"train.iloc[:, 1:] = StandardScaler().fit_transform(train.iloc[:, 1:])\ntest.iloc[:, :] = StandardScaler().fit_transform(test.iloc[:, :])","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:18:24.703403Z","iopub.execute_input":"2022-05-25T16:18:24.703685Z","iopub.status.idle":"2022-05-25T16:19:55.435336Z","shell.execute_reply.started":"2022-05-25T16:18:24.703655Z","shell.execute_reply":"2022-05-25T16:19:55.434147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:19:55.437200Z","iopub.execute_input":"2022-05-25T16:19:55.437600Z","iopub.status.idle":"2022-05-25T16:19:56.036418Z","shell.execute_reply.started":"2022-05-25T16:19:55.437551Z","shell.execute_reply":"2022-05-25T16:19:56.035410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- `label` is the target col in this multi-class classification problem\n- `label` is given as numerical values\n- [need to transform to binary vectors](https://stackoverflow.com/questions/44110426/when-to-use-to-categorical-in-keras) (OHE)","metadata":{}},{"cell_type":"code","source":"trainLabel = to_categorical(train.iloc[:, :1])\nprint(trainLabel)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:19:56.037691Z","iopub.execute_input":"2022-05-25T16:19:56.037957Z","iopub.status.idle":"2022-05-25T16:19:56.046821Z","shell.execute_reply.started":"2022-05-25T16:19:56.037920Z","shell.execute_reply":"2022-05-25T16:19:56.045793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- train a very simple model\n    - train using mini batch SGD, batch size be 128,\n    - run it for 10 epochs (higher epoch might overfit)\n    - use `categorical_crossentropy` for multi-class classification (other possible losses include `binary_crossentropy` for binary classification, `mse` for regression etc)","metadata":{}},{"cell_type":"code","source":"model = models.Sequential()\nmodel.add(layers.Dense(512, activation='relu', input_shape=(784,)))\nmodel.add(layers.Dense(10, activation='softmax'))\nmodel.summary()\n\nmodel.compile(optimizer=optimizers.RMSprop(learning_rate=0.01), loss='categorical_crossentropy', metrics=['accuracy'])\n\nmodel.fit(train.iloc[:, 1:], to_categorical(train.iloc[:, :1]), epochs=10, batch_size=128, validation_split=0.2)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:19:56.049029Z","iopub.execute_input":"2022-05-25T16:19:56.049374Z","iopub.status.idle":"2022-05-25T16:20:39.241101Z","shell.execute_reply.started":"2022-05-25T16:19:56.049328Z","shell.execute_reply":"2022-05-25T16:20:39.240104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- 95.07% accuracy, not a good score\n- there is no way to test the model on test data\n- we can improve the accuracy if optimal hyperparameters are used\n    - we use Stratified K-Fold and increase epochs to check whether that increases accuracy & decreases loss\n    - we will perform a 4:1 split on the train data","metadata":{}},{"cell_type":"code","source":"def makeModel(lr, lossFn, metrics) :\n    model = models.Sequential()\n    model.add(layers.Dense(512, activation='relu', input_shape=(784,)))\n    model.add(layers.Dense(10, activation='softmax'))\n    model.compile(optimizer=optimizers.RMSprop(learning_rate=lr), loss=lossFn, metrics=metrics)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:20:39.244191Z","iopub.execute_input":"2022-05-25T16:20:39.244565Z","iopub.status.idle":"2022-05-25T16:20:39.251431Z","shell.execute_reply.started":"2022-05-25T16:20:39.244531Z","shell.execute_reply":"2022-05-25T16:20:39.250440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numFolds = 1\n\nfor tI_, vI_ in StratifiedKFold(n_splits=5, shuffle=True, random_state=42).split(train.iloc[:, 1:], train.iloc[:, :1]) :\n    print(f'# Fold : {numFolds}/5')\n    trainX, trainY = train.iloc[:, 1:].loc[tI_], train.iloc[:, :1].loc[tI_]\n    validateX, validateY = train.iloc[:, 1:].loc[vI_], train.iloc[:, :1].loc[vI_]\n    model = None\n    model = makeModel(0.001, 'categorical_crossentropy', ['accuracy'])\n    stop_early = EarlyStopping(monitor='loss', patience=3) # stop training early if no performance gain after 3 epochs, reduces training time\n    history = model.fit(trainX, to_categorical(trainY), epochs=20, batch_size=128, validation_data=(validateX, to_categorical(validateY)), verbose=0, callbacks=[stop_early])\n    \n    fig = go.Figure()    \n    fig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"accuracy\"], mode='lines+markers', name='Training Accuracy'))\n    fig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"val_accuracy\"], mode='lines+markers', name='Validation Accuracy'))\n    fig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"loss\"], mode='lines+markers', name='Training Loss'))\n    fig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"val_loss\"], mode='lines+markers', name='Validation Loss'))\n    fig.update_layout(title='Training & Validation metrics', xaxis_title='Epoch #', yaxis_title='Value', width=800, height=400)\n    fig.show()\n\n    print(f'Mean Training Loss : {round(mean(history.history[\"loss\"]), 3)}, Mean Training Accuracy : {round(mean(history.history[\"accuracy\"]), 3)}')\n    print(f'Mean Validation Loss : {round(mean(history.history[\"val_loss\"]), 3)}, Mean Validation Accuracy : {round(mean(history.history[\"val_accuracy\"]), 3)}')\n    numFolds = numFolds + 1","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:20:39.252958Z","iopub.execute_input":"2022-05-25T16:20:39.253843Z","iopub.status.idle":"2022-05-25T16:24:07.091764Z","shell.execute_reply.started":"2022-05-25T16:20:39.253794Z","shell.execute_reply":"2022-05-25T16:24:07.090769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- in general, training loss decreases & accuracy increases with every epoch \n- same cannot be said for validation loss & accuracy\n    - hence, model is getting overfit to training data at higher epochs\n- even with higher epochs validation accuracy stagnates at ~0.97 \n    - we have to find the optimal hyperparameters","metadata":{}},{"cell_type":"code","source":"def makeHyperModel(hp) :\n    model = models.Sequential()\n    model.add(layers.Dense(units=hp.Int('Output1', min_value=32, max_value=1024, step=32), \n                           kernel_regularizer=regularizers.l1_l2(l1=hp.Choice('l1_value', values=[0.0,0.1,0.01,0.001,0.0001]), l2=hp.Choice('l2_value', values=[0.0,0.1,0.01,0.001,0.0001])), \n                           activation='relu', \n                           input_shape=(784,)))\n    model.add(layers.Dropout(hp.Choice('drop_rate', [0.1,0.2,0.3,0.4,0.5])))\n    # will not any additional hidden layers, not enough observations, might overfit\n#     for i in range(1, hp.Int(\"HiddenLayersNum\", 2, 4)):\n#         model.add(layers.Dense(units=hp.Int(f'Output{i}', min_value=32, max_value=1024, step=32), activation=\"relu\"))\n    model.add(layers.Dense(10, activation='softmax'))\n    model.compile(optimizer=optimizers.RMSprop(learning_rate=hp.Choice('learning_rate', values=[0.1, 0.01, 0.001, 0.0001])), loss='categorical_crossentropy', metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:24:07.093799Z","iopub.execute_input":"2022-05-25T16:24:07.094845Z","iopub.status.idle":"2022-05-25T16:24:07.104348Z","shell.execute_reply.started":"2022-05-25T16:24:07.094755Z","shell.execute_reply":"2022-05-25T16:24:07.103331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- will consider Hyperband tuner","metadata":{}},{"cell_type":"code","source":"tuner = Hyperband(makeHyperModel, objective='val_accuracy', max_epochs=20)\ntuner.search_space_summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:24:07.106098Z","iopub.execute_input":"2022-05-25T16:24:07.106628Z","iopub.status.idle":"2022-05-25T16:24:07.172477Z","shell.execute_reply.started":"2022-05-25T16:24:07.106588Z","shell.execute_reply":"2022-05-25T16:24:07.170747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tuner.search(train.iloc[:, 1:],to_categorical(train.iloc[:, :1]), epochs=20, validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', patience=5)])","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:24:07.174160Z","iopub.execute_input":"2022-05-25T16:24:07.175119Z","iopub.status.idle":"2022-05-25T16:52:27.726416Z","shell.execute_reply.started":"2022-05-25T16:24:07.175060Z","shell.execute_reply":"2022-05-25T16:52:27.725665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bestHP = tuner.get_best_hyperparameters()[0]\nprint(f'Optimal #units in the input layer : {bestHP[\"Output1\"]}, L1 : {bestHP[\"l1_value\"]}, L2 : {bestHP[\"l2_value\"]}, learning rate : {bestHP[\"learning_rate\"]}, drop out : {bestHP[\"drop_rate\"]}')\nbestModel = tuner.hypermodel.build(bestHP)\nbestModel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:52:27.728231Z","iopub.execute_input":"2022-05-25T16:52:27.729073Z","iopub.status.idle":"2022-05-25T16:52:27.783973Z","shell.execute_reply.started":"2022-05-25T16:52:27.729029Z","shell.execute_reply":"2022-05-25T16:52:27.782744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = bestModel.fit(train.iloc[:, 1:], to_categorical(train.iloc[:, :1]), epochs=20, batch_size=128, validation_split=0.2)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:52:27.785282Z","iopub.execute_input":"2022-05-25T16:52:27.785535Z","iopub.status.idle":"2022-05-25T16:53:39.049935Z","shell.execute_reply.started":"2022-05-25T16:52:27.785503Z","shell.execute_reply":"2022-05-25T16:53:39.048712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Mean Training Loss : {round(mean(history.history[\"loss\"]), 3)}, Mean Training Accuracy : {round(mean(history.history[\"accuracy\"]), 3)}')\nprint(f'Mean Validation Loss : {round(mean(history.history[\"val_loss\"]), 3)}, Mean Validation Accuracy : {round(mean(history.history[\"val_accuracy\"]), 3)}')","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:53:39.052218Z","iopub.execute_input":"2022-05-25T16:53:39.052487Z","iopub.status.idle":"2022-05-25T16:53:39.060806Z","shell.execute_reply.started":"2022-05-25T16:53:39.052456Z","shell.execute_reply":"2022-05-25T16:53:39.059461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = go.Figure()    \nfig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"accuracy\"], mode='lines+markers', name='Training Accuracy'))\nfig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"val_accuracy\"], mode='lines+markers', name='Validation Accuracy'))\nfig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"loss\"], mode='lines+markers', name='Training Loss'))\nfig.add_trace(go.Scatter(x=list(range(1,11)), y=history.history[\"val_loss\"], mode='lines+markers', name='Validation Loss'))\nfig.update_layout(title='Training & Validation metrics', xaxis_title='Epoch #', yaxis_title='Value', width=800, height=400)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:53:39.063931Z","iopub.execute_input":"2022-05-25T16:53:39.064950Z","iopub.status.idle":"2022-05-25T16:53:39.094238Z","shell.execute_reply.started":"2022-05-25T16:53:39.064859Z","shell.execute_reply":"2022-05-25T16:53:39.093282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- remember the simple baseline model with 1,333,770 trainable parameters, had validation accuracy ~0.896\n- our final model with optimal hyperparameters & 661,450 trainable parameters, our validation accuracy is at its highest at ~0.97\n    - validation accuracy peaks at 10 epochs, validation loss minimizes at 7 epochs\n        - best model starts to overfit after 10 epochs\n    - view the confusion matrix to check our shortcomings","metadata":{}},{"cell_type":"code","source":"bestModel = None\nbestModel = tuner.hypermodel.build(bestHP)\nbestModel.fit(train.iloc[:, 1:], to_categorical(train.iloc[:, :1]), epochs=10, batch_size=128, validation_split=0.2)\npred = bestModel.predict(train.iloc[:, 1:])","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:53:39.095981Z","iopub.execute_input":"2022-05-25T16:53:39.097093Z","iopub.status.idle":"2022-05-25T16:54:20.737661Z","shell.execute_reply.started":"2022-05-25T16:53:39.097047Z","shell.execute_reply":"2022-05-25T16:54:20.736591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.imshow(confusion_matrix(train.iloc[:, :1], numpy.argmax(pred, axis=1)), labels=dict(x=\"Actual digit\", y=\"Predicted Digit\"), x=list(range(0,10)), y=list(range(0,10)), color_continuous_scale='blugrn', text_auto=True)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:54:20.740107Z","iopub.execute_input":"2022-05-25T16:54:20.740494Z","iopub.status.idle":"2022-05-25T16:54:20.847969Z","shell.execute_reply.started":"2022-05-25T16:54:20.740443Z","shell.execute_reply":"2022-05-25T16:54:20.847005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- our best model gets confused between (7 and 9) & (4 and 9)\n- we are ready for prediction","metadata":{}},{"cell_type":"code","source":"finalPred = numpy.argmax(bestModel.predict(test), axis=1)\ndf = pandas.DataFrame(pandas.concat([pandas.Series(range(1,28001)), pandas.Series(finalPred)], axis=1))\ndf.rename({0:'ImageId', 1:'Label'}, axis=1, inplace=True)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:54:20.849295Z","iopub.execute_input":"2022-05-25T16:54:20.849660Z","iopub.status.idle":"2022-05-25T16:54:23.373449Z","shell.execute_reply.started":"2022-05-25T16:54:20.849614Z","shell.execute_reply":"2022-05-25T16:54:23.372629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-25T16:54:23.374758Z","iopub.execute_input":"2022-05-25T16:54:23.375110Z","iopub.status.idle":"2022-05-25T16:54:23.429564Z","shell.execute_reply.started":"2022-05-25T16:54:23.375079Z","shell.execute_reply":"2022-05-25T16:54:23.428600Z"},"trusted":true},"execution_count":null,"outputs":[]}]}