{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas, tensorflow, numpy, random\npandas.set_option('display.max_rows', 10)\npandas.set_option('display.max_columns', None)\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom keras import models, layers, regularizers\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom statistics import mean\nfrom keras_tuner import RandomSearch, Hyperband\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom sklearn.linear_model import LogisticRegression","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-29T07:08:14.638484Z","iopub.execute_input":"2022-05-29T07:08:14.638953Z","iopub.status.idle":"2022-05-29T07:08:14.649865Z","shell.execute_reply.started":"2022-05-29T07:08:14.638916Z","shell.execute_reply":"2022-05-29T07:08:14.648975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- ensure all outputs are reproducible","metadata":{}},{"cell_type":"code","source":"from numpy.random import seed\nseed(42)\ntensorflow.random.set_seed(42)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:44:13.974817Z","iopub.execute_input":"2022-05-28T22:44:13.975082Z","iopub.status.idle":"2022-05-28T22:44:13.982453Z","shell.execute_reply.started":"2022-05-28T22:44:13.975047Z","shell.execute_reply":"2022-05-28T22:44:13.981048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pandas.read_csv('../input/digit-recognizer/train.csv')\ntest = pandas.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:44:13.984986Z","iopub.execute_input":"2022-05-28T22:44:13.985556Z","iopub.status.idle":"2022-05-28T22:44:18.629898Z","shell.execute_reply.started":"2022-05-28T22:44:13.98552Z","shell.execute_reply":"2022-05-28T22:44:18.629162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Missing values in train : {train.isna().sum().sum()}')\nprint(train.info())\nprint()\nprint(f'Missing values in test : {test.isna().sum().sum()}')\nprint(test.info())","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:44:18.631931Z","iopub.execute_input":"2022-05-28T22:44:18.632191Z","iopub.status.idle":"2022-05-28T22:44:18.780483Z","shell.execute_reply.started":"2022-05-28T22:44:18.632157Z","shell.execute_reply":"2022-05-28T22:44:18.779756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:44:18.781567Z","iopub.execute_input":"2022-05-28T22:44:18.782028Z","iopub.status.idle":"2022-05-28T22:44:19.028511Z","shell.execute_reply.started":"2022-05-28T22:44:18.781987Z","shell.execute_reply":"2022-05-28T22:44:19.027829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:44:19.029444Z","iopub.execute_input":"2022-05-28T22:44:19.029666Z","iopub.status.idle":"2022-05-28T22:44:19.27168Z","shell.execute_reply.started":"2022-05-28T22:44:19.029636Z","shell.execute_reply":"2022-05-28T22:44:19.271031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.iloc[:, 1:] = StandardScaler().fit_transform(train.iloc[:, 1:])\ntest.iloc[:, :] = StandardScaler().fit_transform(test.iloc[:, :])","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:44:19.273038Z","iopub.execute_input":"2022-05-28T22:44:19.273487Z","iopub.status.idle":"2022-05-28T22:45:21.976828Z","shell.execute_reply.started":"2022-05-28T22:44:19.273452Z","shell.execute_reply":"2022-05-28T22:45:21.976089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(9)","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:21.977963Z","iopub.execute_input":"2022-05-28T22:45:21.978588Z","iopub.status.idle":"2022-05-28T22:45:22.598083Z","shell.execute_reply.started":"2022-05-28T22:45:21.978547Z","shell.execute_reply":"2022-05-28T22:45:22.597434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = px.histogram(train, x=\"label\", color=\"label\")\nfig.update_layout(bargap=0.5)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:22.599317Z","iopub.execute_input":"2022-05-28T22:45:22.599784Z","iopub.status.idle":"2022-05-28T22:45:23.551306Z","shell.execute_reply.started":"2022-05-28T22:45:22.599746Z","shell.execute_reply":"2022-05-28T22:45:23.550655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- CNN retains information about the spatial context of images, and the images are a 28x28 2D array of pixels\n- however, the train & test data are presented to us as a continuous 1D array of 784 pixel values\n- we need to reshape the train & test data","metadata":{}},{"cell_type":"code","source":"trainReshaped = train.iloc[:, 1:].values.reshape(-1,28,28,1)\ntestReshaped = test.values.reshape(-1,28,28,1)\n\nprint(f'Shape of training data : {trainReshaped.shape}, test data : {testReshaped.shape}')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:23.553393Z","iopub.execute_input":"2022-05-28T22:45:23.554075Z","iopub.status.idle":"2022-05-28T22:45:24.055499Z","shell.execute_reply.started":"2022-05-28T22:45:23.554039Z","shell.execute_reply":"2022-05-28T22:45:24.054816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- let us view 4 images of different digits","metadata":{}},{"cell_type":"code","source":"plt.figure()\nfig, axArr = plt.subplots(1,4) \n\nfor i in range(0, 4) :\n    plt.subplot(220+1+i)\n    plt.imshow(trainReshaped[i+5][:,:], cmap='gray_r')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:24.056639Z","iopub.execute_input":"2022-05-28T22:45:24.056885Z","iopub.status.idle":"2022-05-28T22:45:24.662207Z","shell.execute_reply.started":"2022-05-28T22:45:24.056848Z","shell.execute_reply":"2022-05-28T22:45:24.661569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = models.Sequential()\nmodel.add(layers.Conv2D(32, (3,3), activation='relu', input_shape=(28,28,1)))\nmodel.add(layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(layers.Conv2D(64, (3,3), activation='relu'))\nmodel.add(layers.MaxPooling2D(pool_size=(2,2)))\nmodel.add(layers.Conv2D(64, (3,3), activation='relu'))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\n\nmodel.summary()\n\nmodel.compile(optimizer=optimizers.RMSprop(learning_rate=0.01), loss='categorical_crossentropy', metrics=['accuracy'])\n\nhistory = model.fit(trainReshaped, to_categorical(train.iloc[:, :1]), epochs=10, batch_size=128, validation_split=0.2, verbose=1)\n\nprint(f'Mean validation accuracy : {round(mean(history.history[\"val_accuracy\"]), 3)}, validation loss : {round(mean(history.history[\"val_loss\"]), 3)}')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:24.663276Z","iopub.execute_input":"2022-05-28T22:45:24.6644Z","iopub.status.idle":"2022-05-28T22:45:48.754304Z","shell.execute_reply.started":"2022-05-28T22:45:24.66436Z","shell.execute_reply":"2022-05-28T22:45:48.753369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- a simple ANN architecture of similar capability yields ~95% accuracy on the same dataset\n- a simple architecture using CNN yields ~97% accuracy\n    - we can already see the improvement\n- what can we do to further increase the accuracy & prevent overfitting?\n    - [data augmentation](https://machinelearningmastery.com/image-augmentation-deep-learning-keras/), so the model is able to learn from other spatial perspectives\n        - we will not perform vertical_flip (6 becomes 9)\n        - rotation angle will be limited to 45\n    - test with strided convolutions instead of max pooling\n    - hyperparameter tuning\n    \n    ","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(rotation_range = 45, zoom_range = 0.2, fill_mode = 'nearest', featurewise_center = False, samplewise_center = False, \n                             shear_range = 0.2, width_shift_range = 0.2, height_shift_range = 0.2, horizontal_flip = True)\ndatagen.fit(trainReshaped)  ","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:53.808979Z","iopub.execute_input":"2022-05-28T22:45:53.809238Z","iopub.status.idle":"2022-05-28T22:45:53.913048Z","shell.execute_reply.started":"2022-05-28T22:45:53.80921Z","shell.execute_reply":"2022-05-28T22:45:53.912313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\nfig, axArr = plt.subplots(1,4) \n\nfor i in range(0, 4) :\n    plt.subplot(220+1+i)\n    plt.imshow(trainReshaped[i+5][:,:], cmap='gray_r')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:45:54.173693Z","iopub.execute_input":"2022-05-28T22:45:54.174425Z","iopub.status.idle":"2022-05-28T22:45:54.563297Z","shell.execute_reply.started":"2022-05-28T22:45:54.174386Z","shell.execute_reply":"2022-05-28T22:45:54.562604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- view images generated by data augmentation","metadata":{}},{"cell_type":"code","source":"for x,y in datagen.flow(trainReshaped, train.iloc[:, :1], batch_size=9, shuffle=False) :\n    plt.figure()\n    fig, axArr = plt.subplots(1,4) \n\n    for i in range(0, 4) :\n        plt.subplot(220+1+i)\n        plt.imshow(x[i+5][:,:], cmap='gray_r')\n    break","metadata":{"execution":{"iopub.status.busy":"2022-05-28T22:46:19.092898Z","iopub.execute_input":"2022-05-28T22:46:19.093158Z","iopub.status.idle":"2022-05-28T22:46:19.519744Z","shell.execute_reply.started":"2022-05-28T22:46:19.093129Z","shell.execute_reply":"2022-05-28T22:46:19.519087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"architecture reasoning \n- not too deep (input feature space is not too large)\n- dropout can be limited to 0.25 since there will not be too many parameters\n- initial layers learn low-level features (edges,corners), so 32 5x5 features will suffice\n- lower learning rate & higher epochs can be an improvement\n- validation split can be 0.3 to mimic the competition train:test ratio (7:3)","metadata":{}},{"cell_type":"code","source":"def makeModel1() :\n    \n    model = models.Sequential()\n    \n    model.add(layers.Conv2D(filters=32, kernel_size=(5,5), activation='relu', padding='same', input_shape=(28,28,1)))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Conv2D(filters=32, kernel_size=(5,5), activation='relu', padding='same', input_shape=(28,28,1)))\n    model.add(layers.BatchNormalization())\n    model.add(layers.MaxPooling2D(pool_size=(2,2), strides=(2, 2)))\n\n    model.add(layers.Conv2D(filters=64, kernel_size=(3,3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))    \n    model.add(layers.MaxPooling2D(pool_size=(2,2), strides=(2, 2)))\n    \n    model.add(layers.Conv2D(filters=128, kernel_size=(3,3), padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))\n    model.add(layers.MaxPooling2D(pool_size=(2,2), strides=(2, 2)))\n    \n    model.add(layers.Flatten())\n    model.add(layers.Dense(512))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))    \n    model.add(layers.Dropout(0.25))\n    \n    model.add(layers.Dense(10, activation='softmax'))\n    \n    model.compile(optimizer=optimizers.RMSprop(learning_rate=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-29T03:36:16.302433Z","iopub.execute_input":"2022-05-29T03:36:16.303225Z","iopub.status.idle":"2022-05-29T03:36:16.315425Z","shell.execute_reply.started":"2022-05-29T03:36:16.303183Z","shell.execute_reply":"2022-05-29T03:36:16.314623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def makeModel2() :\n    \n    model = models.Sequential()\n    \n    model.add(layers.Conv2D(filters=64, kernel_size=(5,5), activation='relu', padding='same', input_shape=(28,28,1)))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Conv2D(filters=64, kernel_size=(5,5), activation='relu', padding='same', input_shape=(28,28,1)))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Dropout(0.25))\n\n    model.add(layers.Conv2D(filters=128, kernel_size=(5,5), strides=2, padding='same'))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))    \n    model.add(layers.Dropout(0.25))\n    \n    model.add(layers.Flatten())\n    model.add(layers.Dense(512))\n    model.add(layers.BatchNormalization())\n    model.add(layers.Activation('relu'))    \n    model.add(layers.Dropout(0.25))\n    \n    model.add(layers.Dense(10, activation='softmax'))\n    \n    model.compile(optimizer=optimizers.RMSprop(learning_rate=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2022-05-29T03:36:17.030595Z","iopub.execute_input":"2022-05-29T03:36:17.03111Z","iopub.status.idle":"2022-05-29T03:36:17.041478Z","shell.execute_reply.started":"2022-05-29T03:36:17.031065Z","shell.execute_reply":"2022-05-29T03:36:17.040658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- create ensemble of above two models\n- train each model upto 1000 epochs (patience 100)\n- validation accuracy will be the mean validation accuracy for the last 50 epochs\n- predict according to weighted average of validation accuracy","metadata":{}},{"cell_type":"code","source":"trainX, validateX, trainY, validateY = train_test_split(trainReshaped, train.iloc[:, :1], test_size = 0.3)\nc = 1\nmodel1 = makeModel1()\nmodel2 = makeModel2()\n\nfor model in [model1, model2] :\n    \n    history = model.fit(datagen.flow(trainX, to_categorical(trainY), batch_size=256), validation_data=(validateX, to_categorical(validateY)), epochs=1000, verbose=9, \n                        callbacks=[EarlyStopping(monitor='val_loss', patience=100), \n                                   ModelCheckpoint(filepath='bestModel_ValLoss.h5', monitor='val_loss', save_best_only=True), \n                                   ModelCheckpoint(filepath='bestModel_|ValAccuracy.h5', monitor='val_accuracy', save_best_only=True)]\n                       )    \n    \n    fig = go.Figure()    \n    fig.add_trace(go.Scatter(x=list(range(1,51)), y=history.history[\"accuracy\"][-50:], mode='lines+markers', name='Training Accuracy'))\n    fig.add_trace(go.Scatter(x=list(range(1,51)), y=history.history[\"val_accuracy\"][-50:], mode='lines+markers', name='Validation Accuracy'))\n    fig.add_trace(go.Scatter(x=list(range(1,51)), y=history.history[\"loss\"][-50:], mode='lines+markers', name='Training Loss'))\n    fig.add_trace(go.Scatter(x=list(range(1,51)), y=history.history[\"val_loss\"][-50:], mode='lines+markers', name='Validation Loss'))\n    fig.update_layout(title='Training & Validation metrics', xaxis_title='Epoch #', yaxis_title='Value', width=1200, height=400)\n    fig.show()\n\n    print(f'Mean Training Loss (last 50 epochs) : {round(mean(history.history[\"loss\"][-50:]), 3)}, Mean Training Accuracy : {round(mean(history.history[\"accuracy\"][-50:]), 3)}')\n    print(f'Mean Validation Loss (last 50 epochs) : {round(mean(history.history[\"val_loss\"][-50:]), 3)}, Mean Validation Accuracy : {round(mean(history.history[\"val_accuracy\"][-50:]), 3)}')\n    \n    model.save(f'model_{c}.h5')\n    c = c + 1","metadata":{"execution":{"iopub.status.busy":"2022-05-29T04:44:15.830157Z","iopub.execute_input":"2022-05-29T04:44:15.830903Z","iopub.status.idle":"2022-05-29T06:20:23.275452Z","shell.execute_reply.started":"2022-05-29T04:44:15.830864Z","shell.execute_reply":"2022-05-29T06:20:23.274681Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_1 = models.load_model('./model_1.h5')\nmodel_2 = models.load_model('./model_2.h5')\n\nprint(f'Loss & Accuracy on training data for model_1 : {model_1.evaluate(trainReshaped, to_categorical(train.iloc[:, :1]), batch_size=256)}')\nprint(f'Loss & Accuracy on training data for model_1 : {model_2.evaluate(trainReshaped, to_categorical(train.iloc[:, :1]), batch_size=256)}')","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:40:18.2521Z","iopub.execute_input":"2022-05-29T06:40:18.252527Z","iopub.status.idle":"2022-05-29T06:40:22.305664Z","shell.execute_reply.started":"2022-05-29T06:40:18.252491Z","shell.execute_reply":"2022-05-29T06:40:22.304619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred1 = model_1.predict(trainReshaped, batch_size=256)\npred2 = model_2.predict(trainReshaped, batch_size=256)\n\npred = numpy.hstack((pred1, pred2))","metadata":{"execution":{"iopub.status.busy":"2022-05-29T07:32:08.229729Z","iopub.execute_input":"2022-05-29T07:32:08.230001Z","iopub.status.idle":"2022-05-29T07:32:10.748391Z","shell.execute_reply.started":"2022-05-29T07:32:08.229971Z","shell.execute_reply":"2022-05-29T07:32:10.747623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(pred[0])","metadata":{"execution":{"iopub.status.busy":"2022-05-29T07:34:15.555337Z","iopub.execute_input":"2022-05-29T07:34:15.555631Z","iopub.status.idle":"2022-05-29T07:34:15.561714Z","shell.execute_reply.started":"2022-05-29T07:34:15.555599Z","shell.execute_reply":"2022-05-29T07:34:15.560875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- create a stacked classifier\n    - multi nomial logistic regression will be the meta classifier","metadata":{"execution":{"iopub.status.busy":"2022-05-29T07:20:24.671289Z","iopub.execute_input":"2022-05-29T07:20:24.671989Z","iopub.status.idle":"2022-05-29T07:20:24.677106Z","shell.execute_reply.started":"2022-05-29T07:20:24.67195Z","shell.execute_reply":"2022-05-29T07:20:24.67602Z"}}},{"cell_type":"code","source":"clf = LogisticRegression(random_state=42, penalty='elasticnet', multi_class='multinomial', solver='saga', l1_ratio=0.5).fit(pred, train.iloc[:, :1])\nclf.score(pred, train.iloc[:, :1])","metadata":{"execution":{"iopub.status.busy":"2022-05-29T07:35:04.724024Z","iopub.execute_input":"2022-05-29T07:35:04.724299Z","iopub.status.idle":"2022-05-29T07:35:07.593871Z","shell.execute_reply.started":"2022-05-29T07:35:04.724247Z","shell.execute_reply":"2022-05-29T07:35:07.592753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- now we are ready for prediction","metadata":{}},{"cell_type":"code","source":"testPred1 = model_1.predict(testReshaped, batch_size=256)\ntestPred2 = model_2.predict(testReshaped, batch_size=256)\n\ntestPred = numpy.hstack((testPred1, testPred2))","metadata":{"execution":{"iopub.status.busy":"2022-05-29T07:35:23.799923Z","iopub.execute_input":"2022-05-29T07:35:23.800184Z","iopub.status.idle":"2022-05-29T07:35:25.930167Z","shell.execute_reply.started":"2022-05-29T07:35:23.800155Z","shell.execute_reply":"2022-05-29T07:35:25.929350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"finalPred = clf.predict(testPred)\ndf = pandas.DataFrame(pandas.concat([pandas.Series(range(1,28001)), pandas.Series(finalPred)], axis=1))\ndf.rename({0:'ImageId', 1:'Label'}, axis=1, inplace=True)\ndf.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-29T07:36:10.079311Z","iopub.execute_input":"2022-05-29T07:36:10.079878Z","iopub.status.idle":"2022-05-29T07:36:10.165596Z","shell.execute_reply.started":"2022-05-29T07:36:10.079838Z","shell.execute_reply":"2022-05-29T07:36:10.164714Z"},"trusted":true},"execution_count":null,"outputs":[]}]}