{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport matplotlib.pyplot as plt\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.status.busy":"2021-05-26T17:25:43.380354Z","iopub.execute_input":"2021-05-26T17:25:43.380677Z","iopub.status.idle":"2021-05-26T17:25:43.388147Z","shell.execute_reply.started":"2021-05-26T17:25:43.380616Z","shell.execute_reply":"2021-05-26T17:25:43.387212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.utils.multiclass import unique_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:43.391811Z","iopub.execute_input":"2021-05-26T17:25:43.392331Z","iopub.status.idle":"2021-05-26T17:25:44.227604Z","shell.execute_reply.started":"2021-05-26T17:25:43.392276Z","shell.execute_reply":"2021-05-26T17:25:44.226677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import confusion_matrix\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.preprocessing.image import ImageDataGenerator","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:44.231062Z","iopub.execute_input":"2021-05-26T17:25:44.231326Z","iopub.status.idle":"2021-05-26T17:25:45.114182Z","shell.execute_reply.started":"2021-05-26T17:25:44.23128Z","shell.execute_reply":"2021-05-26T17:25:45.113336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\n","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2021-05-26T17:25:45.115437Z","iopub.execute_input":"2021-05-26T17:25:45.115678Z","iopub.status.idle":"2021-05-26T17:25:49.400365Z","shell.execute_reply.started":"2021-05-26T17:25:45.115634Z","shell.execute_reply":"2021-05-26T17:25:49.399568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:49.402628Z","iopub.execute_input":"2021-05-26T17:25:49.402863Z","iopub.status.idle":"2021-05-26T17:25:52.211503Z","shell.execute_reply.started":"2021-05-26T17:25:49.402824Z","shell.execute_reply":"2021-05-26T17:25:52.210469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:52.212868Z","iopub.execute_input":"2021-05-26T17:25:52.213361Z","iopub.status.idle":"2021-05-26T17:25:52.330924Z","shell.execute_reply.started":"2021-05-26T17:25:52.213153Z","shell.execute_reply":"2021-05-26T17:25:52.330348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Our X will be all the featurs, in this case they are the pixels\n* Our Y are the labels for each number\n* We need to normalize X by dividing it by 255.\n* Then we need to convert Y to one-hot encoded vector.\n* We will cut the data into two parts, train and test\n","metadata":{}},{"cell_type":"code","source":"Y = df['label']\nX = df.drop(['label'], axis = 1)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:52.331859Z","iopub.execute_input":"2021-05-26T17:25:52.332226Z","iopub.status.idle":"2021-05-26T17:25:52.536995Z","shell.execute_reply.started":"2021-05-26T17:25:52.332176Z","shell.execute_reply":"2021-05-26T17:25:52.536065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X / 255.0\nX = X.values.reshape(-1,28,28,1)\nY = np.array(Y)\nY = to_categorical(Y, num_classes = 10)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:52.538089Z","iopub.execute_input":"2021-05-26T17:25:52.53846Z","iopub.status.idle":"2021-05-26T17:25:53.012937Z","shell.execute_reply.started":"2021-05-26T17:25:52.538411Z","shell.execute_reply":"2021-05-26T17:25:53.012303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X.shape)\nprint(Y.shape)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:53.01387Z","iopub.execute_input":"2021-05-26T17:25:53.014241Z","iopub.status.idle":"2021-05-26T17:25:53.018353Z","shell.execute_reply.started":"2021-05-26T17:25:53.014189Z","shell.execute_reply":"2021-05-26T17:25:53.017734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, Y_train, Y_test = train_test_split(X, Y, stratify = Y, random_state = 31, test_size = 0.2)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:53.019227Z","iopub.execute_input":"2021-05-26T17:25:53.019557Z","iopub.status.idle":"2021-05-26T17:25:54.060216Z","shell.execute_reply.started":"2021-05-26T17:25:53.019511Z","shell.execute_reply":"2021-05-26T17:25:54.059236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(type(X_train), type(X_test) , type(Y_train),type( Y_test))","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.061601Z","iopub.execute_input":"2021-05-26T17:25:54.061844Z","iopub.status.idle":"2021-05-26T17:25:54.067055Z","shell.execute_reply.started":"2021-05-26T17:25:54.061802Z","shell.execute_reply":"2021-05-26T17:25:54.065854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g = plt.imshow(X_train[0][:,:,0])","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.067892Z","iopub.execute_input":"2021-05-26T17:25:54.06814Z","iopub.status.idle":"2021-05-26T17:25:54.334976Z","shell.execute_reply.started":"2021-05-26T17:25:54.068085Z","shell.execute_reply":"2021-05-26T17:25:54.333781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.336899Z","iopub.execute_input":"2021-05-26T17:25:54.337317Z","iopub.status.idle":"2021-05-26T17:25:54.367387Z","shell.execute_reply.started":"2021-05-26T17:25:54.337242Z","shell.execute_reply":"2021-05-26T17:25:54.366442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.add(Conv2D(32,(5,5),padding = 'Same',activation ='relu', input_shape = (28,28,1)))\n\nmodel.add(Conv2D(64,(3,3), padding = 'same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2)))\nmodel.add(Dropout(0.2))\n\nmodel.add(Conv2D(128,(3,3), padding = 'same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2)))\nmodel.add(Dropout(0.2))\n\nmodel.add(Conv2D(64,(3,3), padding = 'same', activation = 'relu'))\nmodel.add(MaxPool2D(pool_size = (2,2)))\nmodel.add(Dropout(0.2))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = 'relu'))\nmodel.add(Dropout(0.2))\nmodel.add(Dense(10, activation = 'softmax'))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.36903Z","iopub.execute_input":"2021-05-26T17:25:54.369666Z","iopub.status.idle":"2021-05-26T17:25:54.643755Z","shell.execute_reply.started":"2021-05-26T17:25:54.369604Z","shell.execute_reply":"2021-05-26T17:25:54.642889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* We are trying to augumgent the data.\n* Try zooming in and zooming out.\n* We are also shifting the numbers a bit left and right by width_shift and height_shift\n* Also we cannot do horizontal and vertical flip as it will convert 6 to 9 and make 4 look very odd.\n","metadata":{}},{"cell_type":"code","source":"# Maybe we can try augementing the data to improve the accuracy\ndatagen = ImageDataGenerator(featurewise_center=False, \n                             samplewise_center=False, \n                             featurewise_std_normalization=False, \n                             samplewise_std_normalization=False, \n                             zca_whitening=False, \n                             zca_epsilon=1e-06, \n                             rotation_range=10, \n                             width_shift_range=0.1, \n                             height_shift_range=0.1, \n                             brightness_range=None, \n                             shear_range=0.1, \n                             zoom_range=0.15, \n                             channel_shift_range=0.0, \n                             fill_mode='nearest', \n                             cval=0.0, \n                             horizontal_flip=False, \n                             vertical_flip=False, \n                             rescale=None, \n                             preprocessing_function=None, \n                             data_format=None, validation_split=0.0, dtype=None)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.645108Z","iopub.execute_input":"2021-05-26T17:25:54.645416Z","iopub.status.idle":"2021-05-26T17:25:54.655863Z","shell.execute_reply.started":"2021-05-26T17:25:54.645356Z","shell.execute_reply":"2021-05-26T17:25:54.654562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.657062Z","iopub.execute_input":"2021-05-26T17:25:54.657407Z","iopub.status.idle":"2021-05-26T17:25:54.832412Z","shell.execute_reply.started":"2021-05-26T17:25:54.657356Z","shell.execute_reply":"2021-05-26T17:25:54.831789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Note that optimizer is NOT the normal Adam.\n* We are using Nestrov accelerated Adam\n* You can find more details here https://keras.io/optimizers/\n* Also in tf https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/Nadam\n* It already has learning_rate decay and it is recommended not to tune its hyperparamters.\n* Simple so we are ready to use it directly.","metadata":{}},{"cell_type":"code","source":"model.compile(optimizer = \"Nadam\", loss = \"categorical_crossentropy\", metrics = [\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.833503Z","iopub.execute_input":"2021-05-26T17:25:54.833912Z","iopub.status.idle":"2021-05-26T17:25:54.877319Z","shell.execute_reply.started":"2021-05-26T17:25:54.833852Z","shell.execute_reply":"2021-05-26T17:25:54.876528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Please try adjusting epochs for further improvement.\n* Also adding regularization may help.","metadata":{}},{"cell_type":"code","source":"max_epochs = 30\nbatch_size = 256","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.878241Z","iopub.execute_input":"2021-05-26T17:25:54.878624Z","iopub.status.idle":"2021-05-26T17:25:54.881521Z","shell.execute_reply.started":"2021-05-26T17:25:54.878572Z","shell.execute_reply":"2021-05-26T17:25:54.880905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size = batch_size) ,\n                              epochs = max_epochs, verbose = 1, validation_data = (X_test, Y_test),\n                              steps_per_epoch=X_train.shape[0] // batch_size)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T17:25:54.882455Z","iopub.execute_input":"2021-05-26T17:25:54.882797Z","iopub.status.idle":"2021-05-26T18:37:47.502407Z","shell.execute_reply.started":"2021-05-26T17:25:54.882747Z","shell.execute_reply":"2021-05-26T18:37:47.501462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['acc']\nval_acc = history.history['val_acc']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\n\nepochs = range(1, len(acc) + 1)\n\nplt.plot(epochs, acc, 'g', label='Training acc')\nplt.plot(epochs, val_acc, 'b', label='Validation acc')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\n\nplt.plot(epochs, loss, 'g', label='Training loss')\nplt.plot(epochs, val_loss, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:47.503376Z","iopub.execute_input":"2021-05-26T18:37:47.503667Z","iopub.status.idle":"2021-05-26T18:37:48.027644Z","shell.execute_reply.started":"2021-05-26T18:37:47.503605Z","shell.execute_reply":"2021-05-26T18:37:48.026414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:48.030816Z","iopub.execute_input":"2021-05-26T18:37:48.031151Z","iopub.status.idle":"2021-05-26T18:37:48.145226Z","shell.execute_reply.started":"2021-05-26T18:37:48.031099Z","shell.execute_reply":"2021-05-26T18:37:48.144316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df / 255.0\ntest_arr = test_df.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:48.146632Z","iopub.execute_input":"2021-05-26T18:37:48.147068Z","iopub.status.idle":"2021-05-26T18:37:48.37711Z","shell.execute_reply.started":"2021-05-26T18:37:48.146909Z","shell.execute_reply":"2021-05-26T18:37:48.37602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_arr = model.predict(test_arr)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:37:48.378313Z","iopub.execute_input":"2021-05-26T18:37:48.37856Z","iopub.status.idle":"2021-05-26T18:38:10.595378Z","shell.execute_reply.started":"2021-05-26T18:37:48.378518Z","shell.execute_reply":"2021-05-26T18:38:10.594337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_arr.shape","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:38:10.596665Z","iopub.execute_input":"2021-05-26T18:38:10.597017Z","iopub.status.idle":"2021-05-26T18:38:10.603176Z","shell.execute_reply.started":"2021-05-26T18:38:10.59693Z","shell.execute_reply":"2021-05-26T18:38:10.602118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Y_pred_arr = np.argmax(Y_pred_arr, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:38:10.604236Z","iopub.execute_input":"2021-05-26T18:38:10.604481Z","iopub.status.idle":"2021-05-26T18:38:10.623822Z","shell.execute_reply.started":"2021-05-26T18:38:10.604429Z","shell.execute_reply":"2021-05-26T18:38:10.622923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.Series(Y_pred_arr,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-26T18:38:10.62491Z","iopub.execute_input":"2021-05-26T18:38:10.625675Z","iopub.status.idle":"2021-05-26T18:38:11.033657Z","shell.execute_reply.started":"2021-05-26T18:38:10.625617Z","shell.execute_reply":"2021-05-26T18:38:11.032874Z"},"trusted":true},"execution_count":null,"outputs":[]}]}