{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import libraries","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-21T03:28:21.365156Z","iopub.execute_input":"2022-02-21T03:28:21.365489Z","iopub.status.idle":"2022-02-21T03:28:21.373819Z","shell.execute_reply.started":"2022-02-21T03:28:21.365455Z","shell.execute_reply":"2022-02-21T03:28:21.372861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import keras\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten, Activation, BatchNormalization\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.optimizers import RMSprop","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:21.375518Z","iopub.execute_input":"2022-02-21T03:28:21.37605Z","iopub.status.idle":"2022-02-21T03:28:23.887196Z","shell.execute_reply.started":"2022-02-21T03:28:21.376001Z","shell.execute_reply":"2022-02-21T03:28:23.886028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load the data","metadata":{}},{"cell_type":"code","source":"df_train = pd.pandas.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.pandas.read_csv('/kaggle/input/digit-recognizer/test.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:23.889107Z","iopub.execute_input":"2022-02-21T03:28:23.889368Z","iopub.status.idle":"2022-02-21T03:28:29.607024Z","shell.execute_reply.started":"2022-02-21T03:28:23.889334Z","shell.execute_reply":"2022-02-21T03:28:29.605759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.608805Z","iopub.execute_input":"2022-02-21T03:28:29.609153Z","iopub.status.idle":"2022-02-21T03:28:29.632751Z","shell.execute_reply.started":"2022-02-21T03:28:29.60907Z","shell.execute_reply":"2022-02-21T03:28:29.631785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-----------","metadata":{}},{"cell_type":"markdown","source":"# Preprocess the data","metadata":{}},{"cell_type":"code","source":"def get_shape(d):\n    print(f\"The shape is : {d.shape}\")","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.634651Z","iopub.execute_input":"2022-02-21T03:28:29.635588Z","iopub.status.idle":"2022-02-21T03:28:29.639256Z","shell.execute_reply.started":"2022-02-21T03:28:29.635552Z","shell.execute_reply":"2022-02-21T03:28:29.638465Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Missing values ","metadata":{}},{"cell_type":"code","source":"df_train.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.640791Z","iopub.execute_input":"2022-02-21T03:28:29.641133Z","iopub.status.idle":"2022-02-21T03:28:29.670923Z","shell.execute_reply.started":"2022-02-21T03:28:29.641065Z","shell.execute_reply":"2022-02-21T03:28:29.670229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.672061Z","iopub.execute_input":"2022-02-21T03:28:29.672515Z","iopub.status.idle":"2022-02-21T03:28:29.692382Z","shell.execute_reply.started":"2022-02-21T03:28:29.672479Z","shell.execute_reply":"2022-02-21T03:28:29.691373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### There is no missing values.\n----------","metadata":{}},{"cell_type":"markdown","source":"## Create X, y for train and test","metadata":{}},{"cell_type":"code","source":"y = df_train['label']\nX = df_train.drop(labels = [\"label\"],axis = 1) \nx_test = df_test","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.693685Z","iopub.execute_input":"2022-02-21T03:28:29.694021Z","iopub.status.idle":"2022-02-21T03:28:29.801257Z","shell.execute_reply.started":"2022-02-21T03:28:29.693966Z","shell.execute_reply":"2022-02-21T03:28:29.800319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Num of categories","metadata":{}},{"cell_type":"code","source":"categories = len(pd.unique(df_train['label']))\ncategories","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.802293Z","iopub.execute_input":"2022-02-21T03:28:29.803035Z","iopub.status.idle":"2022-02-21T03:28:29.810626Z","shell.execute_reply.started":"2022-02-21T03:28:29.803002Z","shell.execute_reply":"2022-02-21T03:28:29.809415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### y to categorical","metadata":{}},{"cell_type":"code","source":"y = tf.keras.utils.to_categorical(y, categories)\nget_shape(y)","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.812424Z","iopub.execute_input":"2022-02-21T03:28:29.813348Z","iopub.status.idle":"2022-02-21T03:28:29.822968Z","shell.execute_reply.started":"2022-02-21T03:28:29.81329Z","shell.execute_reply":"2022-02-21T03:28:29.822226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reshape X","metadata":{}},{"cell_type":"code","source":"X = X.values.reshape(-1,28,28,1)\nx_test = x_test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.82575Z","iopub.execute_input":"2022-02-21T03:28:29.826147Z","iopub.status.idle":"2022-02-21T03:28:29.834491Z","shell.execute_reply.started":"2022-02-21T03:28:29.8261Z","shell.execute_reply":"2022-02-21T03:28:29.833306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalizing","metadata":{}},{"cell_type":"code","source":"X.max(axis = 0).max()","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.836046Z","iopub.execute_input":"2022-02-21T03:28:29.836303Z","iopub.status.idle":"2022-02-21T03:28:29.885782Z","shell.execute_reply.started":"2022-02-21T03:28:29.836272Z","shell.execute_reply":"2022-02-21T03:28:29.884735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = X.astype('float' ) / 255\nx_test= x_test.astype('float' ) / 255","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:29.88768Z","iopub.execute_input":"2022-02-21T03:28:29.888001Z","iopub.status.idle":"2022-02-21T03:28:30.15703Z","shell.execute_reply.started":"2022-02-21T03:28:29.88796Z","shell.execute_reply":"2022-02-21T03:28:30.156207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Split X, y into train and validation","metadata":{}},{"cell_type":"code","source":"x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.15, random_state=44)","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:30.158675Z","iopub.execute_input":"2022-02-21T03:28:30.159559Z","iopub.status.idle":"2022-02-21T03:28:30.720731Z","shell.execute_reply.started":"2022-02-21T03:28:30.159515Z","shell.execute_reply":"2022-02-21T03:28:30.719501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"----------","metadata":{}},{"cell_type":"markdown","source":"# Create the model","metadata":{}},{"cell_type":"code","source":"def create_model(kernal, pool_size, in_shape, opti, categories):\n    model = Sequential()\n    # Input and 1st hidden layer\n    model.add(Conv2D(32, kernal, activation='relu', input_shape=in_shape))\n    # BatchNormalization Layer normalizes its inputs\n    model.add(BatchNormalization())\n    # The pooling window reduce the spatial volume of input image after convolution(moves for each pooling step(pool_size here because strides is none))\n    model.add(MaxPooling2D(pool_size=pool_size))\n    # The Dropout layer randomly sets input units to 0 with a frequency of rate at each step during training time, which helps prevent overfitting.\n    # p is the probability of the activations to the shut off.\n    # if p is too small (p = 0.01) --> overfitting  , too larg (p = 0.9) --> underfitting\n    # It'll be turned off for the evaluation\n    model.add(Dropout(0.25))\n    \n    # 2nd hidden layer\n    model.add(Conv2D(64, kernal, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(MaxPooling2D(pool_size=pool_size))\n    model.add(Dropout(0.25))\n    \n    # Final hidden layer and output layer\n    model.add(Flatten())\n    model.add(Dense(256, activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.5))\n    model.add(Dense(categories, activation='softmax'))\n    \n    # Compile the model\n    model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])\n    return model\n  ","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:30.722375Z","iopub.execute_input":"2022-02-21T03:28:30.722739Z","iopub.status.idle":"2022-02-21T03:28:30.734241Z","shell.execute_reply.started":"2022-02-21T03:28:30.722691Z","shell.execute_reply":"2022-02-21T03:28:30.732941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hyperparameters","metadata":{}},{"cell_type":"markdown","source":"#### Input shape:\n##### 28 * 28 = 784 (0 - 783 columns) and there's 1 channel.","metadata":{}},{"cell_type":"code","source":"in_shape = (28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:30.735702Z","iopub.execute_input":"2022-02-21T03:28:30.736389Z","iopub.status.idle":"2022-02-21T03:28:30.750893Z","shell.execute_reply.started":"2022-02-21T03:28:30.736352Z","shell.execute_reply":"2022-02-21T03:28:30.75003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def optimizer(learning_rate):\n    optm = RMSprop(learning_rate=learning_rate)\n    return optm","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:30.752057Z","iopub.execute_input":"2022-02-21T03:28:30.752758Z","iopub.status.idle":"2022-02-21T03:28:30.762128Z","shell.execute_reply.started":"2022-02-21T03:28:30.752721Z","shell.execute_reply":"2022-02-21T03:28:30.761232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kernal = (3, 3)\npool_size = (2, 2)\nlr = 0.001\noptm = optimizer(lr)","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:30.764177Z","iopub.execute_input":"2022-02-21T03:28:30.764455Z","iopub.status.idle":"2022-02-21T03:28:30.774606Z","shell.execute_reply.started":"2022-02-21T03:28:30.764422Z","shell.execute_reply":"2022-02-21T03:28:30.773443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = create_model(kernal, pool_size, in_shape, optm, categories)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:30.77601Z","iopub.execute_input":"2022-02-21T03:28:30.7764Z","iopub.status.idle":"2022-02-21T03:28:30.930278Z","shell.execute_reply.started":"2022-02-21T03:28:30.776368Z","shell.execute_reply":"2022-02-21T03:28:30.929257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train the model","metadata":{}},{"cell_type":"code","source":"batch_size = 128\nepochs = 10\nhist = model.fit(x_train, y_train, batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid))\nmodel.save('mnist_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:28:30.93203Z","iopub.execute_input":"2022-02-21T03:28:30.932704Z","iopub.status.idle":"2022-02-21T03:31:54.546283Z","shell.execute_reply.started":"2022-02-21T03:28:30.932669Z","shell.execute_reply":"2022-02-21T03:31:54.545275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Accuracy vs Loss","metadata":{}},{"cell_type":"code","source":"fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 8))\nax1.plot(hist.history['loss'], color='b', label=\"Training loss\")\nax1.plot(hist.history['val_loss'], color='r', label=\"validation loss\")\nax1.set_xticks(np.arange(1, epochs, 1))\nax2.plot(hist.history['accuracy'], color='b', label=\"Training accuracy\")\nax2.plot(hist.history['val_accuracy'], color='r',label=\"Validation accuracy\")\nax2.set_xticks(np.arange(1, epochs, 1))\nlegend = plt.legend(loc='best', shadow=True)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:33:31.684244Z","iopub.execute_input":"2022-02-21T03:33:31.684585Z","iopub.status.idle":"2022-02-21T03:33:32.116236Z","shell.execute_reply.started":"2022-02-21T03:33:31.68455Z","shell.execute_reply":"2022-02-21T03:33:32.115078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evalution","metadata":{}},{"cell_type":"code","source":"yhat = model.predict(x_test)\n\nyhat = np.argmax(yhat, axis = 1)\n\nyhat = pd.Series(yhat, name=\"Label\")","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:36:07.30291Z","iopub.execute_input":"2022-02-21T03:36:07.303267Z","iopub.status.idle":"2022-02-21T03:36:11.706882Z","shell.execute_reply.started":"2022-02-21T03:36:07.30323Z","shell.execute_reply":"2022-02-21T03:36:11.706022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Save the predictions","metadata":{}},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"), yhat],axis = 1)\n\nsubmission.to_csv(\"mnist_submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-02-21T03:48:37.112293Z","iopub.execute_input":"2022-02-21T03:48:37.113074Z","iopub.status.idle":"2022-02-21T03:48:37.166476Z","shell.execute_reply.started":"2022-02-21T03:48:37.113033Z","shell.execute_reply":"2022-02-21T03:48:37.165421Z"},"trusted":true},"execution_count":null,"outputs":[]}]}