{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# data analyses and wrangling\nimport numpy as np\nimport pandas as pd\n\n# data visualisation\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\n# machine learning\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nimport tensorflow as tf\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau\nfrom tensorflow.keras.optimizers import RMSprop \n\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:35.413155Z","iopub.execute_input":"2021-09-27T22:23:35.413846Z","iopub.status.idle":"2021-09-27T22:23:37.650700Z","shell.execute_reply.started":"2021-09-27T22:23:35.413717Z","shell.execute_reply":"2021-09-27T22:23:37.649711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:37.652534Z","iopub.execute_input":"2021-09-27T22:23:37.652795Z","iopub.status.idle":"2021-09-27T22:23:43.381595Z","shell.execute_reply.started":"2021-09-27T22:23:37.652764Z","shell.execute_reply":"2021-09-27T22:23:43.380638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:43.382954Z","iopub.execute_input":"2021-09-27T22:23:43.383249Z","iopub.status.idle":"2021-09-27T22:23:43.411007Z","shell.execute_reply.started":"2021-09-27T22:23:43.383163Z","shell.execute_reply":"2021-09-27T22:23:43.410158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:43.412226Z","iopub.execute_input":"2021-09-27T22:23:43.412974Z","iopub.status.idle":"2021-09-27T22:23:43.470570Z","shell.execute_reply.started":"2021-09-27T22:23:43.412940Z","shell.execute_reply":"2021-09-27T22:23:43.469987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = train.drop(labels=['label'],axis=1)\ny_train = train['label']\n\n# free some space\ndel train","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:43.472396Z","iopub.execute_input":"2021-09-27T22:23:43.473137Z","iopub.status.idle":"2021-09-27T22:23:43.580640Z","shell.execute_reply.started":"2021-09-27T22:23:43.473090Z","shell.execute_reply":"2021-09-27T22:23:43.579333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.value_counts(sort=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:43.581891Z","iopub.execute_input":"2021-09-27T22:23:43.582148Z","iopub.status.idle":"2021-09-27T22:23:43.597015Z","shell.execute_reply.started":"2021-09-27T22:23:43.582119Z","shell.execute_reply":"2021-09-27T22:23:43.596125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(12,8))\nsns.countplot(y_train)\nplt.xlabel='label'\nplt.ylabel='count'\nplt.title='count of labels'","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:43.598246Z","iopub.execute_input":"2021-09-27T22:23:43.598477Z","iopub.status.idle":"2021-09-27T22:23:44.055908Z","shell.execute_reply.started":"2021-09-27T22:23:43.598451Z","shell.execute_reply":"2021-09-27T22:23:44.055079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:44.057260Z","iopub.execute_input":"2021-09-27T22:23:44.057735Z","iopub.status.idle":"2021-09-27T22:23:44.081529Z","shell.execute_reply.started":"2021-09-27T22:23:44.057693Z","shell.execute_reply":"2021-09-27T22:23:44.080595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Normalization","metadata":{}},{"cell_type":"code","source":"X_train = X_train / 255.0\ntest = test / 255.0","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:44.082919Z","iopub.execute_input":"2021-09-27T22:23:44.083584Z","iopub.status.idle":"2021-09-27T22:23:44.194995Z","shell.execute_reply.started":"2021-09-27T22:23:44.083538Z","shell.execute_reply":"2021-09-27T22:23:44.194284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reshape","metadata":{}},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\nX_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:44.196036Z","iopub.execute_input":"2021-09-27T22:23:44.196378Z","iopub.status.idle":"2021-09-27T22:23:44.201654Z","shell.execute_reply.started":"2021-09-27T22:23:44.196328Z","shell.execute_reply":"2021-09-27T22:23:44.200810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Label encoding","metadata":{}},{"cell_type":"code","source":"y_train = to_categorical(y_train, num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:44.203298Z","iopub.execute_input":"2021-09-27T22:23:44.203689Z","iopub.status.idle":"2021-09-27T22:23:44.214011Z","shell.execute_reply.started":"2021-09-27T22:23:44.203650Z","shell.execute_reply":"2021-09-27T22:23:44.213187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Split training and valdiation set","metadata":{}},{"cell_type":"code","source":"X_train, X_val, Y_train, Y_val = train_test_split(X_train, y_train, test_size = 0.1, random_state=101)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:44.215352Z","iopub.execute_input":"2021-09-27T22:23:44.215911Z","iopub.status.idle":"2021-09-27T22:23:44.621433Z","shell.execute_reply.started":"2021-09-27T22:23:44.215877Z","shell.execute_reply":"2021-09-27T22:23:44.620421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_train[0][:,:,0])","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:44.623030Z","iopub.execute_input":"2021-09-27T22:23:44.623398Z","iopub.status.idle":"2021-09-27T22:23:44.833267Z","shell.execute_reply.started":"2021-09-27T22:23:44.623357Z","shell.execute_reply":"2021-09-27T22:23:44.832277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Define the model","metadata":{}},{"cell_type":"code","source":"model = tf.keras.models.Sequential([\n    tf.keras.layers.Conv2D(filters=32, kernel_size=(5,5), padding='same', activation='relu', input_shape = (28,28,1) ),\n    tf.keras.layers.Conv2D(32, (5,5), padding='same', activation='relu'),\n    tf.keras.layers.MaxPool2D(pool_size=(2, 2)),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Conv2D(filters=64, kernel_size=(3,3), padding='same', activation='relu'),\n    tf.keras.layers.Conv2D(64, (3,3), padding='same', activation='relu'),\n    tf.keras.layers.MaxPool2D(pool_size=(2, 2), strides=(2,2)),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(256, activation='relu'),\n    tf.keras.layers.Dense(128, activation='relu'),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(10, activation='softmax')\n    \n])","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:44.836881Z","iopub.execute_input":"2021-09-27T22:23:44.837168Z","iopub.status.idle":"2021-09-27T22:23:45.010539Z","shell.execute_reply.started":"2021-09-27T22:23:44.837136Z","shell.execute_reply":"2021-09-27T22:23:45.009463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:45.011663Z","iopub.execute_input":"2021-09-27T22:23:45.011900Z","iopub.status.idle":"2021-09-27T22:23:45.016650Z","shell.execute_reply.started":"2021-09-27T22:23:45.011871Z","shell.execute_reply":"2021-09-27T22:23:45.015683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:45.017917Z","iopub.execute_input":"2021-09-27T22:23:45.018131Z","iopub.status.idle":"2021-09-27T22:23:45.041255Z","shell.execute_reply.started":"2021-09-27T22:23:45.018106Z","shell.execute_reply":"2021-09-27T22:23:45.040132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(\n    monitor='val_acc', \n    patience=3, \n    verbose=1, \n    factor=0.5, \n    min_lr=0.00001)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:45.042809Z","iopub.execute_input":"2021-09-27T22:23:45.043044Z","iopub.status.idle":"2021-09-27T22:23:45.049167Z","shell.execute_reply.started":"2021-09-27T22:23:45.043017Z","shell.execute_reply":"2021-09-27T22:23:45.048410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 30 \nbatch_size = 86","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:45.051026Z","iopub.execute_input":"2021-09-27T22:23:45.051611Z","iopub.status.idle":"2021-09-27T22:23:45.060763Z","shell.execute_reply.started":"2021-09-27T22:23:45.051577Z","shell.execute_reply":"2021-09-27T22:23:45.060138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:23:45.061696Z","iopub.execute_input":"2021-09-27T22:23:45.062214Z","iopub.status.idle":"2021-09-27T22:23:45.166780Z","shell.execute_reply.started":"2021-09-27T22:23:45.062159Z","shell.execute_reply":"2021-09-27T22:23:45.165893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val),\n                              verbose = 1, steps_per_epoch=batch_size\n                              , callbacks=[learning_rate_reduction])","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:25:58.339212Z","iopub.execute_input":"2021-09-27T22:25:58.339863Z","iopub.status.idle":"2021-09-27T22:41:19.414772Z","shell.execute_reply.started":"2021-09-27T22:25:58.339823Z","shell.execute_reply":"2021-09-27T22:41:19.414159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:41:26.313216Z","iopub.execute_input":"2021-09-27T22:41:26.313862Z","iopub.status.idle":"2021-09-27T22:41:26.326138Z","shell.execute_reply.started":"2021-09-27T22:41:26.313822Z","shell.execute_reply":"2021-09-27T22:41:26.325162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predict results\nresults = model.predict(test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:41:26.951886Z","iopub.execute_input":"2021-09-27T22:41:26.952193Z","iopub.status.idle":"2021-09-27T22:41:50.092336Z","shell.execute_reply.started":"2021-09-27T22:41:26.952148Z","shell.execute_reply":"2021-09-27T22:41:50.091592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"my_cnn_mnist_datagen.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-27T22:41:50.094530Z","iopub.execute_input":"2021-09-27T22:41:50.095164Z","iopub.status.idle":"2021-09-27T22:41:50.158575Z","shell.execute_reply.started":"2021-09-27T22:41:50.095119Z","shell.execute_reply":"2021-09-27T22:41:50.157615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Done!","metadata":{}}]}