{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n\nfrom sklearn.model_selection import train_test_split\n\nimport tensorflow as tf\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop\nfrom keras.callbacks import ReduceLROnPlateau\n\nnp.random.seed(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Data preparation"},{"metadata":{},"cell_type":"markdown","source":"## 1.1 Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the data\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"], axis = 1) # all except for label column\n\n\ng = sns.countplot(Y_train)\nY_train.value_counts() # ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have ap. even number of all digits"},{"metadata":{},"cell_type":"markdown","source":"## 1.2 Check for null values"},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.isnull().any().describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.isnull().any().describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"No missing values in the train and test datasets."},{"metadata":{},"cell_type":"markdown","source":"## 1.3 Grayscale Normalization"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from 0-255 to 0-1\nX_train = X_train / 255.0\ntest = test / 255.0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.4 Reshaping"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reshape image  (height = 28px, width = 28px , canal = 1)\n# use this reshaping for CNN in Section 2.1\n#X_train = X_train.values.reshape(-1,28,28,1)\n#test = test.values.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# use this reshape for \nX_train = X_train.values.reshape(-1,28,28)\ntest = test.values.reshape(-1,28,28)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"All data are in 28x28x1 3D matrices now (Keras requires one more dimension in the end which correspond to channels, only one channel is needed for gray scale images."},{"metadata":{},"cell_type":"markdown","source":"## 1.5 Label encoding "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encode labels to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0])\n#Y_train = to_categorical(Y_train, num_classes = 10) <- ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We need to encode these lables to one hot vectors (ex : 2 -> [0,0,1,0,0,0,0,0,0,0]).\n(only needed for model from Section 2.1, not for 2.2 !)"},{"metadata":{},"cell_type":"markdown","source":"## 1.6 Splitting into training and valid\nation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"random_seed = 2\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.15, random_state=random_seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.7 Example Images"},{"metadata":{"trusted":true},"cell_type":"code","source":"#g = plt.imshow(X_train[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Modeling\n## 2.1 CNN model "},{"metadata":{"trusted":true},"cell_type":"code","source":"## In -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*2 -> Flatten -> Dense -> Dropout -> Out\n#\n#model = Sequential()\n#model.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n#                 activation ='relu', input_shape = (28,28,1))) #32 filters for the first conv2D layer\n#model.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n#                 activation ='relu')) #32 filters for the 2nd conv2D layer\n#model.add(MaxPool2D(pool_size=(2,2))) #the area size pooled each time\n#model.add(Dropout(0.25)) #regularization parameter= proportion of nodes in the layer are randomly ignored (setting their weights to zero) for each training sample. \n#\n#model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n#                 activation ='relu'))\n#model.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n#                 activation ='relu'))\n#model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\n#model.add(Dropout(0.25))\n#\n#model.add(Flatten()) #to convert the final feature maps into a one single 1D vector\n#model.add(Dense(256, activation = \"relu\")) #activation function max(0,x), used to add non linearity to the network\n#model.add(Dropout(0.5))\n#model.add(Dense(10, activation = \"softmax\")) #outputs distribution of probability of each class","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define the optimizer\n# loss function and an optimisation algorithm\n#optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0) # faster than Stochastic Gradient Descent ('sgd') optimizer\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile the model\n#model.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set a learning rate annealer\n# reduce the LR by half if the accuracy is not improved after 3 epochs\n#learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n#                                            patience=3, \n#                                            verbose=1, \n#                                            factor=0.5, \n#                                            min_lr=0.00001)\n#epochs = 10 # \n#batch_size = 86","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#history = model.fit(X_train, Y_train, batch_size = batch_size, epochs = epochs, \n          validation_data = (X_val, Y_val), verbose = 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Validation accuracy = 0.9905"},{"metadata":{},"cell_type":"markdown","source":"## 2.2 Alternative (Simpler) Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"class myCallback(tf.keras.callbacks.Callback):\n      def on_epoch_end(self, epoch, logs={}):\n        if(logs.get('acc')>0.997):\n          print(\"\\nReached 99% accuracy so cancelling training!\")\n          self.model.stop_training = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"callbacks = myCallback()\nmodel2 = tf.keras.models.Sequential([\n        tf.keras.layers.Flatten(input_shape=(28, 28)),\n        tf.keras.layers.Dense(512, activation=tf.nn.relu),\n        tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n    ])\n\nmodel2.compile(optimizer='adam',\n                  loss='sparse_categorical_crossentropy',\n                  metrics=['accuracy'])\n    \n# model fitting\nhistory = model2.fit(X_train, Y_train, epochs=30, callbacks=[callbacks])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Submitting predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict results\nresults = model2.predict(test)\n\n# select the index with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}