{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Digit Recognizer\n\n2 Models mentioned below will be used for training:\n\n1. NN - Simple 3 layer Neural Network - Keras\n2. CNN - Convolutional Neural Network - Keras"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport numpy as np\nimport keras\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D\nfrom keras.optimizers import SGD\nfrom keras.callbacks import ReduceLROnPlateau\nfrom keras.preprocessing.image import ImageDataGenerator\n\n\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nimport os\nprint(os.listdir(\"../input\"))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 1. Importing Dataset"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Preprocessing"},{"metadata":{"trusted":false},"cell_type":"code","source":"X_train = train.drop(labels = [\"label\"], axis=1)\nY = train[\"label\"]\nY_train = keras.utils.to_categorical(Y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.1 Normalize features"},{"metadata":{"trusted":false},"cell_type":"code","source":"X_train = X_train / 255.\ntest = test / 255.\n# convert Y_train to one_hot\ndel train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.2 Check if there is any missing data"},{"metadata":{"trusted":false},"cell_type":"code","source":"print(np.sum(np.isnan(X_train.values)))\nprint(np.sum(np.isnan(test.values)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.3 Plot some examples"},{"metadata":{"trusted":false},"cell_type":"code","source":"fig = plt.figure(figsize=(10,10))\n\nfor i in range(20):\n    plt.subplot(5,4,i+1)\n    plt.imshow(X_train.iloc[i].values.reshape(28,28),cmap=plt.cm.binary)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2.4 Analyse the class distributions"},{"metadata":{"trusted":false},"cell_type":"code","source":"sns.countplot(Y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Models"},{"metadata":{},"cell_type":"markdown","source":"## Model 1 - Simple Neural Network (Keras)\n\nAccuracy on Kaggle learderboard: **%97.2**\n\nModel has the features mentioned below:\n* 3 layers (2 hidden + 1 output) - 784 --> (64,64) -->  (10)\n* Learning Rate Reduction - LR will decrease when validation accuracy is not improved for 2 epochs in a row\n* Regularization - 2 hidden layers will have L2 regularization"},{"metadata":{"trusted":false},"cell_type":"code","source":"learning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=2, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras import regularizers\nmodel = Sequential()\nmodel.add(Dense(64, activation='relu', input_dim=784, kernel_regularizer=regularizers.l2(0.001)))\nmodel.add(Dense(64, activation='relu', kernel_regularizer=regularizers.l2(0.001)))\nmodel.add(Dense(10, activation='softmax'))\nmodel.compile(optimizer='adam',\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# trainining\nhistory = model.fit(X_train, Y_train, epochs=40, batch_size=128, validation_split=0.1, callbacks = [learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# evaluate model on training data\nres = model.evaluate(X_train, Y_train)\nprint(\"Model accuracy on training data is: %{}\".format(res[1]*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Create submission\nsubmission_df = pd.read_csv(\"../input/sample_submission.csv\", index_col=0)\nprobs = model.predict(test)\nsubmission_df['Label'] = np.argmax(probs,axis=1)\nsubmission_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model 2 - Convolutional Neural Network (Keras)\n\nAccuracy on Kaggle learderboard: **%99.3** \n\nModel used is a VGG-like convnet from Keras website:\n\n"},{"metadata":{"trusted":false},"cell_type":"code","source":"model = Sequential()\n\nmodel.add(Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\nmodel.add(Conv2D(32, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(Conv2D(64, (3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation='softmax'))\n\nsgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)\nmodel.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer=sgd)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"X_train_CNN = X_train.values.reshape((-1,28,28,1))\nX_train_CNN.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"model.fit(X_train_CNN, Y_train, batch_size=128, epochs=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# evaluate model on training data\nres2 = model.evaluate(X_train_CNN, Y_train)\nprint(\"Model accuracy on training data is: %{}\".format(res2[1]*100))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Alternatively augmentation can be used to train the model."},{"metadata":{"trusted":false},"cell_type":"code","source":"# datagen = ImageDataGenerator(\n#     rotation_range = 15,\n#     zca_whitening = True,\n#     width_shift_range=0.15,\n#     height_shift_range=0.15,\n#     )\n# datagen.fit(X_train_CNN)\n\n# history = model.fit_generator(datagen.flow(X_train_CNN,Y_train, batch_size=32),\n#                               epochs = 10,\n#                               verbose = 1, steps_per_epoch=steps_per_epoch=X_train_CNN.shape[0] // 32)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Create Submission"},{"metadata":{"trusted":false},"cell_type":"code","source":"submission_df = pd.read_csv(\"../input/sample_submission.csv\", index_col=0)\nprobs = model.predict(test.values.reshape((-1,28,28,1)))\nsubmission_df['Label'] = np.argmax(probs,axis=1)\nsubmission_df.head()\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}