{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\n\nnp.random.seed(2)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\n\nsns.set(style='white', context='notebook', palette='deep')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"f5aa157f0af8cff84ce5a4defab8f9f77d58fcdf"},"cell_type":"code","source":"X_train = train.drop(\"label\",axis=1)\nY_train = train[\"label\"]\nX_test  = test.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cca6ec2022de8511abe1125c25dcd0c0fd7c0cf"},"cell_type":"code","source":"from sklearn.preprocessing import MinMaxScaler\nscaler = MinMaxScaler()\n# Fit only to the training data\nscaler.fit(X_train)\nX_train = scaler.transform(X_train)\nscaler.fit(X_test)\nX_test = scaler.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"0ad772aef18483af0773074516698f42bb8919df"},"cell_type":"code","source":"X_train = X_train.reshape(-1, 28, 28, 1)\nX_test = X_test.reshape(-1, 28, 28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f38e98bf0eeceeca65479d3e9806271acaedd13a"},"cell_type":"code","source":"from keras.utils import to_categorical\nY_train = to_categorical(Y_train, num_classes= 10)\nrandom_seed = 2\n# Split the train and the validation set for the fitting\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=random_seed)\n#y_test_oneHot = to_categorical(Y_test, num_classes=10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12582717e6b1906038d6c11df796cf033140e1c5"},"cell_type":"code","source":"from keras.layers import Convolution2D, Dense, MaxPooling2D, Dropout, Flatten\nfrom keras.models import Sequential\n\nmodel = Sequential()\nmodel.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same', input_shape = (28, 28, 1), activation='relu'))\nmodel.add(Convolution2D(filters=32, kernel_size=(5, 5), padding='same', activation='relu'))       \nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.3))\n\nmodel.add(Convolution2D(filters=64, kernel_size=(5, 5), padding='same', activation='relu'))       \nmodel.add(Convolution2D(filters=64, kernel_size=(5, 5), padding='same', activation='relu'))       \nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.3))\n\nmodel.add(Flatten())\nmodel.add(Dense(units=256, activation='relu',))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(units=10, activation='softmax'))\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9c7b972736eda5d0fffd7975333557685c634c8a"},"cell_type":"code","source":"# optimizer = 'adam'\n#model.compile(optimizer='adam', loss = 'categorical_crossentropy', metrics=['accuracy'])\n# optimizer = 'RMSprop'\nmodel.compile(optimizer='RMSprop', loss = 'categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d180c1f4edf291246b3ae2fc05d276e38929ed76"},"cell_type":"code","source":"from keras.callbacks import ReduceLROnPlateau\n# To keep the advantage of the fast computation time with a high LR, i decreased the LR dynamically every X steps (epochs) depending if it is necessary (when accuracy is not improved).\n# With the ReduceLROnPlateau function from Keras.callbacks, i choose to reduce the LR by half if the accuracy is not improved after 3 epochs.\n\n# Set a learning rate annealer\nlearning_rate_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                            patience=3, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.000001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"44ec53cc3c3561d40d63ae959c3969af088f35c6"},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n# I did not apply a vertical_flip nor horizontal_flip since it could have lead to misclassify symetrical numbers such as 6 and 9.\n\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"309e493e64c8f027807f0b5cd6a39f014d9eff5e"},"cell_type":"code","source":"history = model.fit_generator(datagen.flow(X_train, Y_train, batch_size=512),\n                              epochs = 30, validation_data=(X_val, Y_val),\n                              verbose = 1, steps_per_epoch=X_train.shape[0]/512, \n                              callbacks=[learning_rate_reduction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7bbce742fbff5e226613bc817428cd0c0fedf5b9"},"cell_type":"code","source":"# predict results\nresults = model.predict(X_test)\n\n# select the indix with the maximum probability\nresults = np.argmax(results,axis = 1)\n\nresults = pd.Series(results,name=\"Label\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"7f6e0a5addcb8232985b6f12d3244fd6ba337bc7"},"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"b191c1cf1483ce4b2458a67c6f2d1ec04fa45497"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}