{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=pd.read_csv('../input/train.csv')\ntest_data=pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head(1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\nimport seaborn as sns\n%matplotlib inline\nimport itertools\n\nnp.random.seed(2)\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Dropout,Conv2D,MaxPooling2D,Flatten\nfrom keras.optimizers import Adam,RMSprop\nfrom keras.utils import to_categorical\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau\n\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix\n\nsns.set(style='white', context='notebook', palette='deep')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"DATA PREPERATION"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train= train_data.drop(labels=['label'],axis=1)\ny_train=train_data['label'] \ndel train_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Build frequency chart of classes to look for class imbalance\n\ng=sns.countplot(y_train)\ny_train.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"raw","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check for missing and null values\nx_train.isnull().any().describe()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.isnull().any().describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split data into Train and Validation\n\nx_train,x_val,y_train,y_val=train_test_split(x_train,y_train,test_size=.2,random_state=2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#One hot encode the labels using to_categorical\ny_train=to_categorical(y_train)\ny_val=to_categorical(y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print((x_train.shape,y_train.shape))\nprint((x_val.shape,y_val.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Reshape the data\nx_train=x_train.values.reshape(-1,28,28,1)\nx_val=x_val.values.reshape(-1,28,28,1)\ntest_data=test_data.values.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create Augmentation Function using ImageDataGenerator\n\ntrain_datagen= ImageDataGenerator(\n                                rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n                                zoom_range = 0.1, # Randomly zoom image \n                                width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n                                height_shift_range=0.1)# randomly shift images vertically (fraction of total height)\n\nval_datagen=ImageDataGenerator(\n                                rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n                                zoom_range = 0.1, # Randomly zoom image \n                                width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n                                height_shift_range=0.1)# randomly shift images vertically (fraction of total height)\n\ntest_datagen=ImageDataGenerator(\n                                rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n                                zoom_range = 0.1, # Randomly zoom image \n                                width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n                                height_shift_range=0.1)# randomly shift images vertically (fraction of total height)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_datagen.fit(x_train)\nval_datagen.fit(x_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Set Learning rate annealer\n\nlrr=ReduceLROnPlateau(\n                     monitor='val_acc',\n                     patience=3,\n                     verbose=1,\n                     factor=0.5,\n                     min_lr=.00001)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set Optimizer\noptimizer= Adam(lr=.001,beta_1=0.9, beta_2=0.999,epsilon=1e-6)\noptimizer_RMS = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define the Model- 2 Convolutional layers with kernel size (32,5,5),Maxpool with kernel(2,2), Dropout=.25\n#                  2 Convolutional layers with kernel size (64,3,3),Maxpool with kernel(2,2), Dropout=.25   \n#                  1 Flatten layer, 1 Dense layer(256) ,dropout =.5 and classification layer(10)\n\nmodel=Sequential()\n\nmodel.add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu',input_shape=(28,28,1)))\nmodel.add(Conv2D(filters=32,kernel_size=(5,5),padding='Same',activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(.25))\n\nmodel.add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu'))\nmodel.add(Conv2D(filters=64,kernel_size=(3,3),padding='Same',activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2,2)))\nmodel.add(Dropout(.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256,activation='relu'))\nmodel.add(Dropout(.5))\nmodel.add(Dense(10,activation='softmax'))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Compile the model\nmodel.compile(optimizer=optimizer_RMS,loss='categorical_crossentropy',metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define batch_size,epochs\nbatch_size=86\nepochs=50","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Fit the model\n\nmodel.fit_generator(train_datagen.flow(x_train,y_train,batch_size=batch_size),epochs=epochs,\n                   validation_data=(x_val,y_val),\n                   steps_per_epoch=x_train.shape[0]//batch_size,callbacks=[lrr],verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#EVALUATE THE MODEL","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plot training loss and validation loss\nfig,ax= plt.subplots(2,1)\nax[0].plot(model.history.history['loss'],color='b',label='Training Loss')\nax[0].plot(model.history.history['val_loss'],color='r',label='Validation_Loss')\n\n#Plot Training and validation accuracy\nax[1].plot(model.history.history['acc'],color='b')\nax[1].plot(model.history.history['val_acc'],color='r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Confusion Matrix\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Predict values from validation dataset\ny_pred=model.predict(x_val)\n#Convert prediction classes to 1 hot vectors\ny_pred_classes=np.argmax(y_pred,axis=1)\n#Convert Validation classes to 1 hot vectors\ny_true=np.argmax(y_val,axis=1)\n\n#Compute the confusion matrix\nconfusion_mtx=confusion_matrix(y_true,y_pred_classes)\n\nplot_confusion_matrix(confusion_mtx, classes = range(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Display Top 6 Error Results\n\nerrors=(y_pred_classes-y_true !=0)\ny_pred_class_errors= y_pred_classes[errors]\ny_true_errors=y_true[errors] #Gives \ny_pred_errors=y_pred[errors]\nx_val_errors = x_val[errors]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_true_errors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred_class_errors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_errors(errors_index,img_errors,pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows,ncols,sharex=True,sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Probabilities of the wrong predicted numbers\ny_pred_errors_prob = np.max(y_pred_errors,axis = 1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_errors = np.diagonal(np.take(y_pred_errors, y_true_errors, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_errors = y_pred_errors_prob - true_prob_errors\n\n# Sorted list of the delta prob errors\nsorted_dela_errors = np.argsort(delta_pred_true_errors)\n\n# Top 6 errors \nmost_important_errors = sorted_dela_errors[-6:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_errors, x_val_errors, y_pred_class_errors, y_true_errors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predict Results\n\nresults=model.predict(test_data)\n# select the index with the maximum probability\nresults=np.argmin(results,axis=1)\nresults = pd.Series(results,name=\"Label\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission=pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"cnn_mnist_datagen.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}