{"cells":[{"metadata":{},"cell_type":"markdown","source":"<img src=\"https://www.katacoda.com/basiafusinska/courses/deep-learning-with-tensorflow/mnist-dataset/assets/MNIST.png\" />"},{"metadata":{},"cell_type":"markdown","source":"# MNIST Dataset"},{"metadata":{},"cell_type":"markdown","source":"### Importing data and libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.chdir('/kaggle/input')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindatafile = pd.read_csv('train.csv')\ntestdatafile = pd.read_csv('test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindatafile.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = traindatafile.pop('label')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import confusion_matrix","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Wrangling"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_val, y_train, y_val = train_test_split(traindatafile, y, test_size=0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"traindatafile.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\nx_train = x_train.values.reshape(-1,28,28,1)\nx_val = x_val.values.reshape(-1,28,28,1)\nx_test= testdatafile.values.reshape(-1,28,28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(x_train.shape)\nprint(x_val.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = x_train.astype(\"float32\")/255.\nx_val = x_val.astype(\"float32\")/255.\nx_test = x_test.astype(\"float32\")/255.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(x_train.shape)\nx_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Importing Keras libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler\nfrom keras.callbacks import TensorBoard","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"y_train = to_categorical(y_train) \ny_val = to_categorical(y_val) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_train.shape)\ny_val.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train and test images (28px x 28px) has been stock into pandas.Dataframe as 1D vectors of 784 values. We reshape all data to 28x28x1 3D matrices.\n\nKeras requires an extra dimension in the end which correspond to channels. MNIST images are gray scaled so it use only one channel. For RGB images, there is 3 channels, we would have reshaped 784px vectors to 28x28x3 3D matrices."},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline \n\ng = plt.imshow(x_train[0][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model CNN "},{"metadata":{},"cell_type":"markdown","source":"<img src= \"https://iq.opengenus.org/content/images/2018/11/cnn.png\" />"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set the CNN model \n# my CNN architechture is In -> [[Conv2D->relu]*2 -> MaxPool2D -> Dropout]*2 -> Flatten -> Dense -> Dropout -> Out\n\nmodel = Sequential()\n\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape = (28,28,1)))\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"this is to for changing images properties "},{"metadata":{"trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(featurewise_center=False,  # set input mean to 0 over the dataset\n                    samplewise_center=False,  # set each sample mean to 0\n                                 featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n                               zca_whitening=False,  # apply ZCA whitening\n        rotation_range=15, # randomly rotate images in the range (degrees, 0 to 180)\n                              zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n                               height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n                               vertical_flip=False)  # randomly flip images)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<img src=\"https://i.ytimg.com/vi/tRsSi_sqXjI/maxresdefault.jpg\" />"},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss='categorical_crossentropy', optimizer = Adam(lr=1e-4), metrics=[\"accuracy\"]) #1e-4, means the 1 is four digits the other way, so 1e-4 = 0.0001.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"learning_rate_min = LearningRateScheduler(lambda x: 1e-3 * 0.9 ** x)\n#tensor= TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None, update_freq=1000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Fitting the data in model"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"hist = model.fit_generator(datagen.flow(x_train, y_train, batch_size=16),\n                           steps_per_epoch=500,\n                           epochs=20, #Increase this when not on Kaggle kernel\n                           verbose=2,  #1 for ETA, 0 for silent\n                           validation_data=(x_val[:400,:], y_val[:400,:]), #For speed\n                           callbacks=[learning_rate_min]) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_loss, final_acc = model.evaluate(x_val, y_val, verbose=0)\nprint(\"Final loss: {0:.4f}, final accuracy: {1:.4f}\".format(final_loss, final_acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(hist.history['loss'], color='r',label='Training loss')\nplt.plot(hist.history['acc'], color='b',label='Training accuracy')\nplt.title('Training loss and Training accuracy')\nplt.show()\n\nplt.plot(hist.history['val_loss'], color='r', label='Validation loss')\nplt.plot(hist.history['val_acc'], color='b', label='Validation accuracy')\nplt.title('validation loss and validation accuracy')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Confusion matrix example"},{"metadata":{},"cell_type":"markdown","source":"<img src=\"https://s3-ap-south-1.amazonaws.com/av-blog-media/wp-content/uploads/2018/03/confusion-matrix-mercari.png\" />"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_predicted = model.predict(x_val)\ny_predictedint = np.argmax(y_predicted, axis=1)\ny_true = np.argmax(y_val, axis=1)\ncm = confusion_matrix(y_true, y_predictedint)\nprint(cm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_hat = model.predict(x_test ,batch_size=64)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_hat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = np.argmax(y_hat,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#####  another way to submit \nwith open(\"submission.csv\", 'w') as f :\n    f.write('ImageId,Label\\n')\n    for i in range(len(y_pred)) :\n        f.write(\"\".join([str(i+1),',',str(y_pred[i]),'\\n']))"},{"metadata":{"trusted":false},"cell_type":"markdown","source":"submissions=pd.DataFrame({\"ImageId\": list(range(1,len(y_pred)+1)),\n                         \"Label\": y_pred})\nsubmissions.to_csv(\"submission.csv\", index=False, header=True)\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}