{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Convolutional Neural Networks (CNN)","metadata":{"_uuid":"8c80a83f7fb1ab87aabbdad97d11627fb0380e8f"}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nprint(os.listdir(\"../input\"))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-07T13:47:59.810023Z","iopub.execute_input":"2023-02-07T13:47:59.810369Z","iopub.status.idle":"2023-02-07T13:47:59.817003Z","shell.execute_reply.started":"2023-02-07T13:47:59.810316Z","shell.execute_reply":"2023-02-07T13:47:59.815890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"1\"></a>\n## Loading the Data Set\n* In this part we load and visualize the data.","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"}},{"cell_type":"code","source":"# Read Train data\ntrain = pd.read_csv(\"../input/train.csv\")\nprint(train.shape)\ntrain.head()","metadata":{"_uuid":"6884db4361a209f639cd6f3bf15231d271ecf563","execution":{"iopub.status.busy":"2023-02-07T13:48:01.680406Z","iopub.execute_input":"2023-02-07T13:48:01.680715Z","iopub.status.idle":"2023-02-07T13:48:06.622569Z","shell.execute_reply.started":"2023-02-07T13:48:01.680676Z","shell.execute_reply":"2023-02-07T13:48:06.620868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read test \ntest= pd.read_csv(\"../input/test.csv\")\nprint(test.shape)\ntest.head()","metadata":{"_uuid":"d7ab3a0db0c87aef2bb65fae5f564ff4b1260a96","execution":{"iopub.status.busy":"2023-02-07T13:48:06.624800Z","iopub.execute_input":"2023-02-07T13:48:06.625290Z","iopub.status.idle":"2023-02-07T13:48:10.119188Z","shell.execute_reply.started":"2023-02-07T13:48:06.625216Z","shell.execute_reply":"2023-02-07T13:48:10.117951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# put labels into y_train variable\nY_train = train[\"label\"]\n# Drop 'label' column\nX_train = train.drop(labels = [\"label\"],axis = 1) ","metadata":{"_uuid":"318d94f6e935a6d8210d0ac03707dfa6e3946475","execution":{"iopub.status.busy":"2023-02-07T13:48:11.488222Z","iopub.execute_input":"2023-02-07T13:48:11.488546Z","iopub.status.idle":"2023-02-07T13:48:11.798426Z","shell.execute_reply.started":"2023-02-07T13:48:11.488497Z","shell.execute_reply":"2023-02-07T13:48:11.797231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# visualize number of digits classes\nplt.figure(figsize=(15,7))\ndigits = sns.countplot(Y_train, palette=\"icefire\")\nplt.title(\"Number of digit classes\")\nY_train.value_counts()","metadata":{"_uuid":"8902b0312e6c047596cf27ebba554a68b82604b2","execution":{"iopub.status.busy":"2023-02-07T13:48:12.655428Z","iopub.execute_input":"2023-02-07T13:48:12.655775Z","iopub.status.idle":"2023-02-07T13:48:13.028851Z","shell.execute_reply.started":"2023-02-07T13:48:12.655714Z","shell.execute_reply":"2023-02-07T13:48:13.027503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the data\nX_train.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T13:48:13.889936Z","iopub.execute_input":"2023-02-07T13:48:13.890627Z","iopub.status.idle":"2023-02-07T13:48:14.450343Z","shell.execute_reply.started":"2023-02-07T13:48:13.890571Z","shell.execute_reply":"2023-02-07T13:48:14.449554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2023-02-07T13:48:16.302258Z","iopub.execute_input":"2023-02-07T13:48:16.302660Z","iopub.status.idle":"2023-02-07T13:48:16.673070Z","shell.execute_reply.started":"2023-02-07T13:48:16.302589Z","shell.execute_reply":"2023-02-07T13:48:16.672034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is no missing values in the train and test dataset. So we can safely go ahead.","metadata":{}},{"cell_type":"code","source":"# plot some samples\nimg = X_train.iloc[0].as_matrix()\nimg = img.reshape((28,28))\nplt.imshow(img,cmap='gray')\nplt.title(train.iloc[0,0])\nplt.axis(\"off\")\nplt.show()","metadata":{"_uuid":"4c9d3a1467cf82a3dff04967a9846906f9758ed4","execution":{"iopub.status.busy":"2023-02-07T13:48:19.683355Z","iopub.execute_input":"2023-02-07T13:48:19.683704Z","iopub.status.idle":"2023-02-07T13:48:19.810992Z","shell.execute_reply.started":"2023-02-07T13:48:19.683648Z","shell.execute_reply":"2023-02-07T13:48:19.809508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot some samples\nimg = X_train.iloc[3].as_matrix()\nimg = img.reshape((28,28))\nplt.imshow(img,cmap='gray')\nplt.title(train.iloc[3,0])\nplt.axis(\"off\")\nplt.show()","metadata":{"_uuid":"d6b119ecbe774a45656d1d157f9b33f38adf8e96","execution":{"iopub.status.busy":"2023-02-07T13:48:20.669362Z","iopub.execute_input":"2023-02-07T13:48:20.669670Z","iopub.status.idle":"2023-02-07T13:48:20.780019Z","shell.execute_reply.started":"2023-02-07T13:48:20.669629Z","shell.execute_reply":"2023-02-07T13:48:20.778853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"2\"></a>\n## Normalization, Reshape and Label Encoding ","metadata":{"_uuid":"74e341f4845b42101182eda3c990e3dc4dc64dff"}},{"cell_type":"code","source":"# Normalize the data\nX_train = X_train / 255.0\ntest = test / 255.0\nprint(\"x_train shape: \",X_train.shape)\nprint(\"test shape: \",test.shape)","metadata":{"_uuid":"f39e537f15757b5da3363c138a33e50bd78bbf49","execution":{"iopub.status.busy":"2023-02-07T13:48:25.442994Z","iopub.execute_input":"2023-02-07T13:48:25.443362Z","iopub.status.idle":"2023-02-07T13:48:26.170335Z","shell.execute_reply.started":"2023-02-07T13:48:25.443306Z","shell.execute_reply":"2023-02-07T13:48:26.169569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshape\nX_train = X_train.values.reshape(-1,28,28,1)\ntest = test.values.reshape(-1,28,28,1)\nprint(\"x_train shape: \",X_train.shape)\nprint(\"test shape: \",test.shape)","metadata":{"_uuid":"d524f6de086928158a76c54d1685a92c1802e230","execution":{"iopub.status.busy":"2023-02-07T13:48:27.907069Z","iopub.execute_input":"2023-02-07T13:48:27.908307Z","iopub.status.idle":"2023-02-07T13:48:27.915197Z","shell.execute_reply.started":"2023-02-07T13:48:27.908198Z","shell.execute_reply":"2023-02-07T13:48:27.914458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Label Encoding \nfrom keras.utils.np_utils import to_categorical # convert to one-hot-encoding\nY_train = to_categorical(Y_train, num_classes = 10)","metadata":{"_uuid":"d15d35ca439dce194a96f4442c7a1c085ce24d28","execution":{"iopub.status.busy":"2023-02-07T13:48:31.154678Z","iopub.execute_input":"2023-02-07T13:48:31.155291Z","iopub.status.idle":"2023-02-07T13:48:31.160994Z","shell.execute_reply.started":"2023-02-07T13:48:31.155207Z","shell.execute_reply":"2023-02-07T13:48:31.160116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"3\"></a>\n## Train Test Split\n* We split the data into train and test sets.\n* test size is 10%.\n* train size is 90%.","metadata":{"_uuid":"111d4a1e66d823363f890f892bba39d692479663"}},{"cell_type":"code","source":"# Split the train and the validation set for the fitting\nfrom sklearn.model_selection import train_test_split\nX_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train, test_size = 0.1, random_state=2)\nprint(\"x_train shape\",X_train.shape)\nprint(\"x_test shape\",X_val.shape)\nprint(\"y_train shape\",Y_train.shape)\nprint(\"y_test shape\",Y_val.shape)","metadata":{"_uuid":"882d9dfb1bee94b2ec1e83911e3e334994d74f4b","execution":{"iopub.status.busy":"2023-02-07T13:48:33.669400Z","iopub.execute_input":"2023-02-07T13:48:33.669961Z","iopub.status.idle":"2023-02-07T13:48:34.608197Z","shell.execute_reply.started":"2023-02-07T13:48:33.669889Z","shell.execute_reply":"2023-02-07T13:48:34.606943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Some examples\nplt.imshow(X_train[155][:,:,0],cmap='gray')\nplt.show()","metadata":{"_uuid":"1cdc57350e1c251ec583093ebba18c51a537f55f","execution":{"iopub.status.busy":"2023-02-07T13:48:51.463181Z","iopub.execute_input":"2023-02-07T13:48:51.463876Z","iopub.status.idle":"2023-02-07T13:48:51.653944Z","shell.execute_reply.started":"2023-02-07T13:48:51.463790Z","shell.execute_reply":"2023-02-07T13:48:51.652681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"4\"></a>\n## Convolutional Neural Network ","metadata":{"_uuid":"caf4b9f7fcae153ac1d4c0246b1defc9d9d30776"}},{"cell_type":"markdown","source":"<a id=\"10\"></a>\n## Implementing with Keras","metadata":{"_uuid":"7b290eb3f3b111d6841f8e4093277be1bde05078","trusted":true}},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport itertools\n\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D\nfrom keras.optimizers import RMSprop,Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau","metadata":{"execution":{"iopub.status.busy":"2023-02-07T13:49:31.762550Z","iopub.execute_input":"2023-02-07T13:49:31.763242Z","iopub.status.idle":"2023-02-07T13:49:31.769367Z","shell.execute_reply.started":"2023-02-07T13:49:31.763186Z","shell.execute_reply":"2023-02-07T13:49:31.768244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(filters = 32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape = (28,28,1)))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Conv2D(filters = 64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))","metadata":{"_uuid":"c441d7b3852cee5d3636272d4da2f96b169f81ac","execution":{"iopub.status.busy":"2023-02-07T13:51:13.829230Z","iopub.execute_input":"2023-02-07T13:51:13.829632Z","iopub.status.idle":"2023-02-07T13:51:13.978852Z","shell.execute_reply.started":"2023-02-07T13:51:13.829575Z","shell.execute_reply":"2023-02-07T13:51:13.977350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"12\"></a>\n### Define Optimizer   \n* Adam optimizer: Change the learning rate\n","metadata":{"_uuid":"9a432df491777ca0019db6f6b972581f2c5bd9f9"}},{"cell_type":"code","source":"# Define the optimizer\noptimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999)","metadata":{"_uuid":"607a02b42636e3115a1ac7a8edcadf61cf5ea1b0","execution":{"iopub.status.busy":"2023-02-07T13:51:29.166082Z","iopub.execute_input":"2023-02-07T13:51:29.166423Z","iopub.status.idle":"2023-02-07T13:51:29.191670Z","shell.execute_reply.started":"2023-02-07T13:51:29.166362Z","shell.execute_reply":"2023-02-07T13:51:29.190426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"13\"></a>\n### Compile Model","metadata":{"_uuid":"91bf542a979c5b191c534876186bf31e70ec7f06"}},{"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"_uuid":"0d1eefc68470b4cdcec04c2570651da3d97676d0","execution":{"iopub.status.busy":"2023-02-07T13:51:33.892430Z","iopub.execute_input":"2023-02-07T13:51:33.892875Z","iopub.status.idle":"2023-02-07T13:51:33.937682Z","shell.execute_reply.started":"2023-02-07T13:51:33.892789Z","shell.execute_reply":"2023-02-07T13:51:33.936748Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"14\"></a>\n### Epochs and Batch Size","metadata":{"_uuid":"d5d8fea4cbb9b53cf1bb5089357a33dc179ee981"}},{"cell_type":"code","source":"epochs = 10  # for better result increase the epochs\nbatch_size = 250","metadata":{"_uuid":"a237feb5e53ecbc8799101cb6e699877faafde77","execution":{"iopub.status.busy":"2023-02-07T13:51:38.955750Z","iopub.execute_input":"2023-02-07T13:51:38.956196Z","iopub.status.idle":"2023-02-07T13:51:38.960622Z","shell.execute_reply.started":"2023-02-07T13:51:38.956121Z","shell.execute_reply":"2023-02-07T13:51:38.959574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"15\"></a>\n### Data Augmentation\n* To avoid overfitting problem, we need to expand artificially our handwritten digit dataset\n* Alter the training data with small transformations to reproduce the variations of digit.\n* For example, the number is not centered The scale is not the same (some who write with big/small numbers) The image is rotated.","metadata":{"_uuid":"e87c60e45759e0c52c2b72ee562fdb964c8008d5"}},{"cell_type":"code","source":"# data augmentation\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # dimesion reduction\n        rotation_range=5,  # randomly rotate images in the range 5 degrees\n        zoom_range = 0.1, # Randomly zoom image 10%\n        width_shift_range=0.1,  # randomly shift images horizontally 10%\n        height_shift_range=0.1,  # randomly shift images vertically 10%\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\ndatagen.fit(X_train)","metadata":{"_uuid":"c339fa7d06e9b73a519ca661bed19de482707d1f","execution":{"iopub.status.busy":"2023-02-07T13:51:45.938049Z","iopub.execute_input":"2023-02-07T13:51:45.938443Z","iopub.status.idle":"2023-02-07T13:51:46.186952Z","shell.execute_reply.started":"2023-02-07T13:51:45.938374Z","shell.execute_reply":"2023-02-07T13:51:46.185782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"16\"></a>\n### Fit the model","metadata":{"_uuid":"e42e43f1b3b9e6da0d99dafff8f4a4b514a07f4c"}},{"cell_type":"code","source":"# Fit the model\nhistory = model.fit_generator(datagen.flow(X_train,Y_train, batch_size=batch_size),\n                              epochs = epochs, validation_data = (X_val,Y_val), steps_per_epoch=X_train.shape[0] // batch_size)","metadata":{"_uuid":"4b2957bb8976a25cdbbbdc3110d68c5035a9773c","execution":{"iopub.status.busy":"2023-02-07T13:51:53.833709Z","iopub.execute_input":"2023-02-07T13:51:53.834349Z","iopub.status.idle":"2023-02-07T14:02:15.898746Z","shell.execute_reply.started":"2023-02-07T13:51:53.834297Z","shell.execute_reply":"2023-02-07T14:02:15.897430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Accuracy of the model is - \" , model.evaluate(X_val,Y_val)[1]*100 , \"%\")","metadata":{"execution":{"iopub.status.busy":"2023-02-07T14:04:52.762289Z","iopub.execute_input":"2023-02-07T14:04:52.762634Z","iopub.status.idle":"2023-02-07T14:04:55.662088Z","shell.execute_reply.started":"2023-02-07T14:04:52.762581Z","shell.execute_reply":"2023-02-07T14:04:55.660550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"17\"></a>\n### Evaluate the model\n* Test Loss visualization\n* Confusion matrix\n","metadata":{"_uuid":"debd5424728e11aa30a8513ac7b4f7377193a2da"}},{"cell_type":"code","source":"# Plot the loss and accuracy curves for training and validation \nplt.plot(history.history['val_loss'], color='b', label=\"validation loss\")\nplt.title(\"Test Loss\")\nplt.xlabel(\"Number of Epochs\")\nplt.ylabel(\"Loss\")\nplt.legend()\nplt.show()","metadata":{"_uuid":"180a06f7ae01e69117c6c8258411cfe1b9b7b991","execution":{"iopub.status.busy":"2023-02-07T14:05:14.951258Z","iopub.execute_input":"2023-02-07T14:05:14.952050Z","iopub.status.idle":"2023-02-07T14:05:15.245318Z","shell.execute_reply.started":"2023-02-07T14:05:14.951948Z","shell.execute_reply":"2023-02-07T14:05:15.243682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# confusion matrix\nimport seaborn as sns\n# Predict the values from the validation dataset\nY_pred = model.predict(X_val)\n# Convert predictions classes to one hot vectors \nY_pred_classes = np.argmax(Y_pred,axis = 1) \n# Convert validation observations to one hot vectors\nY_true = np.argmax(Y_val,axis = 1) \n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(Y_true, Y_pred_classes) \n# plot the confusion matrix\nf,ax = plt.subplots(figsize=(8, 8))\nsns.heatmap(confusion_mtx, annot=True, linewidths=0.01,cmap=\"Reds\",linecolor=\"gray\", fmt= '.1f',ax=ax)\nplt.xlabel(\"Predicted Label\")\nplt.ylabel(\"True Label\")\nplt.title(\"Confusion Matrix\")\nplt.show()","metadata":{"_uuid":"6586e37bd470db822086e191a90388e7175d504f","execution":{"iopub.status.busy":"2023-02-07T14:05:20.659983Z","iopub.execute_input":"2023-02-07T14:05:20.660710Z","iopub.status.idle":"2023-02-07T14:05:24.768911Z","shell.execute_reply.started":"2023-02-07T14:05:20.660650Z","shell.execute_reply":"2023-02-07T14:05:24.767021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<a id=\"18\"></a>\n## Conclusion\nI achieved 99.11% of accuracy with this CNN","metadata":{"_uuid":"9409bc6a7e905d5887c8eec8d056c9f015bf4878"}}]}