{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport random\nrandom.seed(10)\nimport warnings\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.utils.np_utils import to_categorical\nfrom keras.applications.vgg16 import VGG16\nfrom keras.layers import Conv2D, MaxPool2D, Dense, Flatten, Dropout, Activation,BatchNormalization\nfrom keras.models import Sequential\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array, array_to_img\nwarnings.filterwarnings('ignore')\n%matplotlib inline\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-25T02:51:35.015722Z","iopub.execute_input":"2021-06-25T02:51:35.016142Z","iopub.status.idle":"2021-06-25T02:51:35.053847Z","shell.execute_reply.started":"2021-06-25T02:51:35.016106Z","shell.execute_reply":"2021-06-25T02:51:35.052648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import cv2","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:38.279783Z","iopub.execute_input":"2021-06-24T16:03:38.280043Z","iopub.status.idle":"2021-06-24T16:03:38.430543Z","shell.execute_reply.started":"2021-06-24T16:03:38.280017Z","shell.execute_reply":"2021-06-24T16:03:38.429774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:38.433511Z","iopub.execute_input":"2021-06-24T16:03:38.433792Z","iopub.status.idle":"2021-06-24T16:03:43.079004Z","shell.execute_reply.started":"2021-06-24T16:03:38.433767Z","shell.execute_reply":"2021-06-24T16:03:43.078071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:43.080502Z","iopub.execute_input":"2021-06-24T16:03:43.080859Z","iopub.status.idle":"2021-06-24T16:03:43.108757Z","shell.execute_reply.started":"2021-06-24T16:03:43.080821Z","shell.execute_reply":"2021-06-24T16:03:43.107869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(test.shape)\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:43.110095Z","iopub.execute_input":"2021-06-24T16:03:43.110424Z","iopub.status.idle":"2021-06-24T16:03:43.116094Z","shell.execute_reply.started":"2021-06-24T16:03:43.11039Z","shell.execute_reply":"2021-06-24T16:03:43.115069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train = train['label'] # y are the labels, x are the data\ny_train = tf.keras.utils.to_categorical(y_train)\nx_train = train.drop(['label'], axis=1) #deafult is axis=0","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:43.117485Z","iopub.execute_input":"2021-06-24T16:03:43.118262Z","iopub.status.idle":"2021-06-24T16:03:43.203166Z","shell.execute_reply.started":"2021-06-24T16:03:43.11817Z","shell.execute_reply":"2021-06-24T16:03:43.202286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# reshaping before feeding into the network\nx_train = x_train.values.reshape(x_train.shape[0],28,28,1)\ntest = test.values.reshape(test.shape[0],28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:43.204665Z","iopub.execute_input":"2021-06-24T16:03:43.205021Z","iopub.status.idle":"2021-06-24T16:03:43.212199Z","shell.execute_reply.started":"2021-06-24T16:03:43.204984Z","shell.execute_reply":"2021-06-24T16:03:43.211402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('The shape of y_train is: ', y_train.shape)\nprint('The shape of x_train is: ', x_train.shape)\nprint('The shape of x_test is: ', test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:43.216016Z","iopub.execute_input":"2021-06-24T16:03:43.216281Z","iopub.status.idle":"2021-06-24T16:03:43.222106Z","shell.execute_reply.started":"2021-06-24T16:03:43.216234Z","shell.execute_reply":"2021-06-24T16:03:43.221099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# A Glimpse into the data","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor i in range(225):\n    plt.subplot(15,15,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(x_train[i])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:43.224456Z","iopub.execute_input":"2021-06-24T16:03:43.225189Z","iopub.status.idle":"2021-06-24T16:03:52.703598Z","shell.execute_reply.started":"2021-06-24T16:03:43.225151Z","shell.execute_reply":"2021-06-24T16:03:52.702782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalization","metadata":{}},{"cell_type":"code","source":"x_train = x_train.astype('float64')/255.0\ny_train = y_train.astype('float64')\ntest = test.astype('float64')/255.0\n\nx_train = np.asarray(x_train)\ny_train = np.asarray(y_train)\ntest = np.asarray(test)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:52.704627Z","iopub.execute_input":"2021-06-24T16:03:52.70492Z","iopub.status.idle":"2021-06-24T16:03:52.896122Z","shell.execute_reply.started":"2021-06-24T16:03:52.70489Z","shell.execute_reply":"2021-06-24T16:03:52.895125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Function to change size of the image to match the pretrained vgg model","metadata":{}},{"cell_type":"code","source":"def change_size(image):\n    img = array_to_img(image, scale=False) #returns PIL Image\n    img = img.resize((48, 48)) #resize image\n    img = img.convert(mode='RGB') #makes 3 channels\n    arr = img_to_array(img) #convert back to array\n    return arr.astype(np.float64)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:52.897697Z","iopub.execute_input":"2021-06-24T16:03:52.898072Z","iopub.status.idle":"2021-06-24T16:03:52.903062Z","shell.execute_reply.started":"2021-06-24T16:03:52.898033Z","shell.execute_reply":"2021-06-24T16:03:52.902254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_data = [change_size(i) for i in x_train]\ndel x_train\nx_train_data = np.asarray(x_train_data)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:03:52.904318Z","iopub.execute_input":"2021-06-24T16:03:52.904854Z","iopub.status.idle":"2021-06-24T16:04:01.351784Z","shell.execute_reply.started":"2021-06-24T16:03:52.904819Z","shell.execute_reply":"2021-06-24T16:04:01.350928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Partial Transfer Learning on VGG-16 model freezing the first layer","metadata":{}},{"cell_type":"code","source":"# Define the parameters for the VGG16 model\nIMG_WIDTH = 48\nIMG_HEIGHT = 48\nIMG_DEPTH = 3\nBATCH_SIZE = 16\n\nmodel_vgg16 = VGG16(weights='imagenet',\n                  include_top=False, \n                  input_shape=(IMG_HEIGHT, IMG_WIDTH, IMG_DEPTH)\n                 )\nprint(model_vgg16.summary())\n\nmodel_vgg16.layers[1].trainable = False","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:04:01.353071Z","iopub.execute_input":"2021-06-24T16:04:01.353586Z","iopub.status.idle":"2021-06-24T16:04:03.944016Z","shell.execute_reply.started":"2021-06-24T16:04:01.353533Z","shell.execute_reply":"2021-06-24T16:04:03.94316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the trainable status of the individual layers\nfor layer in model_vgg16.layers:\n    print(layer, layer.trainable)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:04:03.946108Z","iopub.execute_input":"2021-06-24T16:04:03.94648Z","iopub.status.idle":"2021-06-24T16:04:03.95305Z","shell.execute_reply.started":"2021-06-24T16:04:03.94643Z","shell.execute_reply":"2021-06-24T16:04:03.952335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model= Sequential()\n\nmodel.add(model_vgg16)\n\nmodel.add(Flatten())\nmodel.add(Dense(1024, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation='softmax'))\n\nmodel.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:04:03.95407Z","iopub.execute_input":"2021-06-24T16:04:03.954391Z","iopub.status.idle":"2021-06-24T16:04:04.046971Z","shell.execute_reply.started":"2021-06-24T16:04:03.954357Z","shell.execute_reply":"2021-06-24T16:04:04.046219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size=128\nepochs=21\n\nmodel.fit(x_train_data, y_train,\n          batch_size=128,\n          epochs=12,\n          verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-24T16:04:04.048105Z","iopub.execute_input":"2021-06-24T16:04:04.048419Z","iopub.status.idle":"2021-06-24T16:48:47.193462Z","shell.execute_reply.started":"2021-06-24T16:04:04.048386Z","shell.execute_reply":"2021-06-24T16:48:47.192402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### As you can see, the accuracy is pathetic, so we train a cnn from scratch with a similar architecture ","metadata":{}},{"cell_type":"markdown","source":"# Training from scratch on VGG 16 model architecture","metadata":{}},{"cell_type":"code","source":"model = Sequential()\nmodel.add(Conv2D(32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu', input_shape=(28,28,1)))\nmodel.add(Conv2D(32, kernel_size = (5,5),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2)))\nmodel.add(Dropout(0.5*0.5))\n\nmodel.add(Conv2D(64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(Conv2D(64, kernel_size = (3,3),padding = 'Same', \n                 activation ='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.5*0.5))\n\nmodel.add(Flatten())\nmodel.add(Dense(256, activation = \"relu\"))\nmodel.add(Dropout(0.5))\nmodel.add(Dense(10, activation = \"softmax\"))\n\nmodel.compile(optimizer=keras.optimizers.Adam(lr=0.001), loss='categorical_crossentropy', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:34:51.553044Z","iopub.execute_input":"2021-06-25T02:34:51.553476Z","iopub.status.idle":"2021-06-25T02:34:54.288422Z","shell.execute_reply.started":"2021-06-25T02:34:51.553443Z","shell.execute_reply":"2021-06-25T02:34:54.287252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:34:59.543555Z","iopub.execute_input":"2021-06-25T02:34:59.54398Z","iopub.status.idle":"2021-06-25T02:34:59.559599Z","shell.execute_reply.started":"2021-06-25T02:34:59.543934Z","shell.execute_reply":"2021-06-25T02:34:59.558195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ny_train = train['label'] # y are the labels, x are the data\ny_train = tf.keras.utils.to_categorical(y_train)\nx_train = train.drop(['label'], axis=1) #deafult is axis=0\n# reshaping before feeding into the network\nx_train = x_train.values.reshape(x_train.shape[0],28,28,1)\ntest = test.values.reshape(test.shape[0],28,28,1)\nhistory = model.fit(x_train, y_train, epochs=15)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:51:39.212701Z","iopub.execute_input":"2021-06-25T02:51:39.213154Z","iopub.status.idle":"2021-06-25T02:55:19.041229Z","shell.execute_reply.started":"2021-06-25T02:51:39.21306Z","shell.execute_reply":"2021-06-25T02:55:19.040161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### This model performs way better and achieves a much better accuracy","metadata":{}},{"cell_type":"code","source":"df = pd.DataFrame(history.history)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:57:03.044553Z","iopub.execute_input":"2021-06-25T02:57:03.044926Z","iopub.status.idle":"2021-06-25T02:57:03.051327Z","shell.execute_reply.started":"2021-06-25T02:57:03.044895Z","shell.execute_reply":"2021-06-25T02:57:03.04991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"accuracy\"].plot()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:57:06.680736Z","iopub.execute_input":"2021-06-25T02:57:06.681131Z","iopub.status.idle":"2021-06-25T02:57:06.874908Z","shell.execute_reply.started":"2021-06-25T02:57:06.681094Z","shell.execute_reply":"2021-06-25T02:57:06.873878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"loss\"].plot()","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:57:31.359273Z","iopub.execute_input":"2021-06-25T02:57:31.359666Z","iopub.status.idle":"2021-06-25T02:57:31.562436Z","shell.execute_reply.started":"2021-06-25T02:57:31.359634Z","shell.execute_reply":"2021-06-25T02:57:31.561347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction=np.argmax(model.predict(test),axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:57:36.173676Z","iopub.execute_input":"2021-06-25T02:57:36.174043Z","iopub.status.idle":"2021-06-25T02:57:42.084081Z","shell.execute_reply.started":"2021-06-25T02:57:36.174013Z","shell.execute_reply":"2021-06-25T02:57:42.082771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:57:45.134069Z","iopub.execute_input":"2021-06-25T02:57:45.134504Z","iopub.status.idle":"2021-06-25T02:57:45.145924Z","shell.execute_reply.started":"2021-06-25T02:57:45.134471Z","shell.execute_reply":"2021-06-25T02:57:45.144665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.Series(prediction, name=\"Label\")\nmylist=[]\n[mylist.append(i) for i in range(1,28001)]\nImageID = pd.Series(mylist, name=\"ImageID\").astype(\"int32\")\nsubmission = pd.concat([ImageID,results],axis = 1)\nsubmission.to_csv(\"submission_2.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:58:08.721057Z","iopub.execute_input":"2021-06-25T02:58:08.721515Z","iopub.status.idle":"2021-06-25T02:58:08.799478Z","shell.execute_reply.started":"2021-06-25T02:58:08.721474Z","shell.execute_reply":"2021-06-25T02:58:08.79814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"execution":{"iopub.status.busy":"2021-06-25T02:38:20.060866Z","iopub.execute_input":"2021-06-25T02:38:20.061289Z","iopub.status.idle":"2021-06-25T02:38:20.904712Z","shell.execute_reply.started":"2021-06-25T02:38:20.061249Z","shell.execute_reply":"2021-06-25T02:38:20.903429Z"},"trusted":true},"execution_count":null,"outputs":[]}]}