{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-08-26T13:08:21.064423Z","iopub.execute_input":"2021-08-26T13:08:21.06482Z","iopub.status.idle":"2021-08-26T13:08:21.078104Z","shell.execute_reply.started":"2021-08-26T13:08:21.064773Z","shell.execute_reply":"2021-08-26T13:08:21.076587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_name = \"/kaggle/input/digit-recognizer\"\ntrain_set = pd.read_csv(os.path.join(dir_name,\"train.csv\"))\ntest_set = pd.read_csv(os.path.join(dir_name,\"test.csv\"))\nsample_submission = pd.read_csv(os.path.join(dir_name,\"sample_submission.csv\"))","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:21.081064Z","iopub.execute_input":"2021-08-26T13:08:21.081639Z","iopub.status.idle":"2021-08-26T13:08:26.146861Z","shell.execute_reply.started":"2021-08-26T13:08:21.081559Z","shell.execute_reply":"2021-08-26T13:08:26.14571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set.info()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.150377Z","iopub.execute_input":"2021-08-26T13:08:26.151158Z","iopub.status.idle":"2021-08-26T13:08:26.21318Z","shell.execute_reply.started":"2021-08-26T13:08:26.151108Z","shell.execute_reply":"2021-08-26T13:08:26.212055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.21535Z","iopub.execute_input":"2021-08-26T13:08:26.215812Z","iopub.status.idle":"2021-08-26T13:08:26.241595Z","shell.execute_reply.started":"2021-08-26T13:08:26.215753Z","shell.execute_reply":"2021-08-26T13:08:26.240502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set.columns","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.243369Z","iopub.execute_input":"2021-08-26T13:08:26.243854Z","iopub.status.idle":"2021-08-26T13:08:26.251923Z","shell.execute_reply.started":"2021-08-26T13:08:26.243809Z","shell.execute_reply":"2021-08-26T13:08:26.250411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" %matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.25397Z","iopub.execute_input":"2021-08-26T13:08:26.25451Z","iopub.status.idle":"2021-08-26T13:08:26.26577Z","shell.execute_reply.started":"2021-08-26T13:08:26.254465Z","shell.execute_reply":"2021-08-26T13:08:26.264204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# imports \nfrom scipy.ndimage import rotate\nfrom tqdm import tqdm\nimport numpy as np\nimport cv2 as cv\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras import datasets, layers, models\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.267572Z","iopub.execute_input":"2021-08-26T13:08:26.268146Z","iopub.status.idle":"2021-08-26T13:08:26.277673Z","shell.execute_reply.started":"2021-08-26T13:08:26.268087Z","shell.execute_reply":"2021-08-26T13:08:26.276506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ndef extract_image_from_dataset(dataset, row_index):\n    image_row = dataset.iloc[row_index,1:]\n    image = np.uint8(np.array(image_row))\n    image = np.resize(image,(28,28))\n    return image\n\ndef convert_array_to_image(image_row):\n    image = np.array(image_row)\n    image = np.resize(image,(28,28))\n    return image\n\ndef convert_label_to_probability(number):\n    probs = np.zeros(10)\n    probs[number] = 1\n    return probs","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.282201Z","iopub.execute_input":"2021-08-26T13:08:26.282746Z","iopub.status.idle":"2021-08-26T13:08:26.292989Z","shell.execute_reply.started":"2021-08-26T13:08:26.282716Z","shell.execute_reply":"2021-08-26T13:08:26.291538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_image(index):\n    img = extract_image_from_dataset(train_set, index)\n    print(train_set.iloc[index,0])\n    plt.imshow(img, cmap=plt.cm.binary)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.29648Z","iopub.execute_input":"2021-08-26T13:08:26.297499Z","iopub.status.idle":"2021-08-26T13:08:26.30542Z","shell.execute_reply.started":"2021-08-26T13:08:26.297452Z","shell.execute_reply":"2021-08-26T13:08:26.303953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_image(0)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.307272Z","iopub.execute_input":"2021-08-26T13:08:26.307934Z","iopub.status.idle":"2021-08-26T13:08:26.495068Z","shell.execute_reply.started":"2021-08-26T13:08:26.30789Z","shell.execute_reply":"2021-08-26T13:08:26.493967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load dataset\nnumber_of_rows = train_set.shape[0]\n\n\ntrain_copy = train_set.copy()\ntrain_y = train_copy.pop(\"label\")\n\n\ntrain_X = list(np.array(train_copy))\ntrain_X = np.array(list(map(lambda x:convert_array_to_image(x), train_X)))\n#train_y = np.array(list(map(lambda x:convert_label_to_probability(x), train_y)))\ntrain_y = np.array(train_y)\ntest_X = np.array(list(map(lambda x:convert_array_to_image(x), list(np.array(test_set)))))\nprint(train_X.shape)\nprint(train_y.shape)\nprint(test_X.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:26.49693Z","iopub.execute_input":"2021-08-26T13:08:26.497413Z","iopub.status.idle":"2021-08-26T13:08:29.102124Z","shell.execute_reply.started":"2021-08-26T13:08:26.497369Z","shell.execute_reply":"2021-08-26T13:08:29.100888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# did not improve model\n# rotation based basic data augmentation\ndef augmentation(train_X, train_y):\n    new_data = []\n    new_labels = []\n\n    for index,image in tqdm(enumerate(train_X)):\n        label = train_y[index]\n        tempImage = image\n        for i in range(3):\n            tempImage = rotate(tempImage, angle=90)\n            new_data.append(tempImage)\n            new_labels.append(label) ","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.103977Z","iopub.execute_input":"2021-08-26T13:08:29.104423Z","iopub.status.idle":"2021-08-26T13:08:29.113281Z","shell.execute_reply.started":"2021-08-26T13:08:29.104378Z","shell.execute_reply":"2021-08-26T13:08:29.11184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# new_data,new_labels =  augmentation(train_X, train_y)\n\"\"\"\ntrain_X = np.concatenate((train_X, np.array(new_data)), axis=0)\ntrain_y = np.concatenate((train_y, np.array(new_labels)), axis=0)\nprint(train_X.shape)\nprint(train_y.shape)\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.115337Z","iopub.execute_input":"2021-08-26T13:08:29.11595Z","iopub.status.idle":"2021-08-26T13:08:29.127454Z","shell.execute_reply.started":"2021-08-26T13:08:29.115908Z","shell.execute_reply":"2021-08-26T13:08:29.126058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X, valid_X, train_y, valid_y = train_test_split(train_X, train_y, test_size=0.20, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.129152Z","iopub.execute_input":"2021-08-26T13:08:29.129693Z","iopub.status.idle":"2021-08-26T13:08:29.244001Z","shell.execute_reply.started":"2021-08-26T13:08:29.129649Z","shell.execute_reply":"2021-08-26T13:08:29.242875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# basic normalization\n\ntrain_X = train_X / 255.0\n#train_y = train_y / 255.0\nvalid_X = valid_X / 255.0\n#valid_y = valid_y / 255.0\ntest_X = test_X / 255.0","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.245706Z","iopub.execute_input":"2021-08-26T13:08:29.246393Z","iopub.status.idle":"2021-08-26T13:08:29.489831Z","shell.execute_reply.started":"2021-08-26T13:08:29.246316Z","shell.execute_reply":"2021-08-26T13:08:29.488392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reshape_dataset(_set):\n    new_shape = tuple(list(_set.shape)[:3] + [1])\n    _set = _set.reshape(new_shape)\n    return _set","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.491296Z","iopub.execute_input":"2021-08-26T13:08:29.491827Z","iopub.status.idle":"2021-08-26T13:08:29.497698Z","shell.execute_reply.started":"2021-08-26T13:08:29.491743Z","shell.execute_reply":"2021-08-26T13:08:29.496332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = reshape_dataset(train_X)\ntrain_y = reshape_dataset(train_y)\nvalid_X = reshape_dataset(valid_X)\nvalid_y = reshape_dataset(valid_y)\ntest_X = reshape_dataset(test_X)\n\nprint(train_X.shape)\nprint(train_y.shape)\nprint(valid_X.shape)\nprint(valid_y.shape)\nprint(test_X.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.499587Z","iopub.execute_input":"2021-08-26T13:08:29.500059Z","iopub.status.idle":"2021-08-26T13:08:29.513731Z","shell.execute_reply.started":"2021-08-26T13:08:29.500013Z","shell.execute_reply":"2021-08-26T13:08:29.512043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\nif \"GPU\" not in device_name:\n    print(\"GPU device not found\")\nprint('Found GPU at: {}'.format(device_name))","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.515808Z","iopub.execute_input":"2021-08-26T13:08:29.516287Z","iopub.status.idle":"2021-08-26T13:08:29.531862Z","shell.execute_reply.started":"2021-08-26T13:08:29.516241Z","shell.execute_reply":"2021-08-26T13:08:29.53017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create model\n# 28x28x3\nactv = \"relu\"\nmodel = models.Sequential()\n#model = tf.keras.models.load_model(\"50_epoch_model.h5\")\nmodel.add(layers.Conv2D(16, (3, 3), activation=actv, input_shape=(28, 28, 1)))\nmodel.add(layers.BatchNormalization())\n\nmodel.add(layers.Conv2D(32, (3, 3), activation=actv))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPooling2D((2, 2), strides=2))\n\nmodel.add(layers.Conv2D(64, (3, 3), activation=actv))\nmodel.add(layers.BatchNormalization())\n\nmodel.add(layers.Conv2D(64, (3, 3), activation=actv))\nmodel.add(layers.BatchNormalization())\n\nmodel.add(layers.Conv2D(64, (3, 3), activation=actv))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.MaxPooling2D((2, 2), strides=2))\nmodel.add(layers.Dropout(0.25))\n\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(256, activation=actv))\nmodel.add(layers.BatchNormalization())\nmodel.add(layers.Dropout(0.25))\n\nmodel.add(layers.Dense(64, activation=actv))\nmodel.add(layers.BatchNormalization())\n\nmodel.add(layers.Dense(10, activation=\"softmax\"))","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.53372Z","iopub.execute_input":"2021-08-26T13:08:29.534533Z","iopub.status.idle":"2021-08-26T13:08:29.850054Z","shell.execute_reply.started":"2021-08-26T13:08:29.534486Z","shell.execute_reply":"2021-08-26T13:08:29.848679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.851902Z","iopub.execute_input":"2021-08-26T13:08:29.852562Z","iopub.status.idle":"2021-08-26T13:08:29.881877Z","shell.execute_reply.started":"2021-08-26T13:08:29.852518Z","shell.execute_reply":"2021-08-26T13:08:29.88063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loss_ = tf.keras.losses.BinaryCrossentropy(from_logits=False)\n#metric_ = tf.keras.metrics.CategoricalAccuracy(name=\"acc\")\n#optimizer_ = tf.keras.optimizers.Adam(learning_rate=0.02)\nmodel.compile(optimizer=\"adam\", loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.886783Z","iopub.execute_input":"2021-08-26T13:08:29.889428Z","iopub.status.idle":"2021-08-26T13:08:29.908651Z","shell.execute_reply.started":"2021-08-26T13:08:29.889384Z","shell.execute_reply":"2021-08-26T13:08:29.907375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X = tf.convert_to_tensor(train_X, dtype=tf.float32)\ntrain_y = tf.convert_to_tensor(train_y, dtype=tf.float32)\nvalid_X = tf.convert_to_tensor(valid_X, dtype=tf.float32)\nvalid_y = tf.convert_to_tensor(valid_y, dtype=tf.float32)\n\nprint(train_X.shape, type(train_X))\nprint(train_y.shape)\nprint(valid_X.shape)\nprint(valid_y.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:29.914518Z","iopub.execute_input":"2021-08-26T13:08:29.917522Z","iopub.status.idle":"2021-08-26T13:08:30.114619Z","shell.execute_reply.started":"2021-08-26T13:08:29.91747Z","shell.execute_reply":"2021-08-26T13:08:30.113514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_X.shape","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:30.124033Z","iopub.execute_input":"2021-08-26T13:08:30.126899Z","iopub.status.idle":"2021-08-26T13:08:30.138972Z","shell.execute_reply.started":"2021-08-26T13:08:30.12685Z","shell.execute_reply":"2021-08-26T13:08:30.137699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_X, train_y, batch_size=64, epochs=70, verbose=1, validation_data=(valid_X, valid_y))","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:08:30.14486Z","iopub.execute_input":"2021-08-26T13:08:30.147614Z","iopub.status.idle":"2021-08-26T13:12:47.617199Z","shell.execute_reply.started":"2021-08-26T13:08:30.147569Z","shell.execute_reply":"2021-08-26T13:12:47.616022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history.history\n\nloss_array = history.history[\"loss\"]\nval_loss_array = history.history[\"val_loss\"]\nacc_array = history.history[\"accuracy\"]\nval_acc_array = history.history[\"val_accuracy\"]\n\ndef loss_graph(tl,vl):\n    plt1, = plt.plot(range(len(tl)), tl)\n    plt2, = plt.plot(range(len(vl)), vl)\n    plt.title('Train and Valid Losses')\n    plt.legend([plt1,plt2],[\"Train\", \"Valid\"])\n    plt.xlabel('Epoch')\n    plt.ylabel('Loss')\n    plt.show()\n\ndef accuracy_graph(ta,va):\n    plt1, = plt.plot(range(len(ta)), ta)\n    plt2, = plt.plot(range(len(va)), va)\n    plt.title('Train and Valid Losses')\n    plt.legend([plt1,plt2],[\"Train\", \"Valid\"])\n    plt.xlabel('Epoch')\n    plt.ylabel('Accuracy')\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:47.619069Z","iopub.execute_input":"2021-08-26T13:12:47.619532Z","iopub.status.idle":"2021-08-26T13:12:47.63101Z","shell.execute_reply.started":"2021-08-26T13:12:47.619487Z","shell.execute_reply":"2021-08-26T13:12:47.629512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss_graph(loss_array, val_loss_array)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:47.632992Z","iopub.execute_input":"2021-08-26T13:12:47.63368Z","iopub.status.idle":"2021-08-26T13:12:47.861593Z","shell.execute_reply.started":"2021-08-26T13:12:47.633604Z","shell.execute_reply":"2021-08-26T13:12:47.860205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_graph(acc_array, val_acc_array)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:47.863492Z","iopub.execute_input":"2021-08-26T13:12:47.86398Z","iopub.status.idle":"2021-08-26T13:12:48.075036Z","shell.execute_reply.started":"2021-08-26T13:12:47.863934Z","shell.execute_reply":"2021-08-26T13:12:48.073712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model(test_X)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:48.076886Z","iopub.execute_input":"2021-08-26T13:12:48.077419Z","iopub.status.idle":"2021-08-26T13:12:48.258953Z","shell.execute_reply.started":"2021-08-26T13:12:48.077372Z","shell.execute_reply":"2021-08-26T13:12:48.257813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_outputs = []\nfor pred in preds:\n    model_outputs.append(np.array(pred).argmax())","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:48.260542Z","iopub.execute_input":"2021-08-26T13:12:48.260978Z","iopub.status.idle":"2021-08-26T13:12:59.855987Z","shell.execute_reply.started":"2021-08-26T13:12:48.260934Z","shell.execute_reply":"2021-08-26T13:12:59.854805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_outputs = np.array(model_outputs)\n_ids = np.array(list(map(lambda x:x+1,list(np.array(test_set.index.values.tolist())))))\n\nprint(model_outputs.shape)\nprint(_ids.shape)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:59.857535Z","iopub.execute_input":"2021-08-26T13:12:59.857969Z","iopub.status.idle":"2021-08-26T13:12:59.907973Z","shell.execute_reply.started":"2021-08-26T13:12:59.857925Z","shell.execute_reply":"2021-08-26T13:12:59.90663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame({\n    \"ImageId\": _ids,\n    \"Label\": model_outputs\n})","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:59.909753Z","iopub.execute_input":"2021-08-26T13:12:59.910316Z","iopub.status.idle":"2021-08-26T13:12:59.917469Z","shell.execute_reply.started":"2021-08-26T13:12:59.91027Z","shell.execute_reply":"2021-08-26T13:12:59.915767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:59.919474Z","iopub.execute_input":"2021-08-26T13:12:59.920009Z","iopub.status.idle":"2021-08-26T13:12:59.934717Z","shell.execute_reply.started":"2021-08-26T13:12:59.919929Z","shell.execute_reply":"2021-08-26T13:12:59.933431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-26T13:12:59.936442Z","iopub.execute_input":"2021-08-26T13:12:59.936915Z","iopub.status.idle":"2021-08-26T13:12:59.996995Z","shell.execute_reply.started":"2021-08-26T13:12:59.93687Z","shell.execute_reply":"2021-08-26T13:12:59.995844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}