{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ntrain=pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest=pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:00.775110Z","iopub.execute_input":"2021-09-15T20:52:00.775442Z","iopub.status.idle":"2021-09-15T20:52:06.090040Z","shell.execute_reply.started":"2021-09-15T20:52:00.775369Z","shell.execute_reply":"2021-09-15T20:52:06.089179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=train.drop('label',axis=1).values.astype('float32')\ny=train['label'].values.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:06.092406Z","iopub.execute_input":"2021-09-15T20:52:06.092996Z","iopub.status.idle":"2021-09-15T20:52:06.237193Z","shell.execute_reply.started":"2021-09-15T20:52:06.092957Z","shell.execute_reply":"2021-09-15T20:52:06.236207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\n\nfrom keras.utils import np_utils\ny = np_utils.to_categorical(y) ","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:06.238958Z","iopub.execute_input":"2021-09-15T20:52:06.239309Z","iopub.status.idle":"2021-09-15T20:52:10.813066Z","shell.execute_reply.started":"2021-09-15T20:52:06.239274Z","shell.execute_reply":"2021-09-15T20:52:10.812195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# # PREVIEW IMAGES\n# plt.figure(figsize=(15,4.5))\n# for i in range(30):  \n#     plt.subplot(3, 10, i+1)\n#     plt.imshow(X[i].reshape((28,28)),cmap=plt.cm.binary)\n#     plt.axis('off')\n# plt.subplots_adjust(wspace=-0.1, hspace=-0.1)\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:10.814758Z","iopub.execute_input":"2021-09-15T20:52:10.815127Z","iopub.status.idle":"2021-09-15T20:52:10.818987Z","shell.execute_reply.started":"2021-09-15T20:52:10.815088Z","shell.execute_reply":"2021-09-15T20:52:10.818043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE MORE IMAGES VIA DATA AUGMENTATION\n#rescale by 225 since there are 225 pixels per digit.\n\nfrom keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img\n\ndatagen_rotate = ImageDataGenerator(\n    rotation_range=10,  \n    zoom_range = 0.10,  \n    width_shift_range=0.1, \n    height_shift_range=0.1,\n    rescale=1/255.)\ndatagen_shear = ImageDataGenerator(shear_range=0.2)\n#datagen_whitening = ImageDataGenerator(zca_whitening=True)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:10.820365Z","iopub.execute_input":"2021-09-15T20:52:10.820927Z","iopub.status.idle":"2021-09-15T20:52:10.832838Z","shell.execute_reply.started":"2021-09-15T20:52:10.820878Z","shell.execute_reply":"2021-09-15T20:52:10.831956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train,X_test,y_train, y_test=train_test_split(X,y,random_state=0)\n#expand 1 more dimention as 1 for colour channel gray\nX_train = X_train.reshape(X_train.shape[0], 28, 28,1)\nX_test = X_test.reshape(X_test.shape[0], 28, 28,1)\nX_test.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:10.834015Z","iopub.execute_input":"2021-09-15T20:52:10.834656Z","iopub.status.idle":"2021-09-15T20:52:11.676144Z","shell.execute_reply.started":"2021-09-15T20:52:10.834622Z","shell.execute_reply":"2021-09-15T20:52:11.675386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.models import Sequential\nfrom tensorflow.keras.layers.experimental import preprocessing\nfrom keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Flatten\n\n# with tpu_strategy.scope():\nmodel = keras.Sequential([\n    layers.InputLayer(input_shape=(28,28,1)),\n\n    layers.Conv2D(filters=32,kernel_size=3,activation='relu'),\n    layers.BatchNormalization(),\n    layers.Conv2D(filters=32,kernel_size=3,activation='relu'),\n    layers.BatchNormalization(),\n    layers.Conv2D(filters=32,kernel_size=5,strides=2,padding='same',activation='relu'),\n    #layers.MaxPooling2D(),\n    layers.BatchNormalization(),\n    layers.Dropout(0.4),\n\n    layers.Conv2D(64, kernel_size = 3, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Conv2D(64, kernel_size = 3, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'),\n    layers.BatchNormalization(),\n    layers.Dropout(0.4),\n\n    layers.Conv2D(128, kernel_size = 3, activation='relu'),\n    layers.BatchNormalization(),\n    layers.Flatten(),\n    layers.Dropout(0.4),\n    layers.Dense(10, activation='softmax'),\n])\n    ","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:11.677385Z","iopub.execute_input":"2021-09-15T20:52:11.677708Z","iopub.status.idle":"2021-09-15T20:52:13.736438Z","shell.execute_reply.started":"2021-09-15T20:52:11.677674Z","shell.execute_reply":"2021-09-15T20:52:13.735595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer='adam',\n    loss='categorical_crossentropy',\n    metrics=['accuracy'],)\n    #options = run_opts)\ndatagen_rotate.fit(X_train)\ndatagen_shear.fit(X_train)\n#datagen_whitening.fit(X_train)\nmodel.fit_generator(datagen_rotate.flow(X_train, y_train, batch_size = 32),\n                    validation_data = (X_test, y_test), steps_per_epoch = len(X_train) // 32,\n                    epochs = 10)\nmodel.fit_generator(datagen_shear.flow(X_train, y_train, batch_size = 32),\n                    validation_data = (X_test, y_test), steps_per_epoch = len(X_train) // 32,\n                    epochs = 10)\n#model.fit_generator(datagen_whitening.flow(X_train, y_train, batch_size = 32),\n#                    validation_data = (X_test, y_test), steps_per_epoch = len(X_train) // 32,\n#                    epochs = 10)\nearly_stopping = keras.callbacks.EarlyStopping(\n    patience=5,\n    min_delta=0.01,\n    restore_best_weights=True,\n)\n\nhistory = model.fit(\n    X_train, y_train,\n    validation_data=(X_test, y_test),\n    batch_size=64,\n    epochs=150,\n)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T20:52:13.740318Z","iopub.execute_input":"2021-09-15T20:52:13.740566Z","iopub.status.idle":"2021-09-15T21:08:41.458605Z","shell.execute_reply.started":"2021-09-15T20:52:13.740542Z","shell.execute_reply":"2021-09-15T21:08:41.457766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# for x_batch, y_batch in datagen_rotate.flow(X_train, y_train, batch_size=9):\n#   for i in range(0, 9):\n#     plt.subplot(330 + 1 + i)\n#     plt.imshow(x_batch[i].reshape(28, 28), cmap=('gray')) \n#   plt.savefig('rotate.png')\n#   plt.show()\n#   break","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:08:41.460464Z","iopub.execute_input":"2021-09-15T21:08:41.460721Z","iopub.status.idle":"2021-09-15T21:08:41.464945Z","shell.execute_reply.started":"2021-09-15T21:08:41.460695Z","shell.execute_reply":"2021-09-15T21:08:41.464085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for x_batch, y_batch in datagen_shear.flow(X_train, y_train, batch_size=9):\n#   for i in range(0, 9):\n#     plt.subplot(330 + 1 + i)\n#     plt.imshow(x_batch[i].reshape(28, 28), cmap=('gray')) \n#   plt.savefig('shear.png')\n#   plt.show()\n#   break","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:08:41.466171Z","iopub.execute_input":"2021-09-15T21:08:41.466543Z","iopub.status.idle":"2021-09-15T21:08:41.473533Z","shell.execute_reply.started":"2021-09-15T21:08:41.466504Z","shell.execute_reply":"2021-09-15T21:08:41.472652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# for x_batch, y_batch in datagen_whitening.flow(X_train, y_train, batch_size=9):\n#   for i in range(0, 9):\n#     plt.subplot(330 + 1 + i)\n#     plt.imshow(x_batch[i].reshape(28, 28), cmap=('gray')) \n#   plt.savefig('whitening.png')\n#   plt.show()\n#   break","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:11:53.560414Z","iopub.execute_input":"2021-09-15T21:11:53.560732Z","iopub.status.idle":"2021-09-15T21:11:54.424278Z","shell.execute_reply.started":"2021-09-15T21:11:53.560702Z","shell.execute_reply":"2021-09-15T21:11:54.423472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history_df = pd.DataFrame(history.history)\nhistory_df[['loss', 'val_loss']].loc[20:].plot(title=\"Loss: Training data vs validation data\")\nhistory_df[['accuracy', 'val_accuracy']].loc[20:].plot(title=\"Accuracy: Training vs validation data\")","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:11:56.780021Z","iopub.execute_input":"2021-09-15T21:11:56.780353Z","iopub.status.idle":"2021-09-15T21:11:57.150351Z","shell.execute_reply.started":"2021-09-15T21:11:56.780319Z","shell.execute_reply":"2021-09-15T21:11:57.149442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# figure = plt.figure(figsize=(20,20))\n# for i in range(15):\n#     figure.add_subplot(1,15,i+1)\n#     plt.imshow(X_train[i+50,:,:,0],cmap=\"gray\")\n#     plt.axis(\"off\")\n#     print(np.squeeze(np.argmax(model.predict(X_train[i+50].reshape(1,28,28,1)),axis=1),axis=0),end=\"\\t\")","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:08:41.523970Z","iopub.status.idle":"2021-09-15T21:08:41.524815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.layers","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:08:41.528272Z","iopub.status.idle":"2021-09-15T21:08:41.530170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# first_conv = model.layers[0]\n# output_layer1 = first_conv(test_images)[4]\n\n# print(output_layer1.shape)\n# fig, ax = plt.subplots(3,3)\n# fig.set_size_inches(10,10)\n# for i in range(9):\n#   ax[i//3][i%3].imshow(output_layer1[:,:,i])","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:08:41.532710Z","iopub.status.idle":"2021-09-15T21:08:41.533367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test.shape, X_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:08:41.535784Z","iopub.status.idle":"2021-09-15T21:08:41.537578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test= test.values.astype('float32')\ntest = test.reshape(test.shape[0], 28, 28,1)\nlabel=model.predict_classes(test)\nlabel","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:12:06.316812Z","iopub.execute_input":"2021-09-15T21:12:06.317164Z","iopub.status.idle":"2021-09-15T21:12:08.093275Z","shell.execute_reply.started":"2021-09-15T21:12:06.317134Z","shell.execute_reply":"2021-09-15T21:12:08.092185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission = pd.DataFrame({'ImageId': list(range(1,len(label)+1)), 'Label': label})\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-15T21:12:09.682907Z","iopub.execute_input":"2021-09-15T21:12:09.683233Z","iopub.status.idle":"2021-09-15T21:12:09.747120Z","shell.execute_reply.started":"2021-09-15T21:12:09.683186Z","shell.execute_reply":"2021-09-15T21:12:09.746324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}