{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow import keras\n\n# regularization\nfrom keras import regularizers\n\n# optimizer\nfrom tensorflow.keras.optimizers import RMSprop\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import LearningRateScheduler\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/digit-recognizer/test.csv\")\nsubmission = pd.read_csv(\"/kaggle/input/digit-recognizer/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train shape: \", train.shape)\nprint(\"test shape: \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train.drop(['label'], axis = 1).values/255\nY = train['label'].values\n\nX_valid = test.values/255\n\n# reshape\nX = X.reshape(X.shape[0],28,28,1)\nX_valid = X_valid.reshape(X_valid.shape[0],28,28,1)\n#Y = tf.keras.utils.to_categorical(Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train shape: \", X.shape)\nprint(\"test shape: \", X_valid.shape)\nprint(\"target shape: \", Y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_dev, Y_train,Y_dev = train_test_split(X,Y,test_size = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(X_train[1][:,:,0])\nplt.title(str(Y_train[1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(X[i][:,:,0], cmap=plt.cm.binary)\n    plt.xlabel(train.label[i])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f = 2\n\nmodel = tf.keras.Sequential([\n    tf.keras.layers.Conv2D(f*16,kernel_size = (3,3), padding = 'same',activation='relu',\n                           kernel_initializer='he_uniform',\n                           input_shape = (28,28,1)),\n    tf.keras.layers.Conv2D(f*16, (3,3), activation = \"relu\", padding = 'same'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(),\n    \n    tf.keras.layers.Conv2D(f*32, kernel_size = (3,3), padding = 'same',activation='relu'),\n    tf.keras.layers.Conv2D(f*32, (3,3), activation = \"relu\", padding = 'same'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(),\n    tf.keras.layers.Dropout(0.25),\n    \n    tf.keras.layers.Conv2D(f*64, kernel_size = (3,3), padding = 'same',activation='relu'),\n    tf.keras.layers.Conv2D(f*64, (3,3), activation = \"relu\", padding = 'same'),\n    tf.keras.layers.BatchNormalization(),\n    tf.keras.layers.MaxPooling2D(),\n    tf.keras.layers.Dropout(0.125),\n    \n    keras.layers.Flatten(),\n    keras.layers.Dense(512, activation = 'relu'),\n    keras.layers.Dense(128, activation = 'relu'),\n    keras.layers.Dense(10, activation='softmax')\n    \n])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils import plot_model\n#tf.keras.utils.plot_model(\n#    model,\n#    to_file='model.png',\n#    show_shapes=True,\n#    show_layer_names=True,\n#    rankdir='TB',\n#    expand_nested=False,\n#    dpi=96\n#)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=Adam(learning_rate=0.0003),\n             loss = 'sparse_categorical_crossentropy',\n             metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler\n\ntrain_datagen = ImageDataGenerator(rotation_range=10,\n                                   width_shift_range=0.15,\n                                   height_shift_range=0.15,\n                                   shear_range=0.05,\n                                   zoom_range=0.15,\n                                   horizontal_flip=False)\n\nvalid_datagen = ImageDataGenerator(horizontal_flip=False,\n                                    #rotation_range=10,\n                                    #width_shift_range=0.25,height_shift_range=0.2,\n                                    #shear_range=0.1,zoom_range=0.25\n                                    )\n\n# add early stopping\ncallback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)\n\n# learning rate decay\ndef lr_decay(epoch, initial_learningrate = 0.0003):#lrv\n    return initial_learningrate * 0.99 ** epoch\n\n# Set a learning rate annealer\nlearning_rate_reduction = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_acc', \n                                            patience=300, \n                                            verbose=1, \n                                            factor=0.5, \n                                            min_lr=0.00001)\n\n# fit model with generated data\nbatchsize = 512*2\nepoch = 45\n\nhistory = model.fit_generator(train_datagen.flow(X, Y, batch_size = batchsize),\n                   steps_per_epoch = 100, \n                    epochs = epoch,\n                   callbacks=[#learning_rate_reduction, \n                              LearningRateScheduler(lr_decay),\n                              callback],\n                   validation_data=valid_datagen.flow(X_dev, Y_dev),\n                    validation_steps=50,\n                   )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add early stopping\n#callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)\n\n# train model\n#history = model.fit(X_train, Y_train, epochs=30,\n#          callbacks=[callback],\n#          validation_data=[X_dev,Y_dev]\n#         )\n\n# fits the model on batches with real-time data augmentation:\n#model.fit_generator(datagen.flow(X_train, Y_train, batch_size=32),\n#                    steps_per_epoch=len(X) // 32, epochs=45,\n                    #callbacks = [callback],\n#                    validation_data = (X_dev, Y_dev)\n#                   )\n\n# evaluate model performance\ntest_loss, test_acc = model.evaluate(X_dev, Y_dev,verbose=2)\n\nprint('\\nTest accuracy: ', test_acc)\nprint('\\nTest loss: ', test_loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yhat = model.predict_classes(X_valid)\nsubmission['Label']=pd.Series(yhat)\nsubmission.to_csv('submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize = (10,5))\nplt.subplot(1,2,1)\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('Model accuracy')\nplt.ylabel('Accuracy')\nplt.xlabel('Epoch')\nplt.legend(['Train','Validation'])\n\nplt.subplot(1,2,2)\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('Model loss')\nplt.ylabel('Loss')\nplt.xlabel('Epoch')\nplt.legend(['Train','Validation'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}