{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport scipy\n\nfrom IPython.display import SVG\nimport IPython.display as display\n\nimport tensorflow as tf\nfrom keras.utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Conv2D, Flatten\nfrom keras.utils import plot_model\nfrom keras.utils import model_to_dot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dfTrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndfTest  = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\nX     = dfTrain[dfTrain.columns[1:]].to_numpy().reshape(42000,28*28)\ny     = to_categorical(dfTrain[dfTrain.columns[0]].to_numpy())\n#y     = dfTrain[dfTrain.columns[0]].to_numpy()\n\ntrain_X, valid_X, train_y, valid_y = train_test_split(X,y,shuffle=False, random_state=1,train_size=0.95)\ntest_X = dfTest.to_numpy().reshape(28000,28*28)\n\nprint(X.shape,y.shape)\nprint(train_X.shape,valid_X.shape,train_y.shape,valid_y.shape)\nprint(test_X.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define various NN Models\n\n* represent 28x28 image as 784 component vector\n* output layer has 10 nodes (representing probability of input being numbers 0-9)\n* model: various number (0 or 4) of hidden layers with various number of nodes (200,100,60,30) and various activation funtions"},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"xPoints = np.linspace(-5,5,1000)\nyPoints = 1/(1+np.exp(-xPoints))\nfig, ax = plt.subplots(1,2,figsize=(15,5))\nax[0].plot(xPoints,yPoints,label='sigmoid')\nyPoints = np.max([np.zeros(len(xPoints)),xPoints],axis=0)\nax[0].plot(xPoints,yPoints,label='relu')\nax[0].set_ylim(-0.5,2)\nax[0].grid(True)\nax[0].legend()\n\nlabels  = np.linspace(0,9,10)\nzPoints = np.random.random(10)\nzPoints[5]=3\nax[1].bar(labels-0.1,zPoints,width=0.2,label='original numbers')\nax[1].bar(labels+0.1,scipy.special.softmax(zPoints),width=0.2,label='softmax numbers')\nax[1].legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelA = tf.keras.Sequential(\n  [\n    tf.keras.layers.Input(shape=(28*28,)),\n    tf.keras.layers.Dense(10, activation='softmax')\n  ])\n\nmodelB = tf.keras.Sequential(\n  [\n    tf.keras.layers.Input(shape=(28*28,)),\n    tf.keras.layers.Dense(200, activation='sigmoid'),\n    tf.keras.layers.Dense(100, activation='sigmoid'),\n    tf.keras.layers.Dense(60, activation='sigmoid'),\n    tf.keras.layers.Dense(30, activation='sigmoid'),\n    tf.keras.layers.Dense(10, activation='softmax')\n  ])\n\nmodelC = tf.keras.Sequential(\n  [\n    tf.keras.layers.Input(shape=(28*28,)),\n    tf.keras.layers.Dense(200, activation='relu'),\n    tf.keras.layers.Dense(100, activation='relu'),\n    tf.keras.layers.Dense(60, activation='relu'),\n    tf.keras.layers.Dense(30, activation='relu'),\n    tf.keras.layers.Dense(10, activation='softmax')\n  ])\n\nmodelD = tf.keras.Sequential(\n  [\n    tf.keras.layers.Input(shape=(28*28,)),\n    tf.keras.layers.Dense(200, activation='relu'),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(100, activation='relu'),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(60, activation='relu'),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(30, activation='relu'),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(10, activation='softmax')\n  ])\n\nmodelA.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nmodelB.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nmodelC.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nmodelD.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train models "},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 100\nEPOCHS = 30\n\nsteps_per_epoch = 42000//BATCH_SIZE  # 60,000 items in this dataset\nprint(\"Steps per epoch: \", steps_per_epoch)\nhistoryA = modelA.fit(train_X, train_y, \n                    steps_per_epoch=steps_per_epoch, \n                    epochs=EPOCHS,\n                    validation_data=(valid_X, valid_y), \n                    validation_steps=1)\n\nhistoryB = modelB.fit(train_X, train_y, \n                    steps_per_epoch=steps_per_epoch, \n                    epochs=EPOCHS,\n                    validation_data=(valid_X, valid_y), \n                    validation_steps=1)\n\nhistoryC = modelC.fit(train_X, train_y, \n                    steps_per_epoch=steps_per_epoch, \n                    epochs=EPOCHS,\n                    validation_data=(valid_X, valid_y), \n                    validation_steps=1)\n\nhistoryD = modelD.fit(train_X, train_y, \n                    steps_per_epoch=steps_per_epoch, \n                    epochs=EPOCHS,\n                    validation_data=(valid_X, valid_y), \n                    validation_steps=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display model performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(15,5))\nax[0].plot(historyA.history['accuracy'],    color='red', linestyle='-',label='1 layer (10) softmax')\nax[0].plot(historyA.history['val_accuracy'],color='blue',linestyle='-')\nax[0].plot(historyB.history['accuracy'],    color='red', linestyle='-.',label='5 layers (200/100/60/30/10) sigmoid/softmax')\nax[0].plot(historyB.history['val_accuracy'],color='blue',linestyle='-.')\nax[0].plot(historyC.history['accuracy'],    color='red', linestyle='--',label='5 layers (200/100/60/30/10) relu/softmax')\nax[0].plot(historyC.history['val_accuracy'],color='blue',linestyle='--')\nax[0].plot(historyD.history['accuracy'],    color='red', linestyle=':',label='5 layers (200/100/60/30/10) relu/softmax + dropout')\nax[0].plot(historyD.history['val_accuracy'],color='blue',linestyle=':')\nax[0].set_title('Model accuracy')\nax[0].set_ylabel('Accuracy')\nax[0].set_xlabel('Epoch')\nax[0].set_ylim(0.75,1.0)\nax[0].legend()\nax[0].grid(True)\n\n# Plot training & validation loss values\nax[1].semilogy(historyA.history['loss'],    color='red', linestyle='-',label='1 layer (10) softmax')\nax[1].semilogy(historyA.history['val_loss'],color='blue',linestyle='-')\nax[1].semilogy(historyB.history['loss'],    color='red', linestyle='-.',label='5 layers (200/100/60/30/10) sigmoid/softmax')\nax[1].semilogy(historyB.history['val_loss'],color='blue',linestyle='-.')\nax[1].semilogy(historyC.history['loss'],    color='red', linestyle='--',label='5 layers (200/100/60/30/10) relu/softmax')\nax[1].semilogy(historyC.history['val_loss'],color='blue',linestyle='--')\nax[1].semilogy(historyD.history['loss'],    color='red', linestyle=':',label='5 layers (200/100/60/30/10) relu/softmax + dropout')\nax[1].semilogy(historyD.history['val_loss'],color='blue',linestyle=':')\nax[1].set_title('Model loss')\nax[1].set_ylabel('Loss')\nax[1].set_xlabel('Epoch')\nax[1].legend()\nax[1].grid(True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define CNN Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 100\nEPOCHS = 30\n\nsteps_per_epoch = 42000//BATCH_SIZE\n\nmodelE = tf.keras.Sequential(\n  [\n    tf.keras.layers.Reshape(input_shape=(28*28,), target_shape=(28,28,1)),\n    tf.keras.layers.Conv2D(kernel_size=6, filters=6,  strides=1, padding='same', activation='relu'),\n    tf.keras.layers.Conv2D(kernel_size=5, filters=12, strides=2, padding='same', activation='relu'),\n    tf.keras.layers.Conv2D(kernel_size=4, filters=24, strides=2, padding='same', activation='relu'),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dropout(0.25),\n    tf.keras.layers.Dense(256, activation='relu'),\n    tf.keras.layers.Dense(10, activation='softmax')\n  ])\n\nmodelF = tf.keras.Sequential(\n  [\n    tf.keras.layers.Reshape(input_shape=(28*28,), target_shape=(28,28,1)),\n    tf.keras.layers.Conv2D(kernel_size=5, filters=24, padding='same', activation='relu'),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Conv2D(kernel_size=5, filters=48, padding='same', activation='relu'),\n    tf.keras.layers.MaxPool2D(),\n    tf.keras.layers.Conv2D(kernel_size=5, filters=64, padding='same', activation='relu'),\n    tf.keras.layers.MaxPool2D(padding='same'),\n    tf.keras.layers.Flatten(),\n    tf.keras.layers.Dense(256, activation='relu'),\n    tf.keras.layers.Dense(10, activation='softmax')\n  ])\n\nmodelE.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])\nmodelF.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{},"cell_type":"markdown","source":"### Train Models"},{"metadata":{"trusted":true},"cell_type":"code","source":"historyE = modelE.fit(train_X, train_y, \n                    steps_per_epoch=steps_per_epoch, \n                    epochs=EPOCHS,\n                    validation_data=(valid_X, valid_y), \n                    validation_steps=1)\n\nhistoryF = modelF.fit(train_X, train_y, \n                    steps_per_epoch=steps_per_epoch, \n                    epochs=EPOCHS,\n                    validation_data=(valid_X, valid_y), \n                    validation_steps=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Display CNN model performance"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(15,5))\nax[0].plot(historyE.history['accuracy'],    color='red', linestyle='-')\nax[0].plot(historyE.history['val_accuracy'],color='blue',linestyle='-')\nax[0].plot(historyF.history['accuracy'],    color='red', linestyle='-.')\nax[0].plot(historyF.history['val_accuracy'],color='blue',linestyle='-.')\nax[0].set_title('Model accuracy')\nax[0].set_ylabel('Accuracy')\nax[0].set_xlabel('Epoch')\nax[0].set_ylim(0.90,1.0)\nax[0].legend()\nax[0].grid(True)\n\n# Plot training & validation loss values\nax[1].semilogy(historyE.history['loss'],    color='red', linestyle='-')\nax[1].semilogy(historyE.history['val_loss'],color='blue',linestyle='-')\nax[1].semilogy(historyF.history['loss'],    color='red', linestyle='-.')\nax[1].semilogy(historyF.history['val_loss'],color='blue',linestyle='-.')\nax[1].set_title('Model loss')\nax[1].set_ylabel('Loss')\nax[1].set_xlabel('Epoch')\nax[1].legend()\nax[1].grid(True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### ResNet + enhanced dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Conv2D, BatchNormalization, Activation\nfrom keras.layers import Add, Flatten, AveragePooling2D, Dense, Dropout\nfrom keras.callbacks import ModelCheckpoint, LearningRateScheduler\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.utils import plot_model\nimport time\n\n\ndef residual_block(inputs, filters, strides=1):\n    \"\"\"Residual block\n    \n    Shortcut after Conv2D -> ReLU -> BatchNorm -> Conv2D\n    \n    Arguments:\n        inputs (tensor): input\n        filters (int): Conv2D number of filterns\n        strides (int): Conv2D square stride dimensions\n\n    Returns:\n        x (tensor): input Tensor for the next layer\n    \"\"\"\n    y = inputs # Shortcut path\n    \n    # Main path\n    x = Conv2D(kernel_size=3,filters=filters,strides=strides,padding='same',)(inputs)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    x = Conv2D(kernel_size=3,filters=filters,strides=1,padding='same',)(x)\n    x = BatchNormalization()(x)\n    \n    # Fit shortcut path dimenstions\n    if strides > 1:\n        y = Conv2D(kernel_size=3,filters=filters,strides=strides,padding='same',)(y)\n        y = BatchNormalization()(y)\n    \n    # Concatenate paths\n    x = Add()([x, y])\n    x = Activation('relu')(x)\n    \n    return x\n    \n    \ndef resnet(input_shape, num_classes, filters, stages):\n    \"\"\"ResNet \n    \n    At the beginning of each stage downsample feature map size \n    by a convolutional layer with strides=2, and double the number of filters.\n    The kernel size is the same for each residual block.\n    \n    Arguments:\n        input_shape (3D tuple): shape of input Tensor\n        filters (int): Conv2D number of filterns\n        stages (1D list): list of number of resiual block in each stage eg. [2, 5, 5, 2]\n    \n    Returns:\n        model (Model): Keras model\n    \"\"\"\n    # Start model definition\n    inputs = Input(shape=input_shape)\n    x = Conv2D(kernel_size=7,filters=filters,strides=1,padding='same',)(inputs)\n    x = BatchNormalization()(x)\n    x = Activation('relu')(x)\n    \n    # Stack residual blocks\n    for stage in stages:\n        x = residual_block(x, filters, strides=2)\n        for i in range(stage-1):\n            x = residual_block(x, filters)\n        filters *= 2\n        \n    # Pool -> Flatten -> Classify\n    x = AveragePooling2D(4)(x)\n    x = Flatten()(x)\n    x = Dropout(0.3)(x)\n    x = Dense(int(filters/4), activation='relu')(x)\n    outputs = Dense(num_classes, activation='softmax')(x)\n    \n    # Instantiate model\n    model = Model(inputs=inputs, outputs=outputs)\n    return model \n\n\n# Reshape and normalize\nX = dfTrain.drop(columns=['label']).values.reshape(-1, 28, 28, 1) / 255\ny = dfTrain['label'].values\n\n# Get training and testing datasets\nX_train, X_valid, y_train, y_valid = train_test_split(X, y,shuffle=False, random_state=1,train_size=0.95)\nX_test = dfTest.values.reshape(-1, 28, 28, 1) / 255\n\nprint(X.shape,y.shape)\nprint(X_train.shape,X_valid.shape,y_train.shape,y_valid.shape)\nprint(test_X.shape)\n\nepochs=10\nfilters=64\nstages=[3, 3, 3]\nbatch_size=128\n\nmodelG = resnet(input_shape=X[0].shape,num_classes=np.unique(y).shape[-1],filters=filters,stages=stages)\nmodelH = resnet(input_shape=X[0].shape,num_classes=np.unique(y).shape[-1],filters=filters,stages=stages)\nmodelG.compile(optimizer='adam', loss='sparse_categorical_crossentropy',metrics=['accuracy'])\nmodelH.compile(optimizer='adam', loss='sparse_categorical_crossentropy',metrics=['accuracy'])\n\n\n# Define callbacks\ncheckpoint = ModelCheckpoint(\n    filepath=f'resnet-{int(time.time())}.dhf5',\n    monitor='loss',\n    save_best_only=True\n)\n\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.8**x)\ncallbacks = [checkpoint, annealer]\n\n# Define data generator\ndatagen = ImageDataGenerator(  \n    rotation_range=10,  \n    zoom_range=0.1, \n    width_shift_range=0.1, \n    height_shift_range=0.1\n)\ndatagen.fit(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"historyG = modelG.fit(\n    X_train, y_train,\n    validation_data=(X_valid, y_valid),\n    epochs=epochs, \n    verbose=1,\n    callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"historyH = modelH.fit(\n    datagen.flow(X_train, y_train, batch_size=batch_size),\n    validation_data=(X_valid, y_valid),\n    epochs=epochs, \n    verbose=1,\n    callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(15,5))\nax[0].plot(historyG.history['accuracy'],    color='red', linestyle='-',label='resNet')\nax[0].plot(historyG.history['val_accuracy'],color='blue',linestyle='-')\nax[0].plot(historyH.history['accuracy'],    color='red', linestyle='-.',label='resNet + enhanced data')\nax[0].plot(historyH.history['val_accuracy'],color='blue',linestyle='-.')\nax[0].set_title('Model accuracy')\nax[0].set_ylabel('Accuracy')\nax[0].set_xlabel('Epoch')\nax[0].set_ylim(0.90,1.0)\nax[0].legend()\nax[0].grid(True)\n\n# Plot training & validation loss values\nax[1].semilogy(historyG.history['loss'],    color='red', linestyle='-',label='resNet')\nax[1].semilogy(historyG.history['val_loss'],color='blue',linestyle='-')\nax[1].semilogy(historyH.history['loss'],    color='red', linestyle='-.',label='resNet + enhanced data')\nax[1].semilogy(historyH.history['val_loss'],color='blue',linestyle='-.')\nax[1].set_title('Model loss')\nax[1].set_ylabel('Loss')\nax[1].set_xlabel('Epoch')\nax[1].legend()\nax[1].grid(True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def writeSubmission(modelDict, test):\n    for k in modelDict.keys():\n        model = modelDict[k]\n        output     = model.predict(test)\n        prediction = np.argmax(output,axis=1)\n\n        dfOut = pd.DataFrame(data={'ImageId':np.arange(1,28001,1),'Label':prediction})\n        dfOut.to_csv('submission'+k+'.csv', index=False)\n        if k == 'H':\n            dfOut.to_csv('submission.csv', index=False)\n        \nmodelDict1 = {'A':modelA, 'B':modelB, 'C':modelC, 'D':modelD, 'E':modelE, 'F':modelF}\nmodelDict2 = {'G':modelG, 'H':modelH}\n\nwriteSubmission(modelDict1, test_X)\nwriteSubmission(modelDict2, X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}