{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import Necessary Packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport tensorflow as tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Train Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"# load training data\ntrain_data = pd.read_csv('../input/digit-recognizer/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploratory Data Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'There are {train_data.shape[0]} rows and {train_data.shape[1]} columns')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Separate features X (independent variables) and target/label y (dependent variable)\nX_train = train_data.values[:, 1:]\ny_train = train_data.values[:, 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# the value range of X\nX_train.min(), X_train.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# the array shape of a single data (image)\nX_train[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sqrt(784)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = 10\n\n# reshape the image data into 2D array, and plot the image\nplt.imshow(X_train[idx].reshape(28,28), cmap=plt.cm.binary);\n\n# show the label of the corresponding image\ny_train[idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# count and show the number of data for each label/target y\nsns.countplot(y_train);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Preprocessing"},{"metadata":{"trusted":true,"_uuid":"bc42f0dc1270497e7952518444ed53b310f721f3"},"cell_type":"code","source":"# normalizing features (pixels)\nX_train = X_train / 255.0\n\n# one-hot-encoding target (digit 0-9)\ny_train = tf.keras.utils.to_categorical(y_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build CNN Model"},{"metadata":{"trusted":true,"_uuid":"1a44f582b05a7b6503686e571aef0097eb64a72a"},"cell_type":"code","source":"def build_model():\n    tf.keras.backend.clear_session()\n\n    # define a model\n    model = tf.keras.Sequential()\n\n    # add the first convolution layer\n    model.add(tf.keras.layers.Convolution2D(filters = 16, kernel_size = (3,3), activation = 'relu', input_shape = (28, 28, 1)))\n\n    # add the first pooling layer\n    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2,2)))\n\n    # add regularization\n    model.add(tf.keras.layers.Dropout(0.3))\n\n    # add the second convolution layer\n    model.add(tf.keras.layers.Convolution2D(filters = 16, kernel_size = (3,3), activation = 'relu'))\n\n    # add the second pooling layer\n    model.add(tf.keras.layers.MaxPooling2D(pool_size=(2,2)))\n\n    # add regularization\n    model.add(tf.keras.layers.Dropout(0.3))\n\n    # flatten the array (from 2D to 1D)\n    model.add(tf.keras.layers.Flatten())\n\n    # add the first fully-connected-layer\n    model.add(tf.keras.layers.Dense(units = 128, activation = 'relu'))\n\n    # add the second fully-connected-layer\n    model.add(tf.keras.layers.Dense(units = 128, activation = 'relu'))\n\n    # add the output layer\n    model.add(tf.keras.layers.Dense(units = 10, activation = 'softmax'))\n\n    # compile the model\n    model.compile(optimizer = 'adam', loss = 'categorical_crossentropy', metrics = ['accuracy']) \n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# show the network architecture\nmodel = build_model()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model (Single Validation)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# create train and validation data\nfrom sklearn.model_selection import train_test_split\nX_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=9999)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4085db764157c24c5327674bd977ec0033c2ccf6"},"cell_type":"code","source":"# reshape data to fit Keras's input configuration (rank 4 tensor: (rows, pixel, pixel, channel))\n\nX_tr = X_tr.reshape(X_tr.shape[0], 28, 28, 1)\nX_val = X_val.reshape(X_val.shape[0], 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"091d55c477fa9e2cf10d8e813a2d63b86dfa50f5","scrolled":true},"cell_type":"code","source":"# define early stopping\nearlystop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)\n\n# fit the model and save the information in history\nhistory = model.fit(\n    X_tr,\n    y_tr,\n    batch_size = 64,\n    epochs = 100,\n    validation_data = (X_val, y_val),\n    callbacks = [earlystop])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check History"},{"metadata":{"trusted":true,"_uuid":"044bab4bcd166a00539d320f0060e7670c43983c"},"cell_type":"code","source":"# plot history\nfig, ax = plt.subplots(1, 2, figsize=(18,6))\nax[0].plot(history.history['loss'], label='train')\nax[0].plot(history.history['val_loss'], label='valid')\nax[1].plot(history.history['acc'], label='train')\nax[1].plot(history.history['val_acc'], label='valid')\nax[0].set_title('Loss')\nax[1].set_title('Accuracy')\nax[0].legend(); ax[1].legend();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evaluation on Validation Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"results = model.evaluate(X_val, y_val, batch_size=64)\nprint(f\"valid loss: {results[0]}, valid acc: {round(results[1]*100,2)}%\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get probabilities\nprobabilities = model.predict(X_val)\n\n# get the prediction class\ny_pred = np.argmax(probabilities, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pick random data\nidx = np.random.randint(0, X_val.shape[0], 32)\n\nfig, ax = plt.subplots(4, 8, figsize=(18,12))\nk = 0\nfor i in range(4):\n    for j in range(8):\n        x = X_val[idx[k]]\n        y = np.argmax(y_val[idx[k]])\n        ypred = y_pred[idx[k]]\n        ax[i,j].imshow(x.reshape(28,28), cmap = plt.cm.binary)\n        ax[i,j].set_title(f'pred:{ypred} ---- truth:{y}')\n        ax[i,j].axis('off')\n        k += 1    \nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize The Layers"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get Convolution and Pooling layers\nlayer_number = [0,1,3,4]\nlayers = [model.layers[i].output for i in layer_number]\n\n# get the layer names\nlayer_names = [model.layers[i].name for i in layer_number]\n\n# define visualization model\nviz_model = tf.keras.models.Model(inputs = model.input, outputs = layers)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# define a sample input image\nidx = 16\nxviz = X_val[idx].reshape(1,28,28,1)\nyviz = y_val[idx]\n\n# get feature maps for the image\nfeature_maps = viz_model.predict(xviz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge feature maps for each layer\n\nfeats = []\nfor i, name in enumerate(layer_names):\n    N = feature_maps[i].shape[-1]\n    size = feature_maps[i].shape[1]\n    ll = [feature_maps[i][:,:,:,j].reshape(size,size) for j in range(N)]\n    feats.append(np.hstack(ll))\n\n# plot the feature maps\nfig, ax = plt.subplots(len(layer_names), 1, figsize=(18,2*len(layer_names)))\nfor i, (name, feat) in enumerate(zip(layer_names,feats)):\n    ax[i].axis('off')\n    ax[i].set_title(name)\n    ax[i].imshow(feat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get probabilities\nprobabilities = model.predict(xviz)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(18,4))\nax[0].imshow(xviz.reshape(28,28), cmap = plt.cm.binary)\nax[0].set_title(f'truth: {np.argmax(yviz)} --- prediction: {np.argmax(probabilities)}')\nax[0].axis('off')\nax[1].set_xlabel('digit')\nax[1].set_ylabel('probability')\nsns.barplot(np.arange(10), probabilities[0], ax=ax[1]);\ndf = pd.DataFrame({'digit': range(10), 'probability (%)': probabilities[0]*100})\ndf.sort_values(['probability (%)'], ascending=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Test Data"},{"metadata":{"trusted":true,"_uuid":"0f6805fe4c20244e3fe99ce1b803428e890bd684"},"cell_type":"code","source":"test = pd.read_csv('../input/digit-recognizer/test.csv')\nX_test = test.values / 255.0\nX_test = X_test.reshape(X_test.shape[0], 28, 28, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Models (Cross Validation)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import accuracy_score\n\ndef kfold_validation(X_train, y_train, X_test, n_folds=5, seed=9999):\n    \n    # Kfold split\n    kf = StratifiedKFold(n_splits = n_folds, shuffle = True, random_state = seed)\n\n    # Create oof sets for prediction storage.\n    oof_train = np.zeros((X_train.shape[0]))\n    oof_test = np.zeros((X_test.shape[0], n_folds))\n    \n    # Save all models validation accuracy\n    folds_acc = []\n    \n    # Define early stopping\n    earlystop = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)\n    \n    # Reshape data to fit Keras's input configuration (rank 4 tensor: (rows, pixel, pixel, channel))\n    X_test = X_test.reshape(X_test.shape[0], 28, 28, 1)\n        \n    for ifold, (train_index, valid_index) in enumerate(kf.split(X=X_train, y=y_train)):\n        assert len(np.intersect1d(train_index, valid_index)) == 0, '\\\n        Train and test indices must not overlap.'\n        \n        print('Running on Dataset: {}'.format(ifold + 1))\n\n        # Create train and validation sets based on KFold indices.\n        X_tr = X_train[train_index,:]\n        X_val = X_train[valid_index,:]\n        y_tr = y_train[train_index]\n        y_val = y_train[valid_index]\n\n        # Reshape data to fit Keras's input configuration (rank 4 tensor: (rows, pixel, pixel, channel))\n        X_tr = X_tr.reshape(X_tr.shape[0], 28, 28, 1)\n        X_val = X_val.reshape(X_val.shape[0], 28, 28, 1)\n\n        # one-hot-encoding target (digit 0-9)\n        y_tr = tf.keras.utils.to_categorical(y_tr)\n        y_val = tf.keras.utils.to_categorical(y_val)\n\n        # Restart model\n        model = build_model()\n        \n        # Train model\n        model.fit(\n            X_tr,\n            y_tr,\n            batch_size = 64,\n            epochs = 100,\n            validation_data = (X_val, y_val),\n            verbose = 0,\n            callbacks = [earlystop])\n\n        # Evaluation\n        results = model.evaluate(X_val, y_val, batch_size=64)\n        print(f\"valid loss: {results[0]}, valid acc: {round(results[1]*100,2)}%\\n\")\n        \n        # Prediction on validation data and store them in oof (out of folds) sets\n        oof_train[valid_index] = np.argmax(model.predict(X_val), axis=1)\n        oof_test[:, ifold] = np.argmax(model.predict(X_test), axis=1)\n        \n        # Save accuracy\n        folds_acc.append(accuracy_score(np.argmax(y_val, axis=1), oof_train[valid_index]))\n             \n    print('Mean KFold ACC: {:.4f}'.format(round(np.mean(folds_acc)*100,2)))\n    print('Std KFold ACC: {:.4f}'.format(round(np.std(folds_acc)*100,2)))\n\n    return oof_train, oof_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train_data.values[:, 0]\noof_train, oof_test = kfold_validation(X_train, y_train, X_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Out of Folds (OOF) Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# final prediction (average)\nfinal_pred = np.mean(oof_test, axis=1)\nfinal_pred.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submit The Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# build data frame\ndf = pd.DataFrame({\"ImageId\": range(1,28001), \"Label\": np.round(final_pred).astype('int16')})\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22ef5ebf6cd0de971c955311b90c2b1bb62fe73d"},"cell_type":"code","source":"# submission\ndf.to_csv(\"pred.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}