{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom IPython.display import display\nimport gc\nfrom sklearn import model_selection, preprocessing, metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:38:35.553574Z","iopub.execute_input":"2021-08-29T11:38:35.55394Z","iopub.status.idle":"2021-08-29T11:38:36.42057Z","shell.execute_reply.started":"2021-08-29T11:38:35.553859Z","shell.execute_reply":"2021-08-29T11:38:36.419754Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:38:36.422061Z","iopub.execute_input":"2021-08-29T11:38:36.422432Z","iopub.status.idle":"2021-08-29T11:38:40.242953Z","shell.execute_reply.started":"2021-08-29T11:38:36.422375Z","shell.execute_reply":"2021-08-29T11:38:40.242147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"`label` column contains target variable while `pixel*` ones the features.","metadata":{}},{"cell_type":"code","source":"# looking for missing values\ndf.isnull().any(axis=None)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:38:40.244645Z","iopub.execute_input":"2021-08-29T11:38:40.244977Z","iopub.status.idle":"2021-08-29T11:38:40.262485Z","shell.execute_reply.started":"2021-08-29T11:38:40.244941Z","shell.execute_reply":"2021-08-29T11:38:40.261534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are not any missing values in the training dataset.","metadata":{}},{"cell_type":"code","source":"# describe features and labels\npd.DataFrame(\n    [\n        [df.drop(columns=\"label\").min().min(), df.drop(columns=\"label\").max().max(), df.iloc[:,1:].values.dtype],\n        [df.label.min(), df.label.max(), df.label.values.dtype]\n    ],\n    columns=[\"min\", \"max\", \"dtype\"], index=[\"pixels\", \"labels\"]\n)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:38:40.264196Z","iopub.execute_input":"2021-08-29T11:38:40.264556Z","iopub.status.idle":"2021-08-29T11:38:40.506271Z","shell.execute_reply.started":"2021-08-29T11:38:40.264518Z","shell.execute_reply":"2021-08-29T11:38:40.505346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Pixels contain color values in tha range $[0,255] \\cap \\mathbb{N}$ and labels are in $[0,9] \\cap \\mathbb{N}$ (they are digits).","metadata":{}},{"cell_type":"code","source":"fig,ax = plt.subplots()\nsns.histplot(data=df.label, stat=\"probability\", ax=ax)\nax.set_title(\"Digits distribution\")\nax.set_xticks([i for i in range(10)])\nax.set_xlabel(\"digits\")\n_ = ax.set_ylabel(\"relative fraction\")","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:38:40.507695Z","iopub.execute_input":"2021-08-29T11:38:40.508041Z","iopub.status.idle":"2021-08-29T11:38:40.781163Z","shell.execute_reply.started":"2021-08-29T11:38:40.507997Z","shell.execute_reply":"2021-08-29T11:38:40.780415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training data are splitted for validation: $70\\%$ in training values, $30\\%$ in validation.\nSplitting is done maintaining the ratios of digits in the sets.","metadata":{}},{"cell_type":"code","source":"train, valid = model_selection.train_test_split(df, train_size=0.7, stratify=df.label, random_state=1)\ntrain.shape","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-29T11:38:40.782442Z","iopub.execute_input":"2021-08-29T11:38:40.782764Z","iopub.status.idle":"2021-08-29T11:38:41.072508Z","shell.execute_reply.started":"2021-08-29T11:38:40.782727Z","shell.execute_reply":"2021-08-29T11:38:41.071538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\n_ = gc.collect","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Convolutional Neural Network (CNN)","metadata":{}},{"cell_type":"code","source":"def get_x(df):\n    df = df.drop(columns=\"label\", errors=\"ignore\")\n    return df.values\n\ndef get_y(df):\n    y = pd.get_dummies(df.label)\n    return y.values","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-29T11:38:41.073994Z","iopub.execute_input":"2021-08-29T11:38:41.074349Z","iopub.status.idle":"2021-08-29T11:38:41.079727Z","shell.execute_reply.started":"2021-08-29T11:38:41.074311Z","shell.execute_reply":"2021-08-29T11:38:41.078698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Model training is done on reshaping events in a $28 \\times 28 \\times 1$ image.\nThis dimensions are correct as it can be verified drawing a picture.","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots()\nax.set_yticks([])\nax.set_xticks([])\n_ = ax.imshow(get_x(train)[1].reshape((28,28)), cmap=\"Greys\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-29T11:38:41.082943Z","iopub.execute_input":"2021-08-29T11:38:41.08356Z","iopub.status.idle":"2021-08-29T11:38:41.248254Z","shell.execute_reply.started":"2021-08-29T11:38:41.08352Z","shell.execute_reply":"2021-08-29T11:38:41.244933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now CNN model can be trained.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras import models, layers\n\nmodel = models.Sequential(\n    [\n        layers.Reshape(input_shape=(784,), target_shape=(28,28,1), name=\"reshape_in\"),\n        \n        layers.Conv2D(filters=32, kernel_size=(5,5), activation=\"relu\", padding=\"same\", name=\"conv32a\"),\n        layers.Conv2D(filters=32, kernel_size=(5,5), activation=\"relu\", padding=\"same\", name=\"conv32b\"),\n        layers.MaxPooling2D(pool_size=(2,2), name=\"pool32\"),\n        layers.Dropout(rate=0.25, name=\"drop32\"),\n        \n        layers.Conv2D(filters=64, kernel_size=(3,3), activation=\"relu\", padding=\"same\", name=\"conv64a\"),\n        layers.Conv2D(filters=64, kernel_size=(3,3), activation=\"relu\", padding=\"same\", name=\"conv64b\"),\n        layers.MaxPooling2D(pool_size=(2,2), strides=(2,2), name=\"pool64\"),\n        layers.Dropout(rate=0.25, name=\"drop64\"),\n        \n        layers.Flatten(name=\"flat\"),\n        \n        layers.Dense(units=256, activation=\"relu\", name=\"dense256\"),\n        layers.Dense(units=10, activation=\"softmax\", name=\"out\")\n    ],\n    name=\"CNN\"\n)\n\nmodel.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\nmodel.summary()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-29T11:38:41.252751Z","iopub.execute_input":"2021-08-29T11:38:41.253138Z","iopub.status.idle":"2021-08-29T11:38:47.067068Z","shell.execute_reply.started":"2021-08-29T11:38:41.253079Z","shell.execute_reply":"2021-08-29T11:38:47.066133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_history = model.fit(\n    x = get_x(train), y = get_y(train),\n    validation_data = (get_x(valid), get_y(valid)),\n    batch_size = 1000, epochs = 20,\n    use_multiprocessing = True\n)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-29T11:38:47.068442Z","iopub.execute_input":"2021-08-29T11:38:47.06879Z","iopub.status.idle":"2021-08-29T11:39:16.772273Z","shell.execute_reply.started":"2021-08-29T11:38:47.068752Z","shell.execute_reply":"2021-08-29T11:39:16.771437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(6.4*2,4.8))\n\nfor i,metric in enumerate([\"loss\", \"accuracy\"]):\n    for j,prefix in enumerate([\"\", \"val_\"]):\n        ax[i].plot([k+1 for k in range(20)], training_history.history[prefix + metric], label=prefix+metric)\n    ax[i].set_xlabel(\"epoch\")\n    ax[i].set_ylabel(metric)\n    ax[i].set_xticks([i*2 for i in range(11)])\n    ax[i].legend()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-29T11:39:16.775079Z","iopub.execute_input":"2021-08-29T11:39:16.775351Z","iopub.status.idle":"2021-08-29T11:39:17.118088Z","shell.execute_reply.started":"2021-08-29T11:39:16.775322Z","shell.execute_reply":"2021-08-29T11:39:17.117269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(get_x(valid))\nroc_curves = [metrics.roc_curve(y_score = y_pred[:,i], y_true = get_y(valid)[:,i]) for i in range(10)]\n\nfig, ax = plt.subplots()\nfor i in range(10):\n    ax.plot(roc_curves[i][0], roc_curves[i][1], label=\"{} digit\".format(i))\nax.set_title(\"ROC curves\")\nax.set_xlabel(\"false positive rate\")\nax.set_ylabel(\"true positive rate\")\n_ = ax.legend()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-29T11:39:17.119359Z","iopub.execute_input":"2021-08-29T11:39:17.119959Z","iopub.status.idle":"2021-08-29T11:39:18.455411Z","shell.execute_reply.started":"2021-08-29T11:39:17.119914Z","shell.execute_reply":"2021-08-29T11:39:18.454546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train; del training_history; del y_pred; del roc_curves\n_ = gc.collect()","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"test = pd.read_csv(\"../input/digit-recognizer/test.csv\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:39:18.456839Z","iopub.execute_input":"2021-08-29T11:39:18.457197Z","iopub.status.idle":"2021-08-29T11:39:21.09405Z","shell.execute_reply.started":"2021-08-29T11:39:18.457156Z","shell.execute_reply":"2021-08-29T11:39:21.093241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# looking for missing values\ntest.isnull().any(axis=None)","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:39:21.095377Z","iopub.execute_input":"2021-08-29T11:39:21.095724Z","iopub.status.idle":"2021-08-29T11:39:21.112206Z","shell.execute_reply.started":"2021-08-29T11:39:21.095687Z","shell.execute_reply":"2021-08-29T11:39:21.111496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\", index_col=0)\nsubmission.head()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-08-29T11:39:21.113447Z","iopub.execute_input":"2021-08-29T11:39:21.113815Z","iopub.status.idle":"2021-08-29T11:39:21.137067Z","shell.execute_reply.started":"2021-08-29T11:39:21.113777Z","shell.execute_reply":"2021-08-29T11:39:21.136149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model.predict(get_x(test))\nsubmission.Label = y_pred.argmax(axis=-1)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:39:21.138295Z","iopub.execute_input":"2021-08-29T11:39:21.138677Z","iopub.status.idle":"2021-08-29T11:39:22.801373Z","shell.execute_reply.started":"2021-08-29T11:39:21.138638Z","shell.execute_reply":"2021-08-29T11:39:22.79994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig,ax = plt.subplots()\nsns.histplot(data=submission, stat=\"probability\", ax=ax)\nax.set_title(\"Digits distribution in submission\")\nax.set_xlabel(\"digits\")\nax.set_ylabel(\"relative fraction\")\nax.legend()\n_ = ax.set_xticks([i for i in range(10)])","metadata":{"execution":{"iopub.status.busy":"2021-08-29T11:39:22.803564Z","iopub.execute_input":"2021-08-29T11:39:22.804205Z","iopub.status.idle":"2021-08-29T11:39:23.147825Z","shell.execute_reply.started":"2021-08-29T11:39:22.80413Z","shell.execute_reply":"2021-08-29T11:39:23.146791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"sample_submission.csv\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-08-29T11:39:23.149279Z","iopub.execute_input":"2021-08-29T11:39:23.149665Z","iopub.status.idle":"2021-08-29T11:39:23.208918Z","shell.execute_reply.started":"2021-08-29T11:39:23.149626Z","shell.execute_reply":"2021-08-29T11:39:23.208083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test; del submission; del y_pred; del model\n_ = gc.collect()","metadata":{"_kg_hide-input":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}