{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-27T17:12:50.780129Z","iopub.execute_input":"2023-01-27T17:12:50.780767Z","iopub.status.idle":"2023-01-27T17:12:50.826013Z","shell.execute_reply.started":"2023-01-27T17:12:50.780507Z","shell.execute_reply":"2023-01-27T17:12:50.824705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the CSV\n\ndf_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\nnum_train, num_test, num_features = df_train.shape[0], df_test.shape[0], df_test.shape[1]\n\nprint(df_train.shape, df_test.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T17:32:36.154141Z","iopub.execute_input":"2023-01-27T17:32:36.154511Z","iopub.status.idle":"2023-01-27T17:32:38.78334Z","shell.execute_reply.started":"2023-01-27T17:32:36.15448Z","shell.execute_reply":"2023-01-27T17:32:38.782082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nrandom_index = 45\n\nim_size = int(np.sqrt(num_features))\nplt.imshow(np.array(df_train.iloc[random_index,1:]).reshape(im_size,im_size),cmap='gray')","metadata":{"execution":{"iopub.status.busy":"2023-01-27T18:56:43.734172Z","iopub.execute_input":"2023-01-27T18:56:43.734741Z","iopub.status.idle":"2023-01-27T18:56:43.867241Z","shell.execute_reply.started":"2023-01-27T18:56:43.73463Z","shell.execute_reply":"2023-01-27T18:56:43.866502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Basic data attributes\n\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T17:17:56.504927Z","iopub.execute_input":"2023-01-27T17:17:56.505297Z","iopub.status.idle":"2023-01-27T17:17:56.54388Z","shell.execute_reply.started":"2023-01-27T17:17:56.505261Z","shell.execute_reply":"2023-01-27T17:17:56.542951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# What are the labels?\nprint(\"Unique label values\",df_train['label'].unique())\nprint(\"Distribution: \", df_train['label'].hist(bins=10))\ny = df_train['label'].values\n\n\n'''\nWe have some data belonging to all the classes.\n\n'''","metadata":{"execution":{"iopub.status.busy":"2023-01-27T17:29:02.337791Z","iopub.execute_input":"2023-01-27T17:29:02.338884Z","iopub.status.idle":"2023-01-27T17:29:02.47443Z","shell.execute_reply.started":"2023-01-27T17:29:02.338848Z","shell.execute_reply":"2023-01-27T17:29:02.473672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Missing features / feature coverage\nprint('No missing features? ', len(df_train.count(axis=0).unique())==1)\n\n# Feature coverage\ndf_train_feature_coverage = pd.DataFrame((df_train.count(axis=0) / num_train)*100)\ndf_train_feature_coverage","metadata":{"execution":{"iopub.status.busy":"2023-01-27T18:46:14.073546Z","iopub.execute_input":"2023-01-27T18:46:14.074275Z","iopub.status.idle":"2023-01-27T18:46:15.829883Z","shell.execute_reply.started":"2023-01-27T18:46:14.07424Z","shell.execute_reply":"2023-01-27T18:46:15.82861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T18:46:54.995323Z","iopub.execute_input":"2023-01-27T18:46:54.995958Z","iopub.status.idle":"2023-01-27T18:46:56.918468Z","shell.execute_reply.started":"2023-01-27T18:46:54.995915Z","shell.execute_reply":"2023-01-27T18:46:56.916952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* Range of pixels 0 to 255","metadata":{}},{"cell_type":"markdown","source":"# Get training data and labels","metadata":{}},{"cell_type":"code","source":"x,y = df_train.drop(['label'],axis=1).values, df_train['label'].values","metadata":{"execution":{"iopub.status.busy":"2023-01-27T17:43:27.508515Z","iopub.execute_input":"2023-01-27T17:43:27.508885Z","iopub.status.idle":"2023-01-27T17:43:27.559579Z","shell.execute_reply.started":"2023-01-27T17:43:27.508855Z","shell.execute_reply":"2023-01-27T17:43:27.558244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_val, y_train, y_val = train_test_split(x,y,test_size=0.1,random_state=12)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:12:52.399631Z","iopub.execute_input":"2023-01-27T19:12:52.400048Z","iopub.status.idle":"2023-01-27T19:12:52.626618Z","shell.execute_reply.started":"2023-01-27T19:12:52.400015Z","shell.execute_reply":"2023-01-27T19:12:52.625187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train.shape, X_val.shape)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:12:59.12481Z","iopub.execute_input":"2023-01-27T19:12:59.125169Z","iopub.status.idle":"2023-01-27T19:12:59.132315Z","shell.execute_reply.started":"2023-01-27T19:12:59.125142Z","shell.execute_reply":"2023-01-27T19:12:59.131008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train some models","metadata":{}},{"cell_type":"markdown","source":"## KNN","metadata":{}},{"cell_type":"markdown","source":" Use the \"k\" nearest neighbors to predict class value","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier(n_neighbors=3)\nknn.fit(X_train, y_train)\nprint(\"training accuracy : {:.2f}\".format(knn.score(X_train, y_train)))\nprint(\"validation accuracy : {:.2f}\".format(knn.score(X_val, y_val)))","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:25:12.720237Z","iopub.execute_input":"2023-01-27T19:25:12.721423Z","iopub.status.idle":"2023-01-27T19:25:53.654801Z","shell.execute_reply.started":"2023-01-27T19:25:12.721366Z","shell.execute_reply":"2023-01-27T19:25:53.653746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Decision Tree\n\n- Non-linear model\n- Series of binary questions\n- Criteria : Pick feature acc to quality of split, Gini index (1 - sum(p_c^2))\n    - likelihood of new, random data being misclassified\n    \n- Max depth = 2 for illustration!\n","metadata":{}},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier, export_graphviz\ndtree = DecisionTreeClassifier(max_depth=2)\ndtree.fit(x_train , y_train)\nprint(\"training accuracy : {:.2f}\".format(dtree.score(x_train , y_train)))\nprint(\"validation accuracy : {:.2f}\".format(dtree.score(x_val , y_val)))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:56:15.821768Z","iopub.execute_input":"2023-01-27T19:56:15.8233Z","iopub.status.idle":"2023-01-27T19:56:17.410347Z","shell.execute_reply.started":"2023-01-27T19:56:15.823242Z","shell.execute_reply":"2023-01-27T19:56:17.409093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tree_dot = export_graphviz(dtree, out_file=None)\nprint(tree_dot)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:49:12.461577Z","iopub.execute_input":"2023-01-27T19:49:12.461983Z","iopub.status.idle":"2023-01-27T19:49:12.468802Z","shell.execute_reply.started":"2023-01-27T19:49:12.46195Z","shell.execute_reply":"2023-01-27T19:49:12.46802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import graphviz\ngraphviz.Source(tree_dot)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:49:54.433019Z","iopub.execute_input":"2023-01-27T19:49:54.433653Z","iopub.status.idle":"2023-01-27T19:49:55.466558Z","shell.execute_reply.started":"2023-01-27T19:49:54.433607Z","shell.execute_reply":"2023-01-27T19:49:55.46547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\ncmdtree = confusion_matrix(y_val, dtree.predict(x_val))\ncmdtree ,dtree.score(x_val , y_val)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:55:00.278133Z","iopub.execute_input":"2023-01-27T19:55:00.278548Z","iopub.status.idle":"2023-01-27T19:55:00.301672Z","shell.execute_reply.started":"2023-01-27T19:55:00.278511Z","shell.execute_reply":"2023-01-27T19:55:00.300696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtree.feature_importances_","metadata":{"execution":{"iopub.status.busy":"2023-01-27T19:58:24.931947Z","iopub.execute_input":"2023-01-27T19:58:24.93263Z","iopub.status.idle":"2023-01-27T19:58:24.946792Z","shell.execute_reply.started":"2023-01-27T19:58:24.932594Z","shell.execute_reply":"2023-01-27T19:58:24.945266Z"},"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Random Forest","metadata":{}},{"cell_type":"markdown","source":"- Ensemble models\n- use different sub-samples for different trees\n- average, control over-fitting","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nforest = RandomForestClassifier(n_estimators=3)\nforest.fit(X_train , y_train)\nprint(\"training accuracy : {:.2f}\".format(forest.score(x_train , y_train)))\nprint(\"validation accuracy : {:.2f}\".format(forest.score(x_val , y_val)))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:03:46.396088Z","iopub.execute_input":"2023-01-27T20:03:46.39715Z","iopub.status.idle":"2023-01-27T20:03:47.275341Z","shell.execute_reply.started":"2023-01-27T20:03:46.397101Z","shell.execute_reply":"2023-01-27T20:03:47.274436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Small neural network","metadata":{}},{"cell_type":"markdown","source":"- We generally try and scale data between 0-1 to reduce training time.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:09:06.026875Z","iopub.execute_input":"2023-01-27T20:09:06.027248Z","iopub.status.idle":"2023-01-27T20:09:06.032775Z","shell.execute_reply.started":"2023-01-27T20:09:06.027213Z","shell.execute_reply":"2023-01-27T20:09:06.031475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_nn, X_val_nn = X_train / 255.0 , X_val / 255.0\nX_train_nn.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:15:55.749791Z","iopub.execute_input":"2023-01-27T20:15:55.750192Z","iopub.status.idle":"2023-01-27T20:15:55.810601Z","shell.execute_reply.started":"2023-01-27T20:15:55.750151Z","shell.execute_reply":"2023-01-27T20:15:55.809569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = keras.models.Sequential()\nmodel.add(layers.Dense(32, input_dim=num_features, activation='relu'))\nmodel.add(layers.Dense(16, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:16:30.345243Z","iopub.execute_input":"2023-01-27T20:16:30.345593Z","iopub.status.idle":"2023-01-27T20:16:30.499175Z","shell.execute_reply.started":"2023-01-27T20:16:30.345563Z","shell.execute_reply":"2023-01-27T20:16:30.497969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=\"Adam\",loss=\"sparse_categorical_crossentropy\",metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:23:03.09054Z","iopub.execute_input":"2023-01-27T20:23:03.091222Z","iopub.status.idle":"2023-01-27T20:23:03.101515Z","shell.execute_reply.started":"2023-01-27T20:23:03.091182Z","shell.execute_reply":"2023-01-27T20:23:03.100108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(X_train_nn, y_train, epochs=3, batch_size=100)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:23:03.940438Z","iopub.execute_input":"2023-01-27T20:23:03.941009Z","iopub.status.idle":"2023-01-27T20:23:07.562016Z","shell.execute_reply.started":"2023-01-27T20:23:03.940982Z","shell.execute_reply":"2023-01-27T20:23:07.560906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_train = model.evaluate(X_train_nn, y_train)\nscores_val = model.evaluate(X_val_nn, y_val)","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:25:01.759608Z","iopub.execute_input":"2023-01-27T20:25:01.760969Z","iopub.status.idle":"2023-01-27T20:25:04.18742Z","shell.execute_reply.started":"2023-01-27T20:25:01.760915Z","shell.execute_reply":"2023-01-27T20:25:04.186601Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"training accuracy : {:.2f}\".format(scores_train[1]*100))\nprint(\"validation accuracy : {:.2f}\".format(scores_val[1]*100))\n","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:25:57.854034Z","iopub.execute_input":"2023-01-27T20:25:57.854404Z","iopub.status.idle":"2023-01-27T20:25:57.859414Z","shell.execute_reply.started":"2023-01-27T20:25:57.854372Z","shell.execute_reply":"2023-01-27T20:25:57.858763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submit","metadata":{}},{"cell_type":"code","source":"X_test_nn = df_test.values / 255.0\ny_test_probs = model.predict(X_test_nn)\ny_test_probs","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:36:01.93487Z","iopub.execute_input":"2023-01-27T20:36:01.935503Z","iopub.status.idle":"2023-01-27T20:36:03.31892Z","shell.execute_reply.started":"2023-01-27T20:36:01.935467Z","shell.execute_reply":"2023-01-27T20:36:03.317418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = np.argmax(y_test_probs, axis=-1)\ny_test","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:36:25.518501Z","iopub.execute_input":"2023-01-27T20:36:25.518862Z","iopub.status.idle":"2023-01-27T20:36:25.526983Z","shell.execute_reply.started":"2023-01-27T20:36:25.518836Z","shell.execute_reply":"2023-01-27T20:36:25.52566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Conver to ImageId,Label format\ndf_submit = pd.DataFrame({'ImageId':range(1,y_test.size+1),'Label': y_test})\ndf_submit.to_csv('prediction.csv',index=False)   ","metadata":{"execution":{"iopub.status.busy":"2023-01-27T20:38:19.98339Z","iopub.execute_input":"2023-01-27T20:38:19.9838Z","iopub.status.idle":"2023-01-27T20:38:20.020566Z","shell.execute_reply.started":"2023-01-27T20:38:19.983768Z","shell.execute_reply":"2023-01-27T20:38:20.019425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}