{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-09-11T19:11:25.940137Z","iopub.execute_input":"2023-09-11T19:11:25.940591Z","iopub.status.idle":"2023-09-11T19:11:26.357945Z","shell.execute_reply.started":"2023-09-11T19:11:25.940555Z","shell.execute_reply":"2023-09-11T19:11:26.357002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndata_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:11:26.359533Z","iopub.execute_input":"2023-09-11T19:11:26.360399Z","iopub.status.idle":"2023-09-11T19:11:33.039684Z","shell.execute_reply.started":"2023-09-11T19:11:26.360367Z","shell.execute_reply":"2023-09-11T19:11:33.038540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from PIL import Image as im\nimport matplotlib.pyplot as plt\n\n# let's look at first example\nfirst_image = np.array(data_train.head(1))[0][1:]\nprint(first_image)\nfirst_image = np.reshape(first_image,(28,28)).astype('uint8')\nimage = im.fromarray(first_image)\nplt.imshow(image)\nprint(f'\\n LABEL: {np.array(data_train.head(1))[0][0]} \\n')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:11:33.041003Z","iopub.execute_input":"2023-09-11T19:11:33.041444Z","iopub.status.idle":"2023-09-11T19:11:33.366125Z","shell.execute_reply.started":"2023-09-11T19:11:33.041407Z","shell.execute_reply":"2023-09-11T19:11:33.364930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# divide data set\nfrom sklearn.model_selection import train_test_split\nX,y = data_train.drop(axis=1,labels='label'),data_train.label\nX_train,X_valid,y_train,y_valid = train_test_split(X,y,test_size=0.3)\ny.hist()\nplt.title('Label distribution')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:11:33.368793Z","iopub.execute_input":"2023-09-11T19:11:33.369248Z","iopub.status.idle":"2023-09-11T19:11:34.845488Z","shell.execute_reply.started":"2023-09-11T19:11:33.369217Z","shell.execute_reply":"2023-09-11T19:11:34.844257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Good, even label spread, time to create a model","metadata":{}},{"cell_type":"code","source":"# process y to categorical data\nfrom sklearn.preprocessing import LabelEncoder\nimport tensorflow as tf\n\ny_train_categorical = tf.keras.utils.to_categorical(y_train)\nprint(y_train_categorical)\ny_valid_categorical = tf.keras.utils.to_categorical(y_valid)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:11:34.846829Z","iopub.execute_input":"2023-09-11T19:11:34.847155Z","iopub.status.idle":"2023-09-11T19:11:44.354779Z","shell.execute_reply.started":"2023-09-11T19:11:34.847128Z","shell.execute_reply":"2023-09-11T19:11:44.353581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some imports for NN\nfrom tensorflow.keras.layers import Dense,Dropout\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.regularizers import l2\n","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:11:44.358925Z","iopub.execute_input":"2023-09-11T19:11:44.359591Z","iopub.status.idle":"2023-09-11T19:11:44.367461Z","shell.execute_reply.started":"2023-09-11T19:11:44.359557Z","shell.execute_reply":"2023-09-11T19:11:44.366349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lambda_ = 0.00\nepochs = 30\nlearning_rate = 0.001\n\nmodel = Sequential([\n    Dense(units=112,activation='relu',kernel_regularizer=l2(lambda_),name=\"L1\"),\n    Dense(units=112,activation='relu',kernel_regularizer=l2(lambda_),name=\"L2\"),\n    Dense(units=56,activation='relu',kernel_regularizer=l2(lambda_),name=\"L3\"),\n    Dense(units=10,activation='softmax',name=\"output\")\n])\nmodel.compile(loss='categorical_crossentropy',optimizer=Adam(learning_rate=learning_rate),metrics=['accuracy','Recall','Precision'])\nmodel.fit(X_train,y_train_categorical,epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:11:44.369281Z","iopub.execute_input":"2023-09-11T19:11:44.370009Z","iopub.status.idle":"2023-09-11T19:13:15.787018Z","shell.execute_reply.started":"2023-09-11T19:11:44.369969Z","shell.execute_reply":"2023-09-11T19:13:15.785945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions\ny_train_predict = model.predict(X_train)\ny_valid_predict = model.predict(X_valid)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:13:15.790446Z","iopub.execute_input":"2023-09-11T19:13:15.791303Z","iopub.status.idle":"2023-09-11T19:13:19.181044Z","shell.execute_reply.started":"2023-09-11T19:13:15.791269Z","shell.execute_reply":"2023-09-11T19:13:19.179859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.losses import CategoricalCrossentropy\ncce = CategoricalCrossentropy()\ntrain_loss = cce(y_train_predict,y_train_categorical).numpy()\nvalid_loss = cce(y_valid_predict,y_valid_categorical).numpy()\nprint(f'y-train error: {train_loss}')\nprint(f'y-valid error: {valid_loss}')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:13:19.184467Z","iopub.execute_input":"2023-09-11T19:13:19.184900Z","iopub.status.idle":"2023-09-11T19:13:19.226151Z","shell.execute_reply.started":"2023-09-11T19:13:19.184866Z","shell.execute_reply":"2023-09-11T19:13:19.225236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_predict = model.predict(data_test)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:13:19.227553Z","iopub.execute_input":"2023-09-11T19:13:19.228124Z","iopub.status.idle":"2023-09-11T19:13:21.515365Z","shell.execute_reply.started":"2023-09-11T19:13:19.228086Z","shell.execute_reply":"2023-09-11T19:13:21.514098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict = []\nfor values in y_test_predict:\n    predict.append(np.argmax(values))\n    \n# show first test image\nprint(predict[0])\nimage = np.array(data_test.head(1))\nimage = np.reshape(image,(28,28)).astype('uint8')\nimage = im.fromarray(image)\nplt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:13:21.516993Z","iopub.execute_input":"2023-09-11T19:13:21.517325Z","iopub.status.idle":"2023-09-11T19:13:21.952488Z","shell.execute_reply.started":"2023-09-11T19:13:21.517297Z","shell.execute_reply":"2023-09-11T19:13:21.951195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(data={'ImageId':data_test.index,'Label':predict})\nsubmission.ImageId = submission.ImageId.add(1)\nsubmission.set_index('ImageId',inplace=True)\nprint(submission.head(5))\nsubmission.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:13:21.954587Z","iopub.execute_input":"2023-09-11T19:13:21.955504Z","iopub.status.idle":"2023-09-11T19:13:22.201361Z","shell.execute_reply.started":"2023-09-11T19:13:21.955470Z","shell.execute_reply":"2023-09-11T19:13:22.199782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CNN Part","metadata":{}},{"cell_type":"code","source":"X_train = np.array(X_train).reshape(-1,28,28,1)\nX_valid = np.array(X_valid).reshape(-1,28,28,1)\nX_test = np.array(data_test).reshape(-1,28,28,1)\nprint(X_train[0].shape)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T19:52:25.798215Z","iopub.execute_input":"2023-09-11T19:52:25.798867Z","iopub.status.idle":"2023-09-11T19:52:26.015680Z","shell.execute_reply.started":"2023-09-11T19:52:25.798819Z","shell.execute_reply":"2023-09-11T19:52:26.014454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's find out how convolutional layers works\nfrom tensorflow.keras.layers import Conv2D,MaxPooling2D,Flatten\n\nlambda_ = 0.00\nepochs = 30\nlearning_rate = 0.001\n\nmodel_cnn = Sequential([\n    Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n    MaxPooling2D((2, 2)),\n    Conv2D(32, (3, 3), activation='relu'),\n    MaxPooling2D((2, 2)),\n    Flatten(),\n    Dense(64, activation='relu'),\n    Dense(units=10,activation='softmax',name=\"output\")\n])\n\nmodel_cnn.compile(loss='categorical_crossentropy',optimizer=Adam(learning_rate=learning_rate),metrics=['accuracy','Recall','Precision'])\nmodel_cnn.fit(X_train,y_train_categorical,epochs=epochs)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T20:02:36.773266Z","iopub.execute_input":"2023-09-11T20:02:36.773739Z","iopub.status.idle":"2023-09-11T20:11:00.430285Z","shell.execute_reply.started":"2023-09-11T20:02:36.773703Z","shell.execute_reply":"2023-09-11T20:11:00.428860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions\ny_train_predict = model_cnn.predict(X_train)\ny_valid_predict = model_cnn.predict(X_valid)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T20:13:55.782222Z","iopub.execute_input":"2023-09-11T20:13:55.782705Z","iopub.status.idle":"2023-09-11T20:14:03.841747Z","shell.execute_reply.started":"2023-09-11T20:13:55.782672Z","shell.execute_reply":"2023-09-11T20:14:03.840585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cce = CategoricalCrossentropy()\ntrain_loss = cce(y_train_predict,y_train_categorical).numpy()\nvalid_loss = cce(y_valid_predict,y_valid_categorical).numpy()\nprint(f'y-train error: {train_loss}')\nprint(f'y-valid error: {valid_loss}')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T20:14:06.082038Z","iopub.execute_input":"2023-09-11T20:14:06.082463Z","iopub.status.idle":"2023-09-11T20:14:06.101694Z","shell.execute_reply.started":"2023-09-11T20:14:06.082429Z","shell.execute_reply":"2023-09-11T20:14:06.100715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_predict = model_cnn.predict(X_test)\npredict = []\nfor values in y_test_predict:\n    predict.append(np.argmax(values))\n    \n# show first test image\nprint(predict[0])\nimage = np.array(data_test.head(1))\nimage = np.reshape(image,(28,28)).astype('uint8')\nimage = im.fromarray(image)\nplt.imshow(image)","metadata":{"execution":{"iopub.status.busy":"2023-09-11T20:29:03.946119Z","iopub.execute_input":"2023-09-11T20:29:03.946552Z","iopub.status.idle":"2023-09-11T20:29:09.672368Z","shell.execute_reply.started":"2023-09-11T20:29:03.946504Z","shell.execute_reply":"2023-09-11T20:29:09.670930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(data={'ImageId':data_test.index,'Label':predict})\nsubmission.ImageId = submission.ImageId.add(1)\nsubmission.set_index('ImageId',inplace=True)\nprint(submission.head(5))\nsubmission.to_csv('submissionCNN.csv')","metadata":{"execution":{"iopub.status.busy":"2023-09-11T20:29:15.338321Z","iopub.execute_input":"2023-09-11T20:29:15.338766Z","iopub.status.idle":"2023-09-11T20:29:15.558336Z","shell.execute_reply.started":"2023-09-11T20:29:15.338732Z","shell.execute_reply":"2023-09-11T20:29:15.556978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Things I learned from this notebook:\n- how to make categorical NN\n- new keras layers, Dropout,Convolutional, MaxPooling\n- difference between normal cross_entropy vs sparse cross_entropy","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}