{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-05T09:41:56.247357Z","iopub.execute_input":"2023-05-05T09:41:56.247926Z","iopub.status.idle":"2023-05-05T09:41:56.257766Z","shell.execute_reply.started":"2023-05-05T09:41:56.247880Z","shell.execute_reply":"2023-05-05T09:41:56.256594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom PIL import Image\nimport seaborn as sns\nimport numpy as np \n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing\n\n\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:41:56.664575Z","iopub.execute_input":"2023-05-05T09:41:56.666033Z","iopub.status.idle":"2023-05-05T09:41:56.671833Z","shell.execute_reply.started":"2023-05-05T09:41:56.665988Z","shell.execute_reply":"2023-05-05T09:41:56.670757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(r'/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv(r'/kaggle/input/digit-recognizer/test.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:41:57.171319Z","iopub.execute_input":"2023-05-05T09:41:57.171770Z","iopub.status.idle":"2023-05-05T09:42:03.741540Z","shell.execute_reply.started":"2023-05-05T09:41:57.171731Z","shell.execute_reply":"2023-05-05T09:42:03.739978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop(['label'], axis = 1) # drop label col\ny = train['label'] # set target col","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:03.743264Z","iopub.execute_input":"2023-05-05T09:42:03.743655Z","iopub.status.idle":"2023-05-05T09:42:03.928183Z","shell.execute_reply.started":"2023-05-05T09:42:03.743619Z","shell.execute_reply":"2023-05-05T09:42:03.926751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=y)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:03.929896Z","iopub.execute_input":"2023-05-05T09:42:03.930437Z","iopub.status.idle":"2023-05-05T09:42:04.174417Z","shell.execute_reply.started":"2023-05-05T09:42:03.930389Z","shell.execute_reply":"2023-05-05T09:42:04.173081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Distribution of numbers from 0 to 9 looks well.","metadata":{}},{"cell_type":"markdown","source":"# Tackle with missing data","metadata":{}},{"cell_type":"code","source":"X.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:04.177355Z","iopub.execute_input":"2023-05-05T09:42:04.177837Z","iopub.status.idle":"2023-05-05T09:42:04.204566Z","shell.execute_reply.started":"2023-05-05T09:42:04.177791Z","shell.execute_reply":"2023-05-05T09:42:04.203706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isnull().any().describe()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:04.205942Z","iopub.execute_input":"2023-05-05T09:42:04.206894Z","iopub.status.idle":"2023-05-05T09:42:04.226394Z","shell.execute_reply.started":"2023-05-05T09:42:04.206855Z","shell.execute_reply":"2023-05-05T09:42:04.225197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It look like somone collected carefuly data, there is no missing values in both datasets. So go ahead!","metadata":{}},{"cell_type":"markdown","source":"# Normalization\n\nResult of normalization will be acceleration of computing for CNN neural network","metadata":{}},{"cell_type":"code","source":"X /= 255.0\ntest /= 255.0\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:04.802363Z","iopub.execute_input":"2023-05-05T09:42:04.802847Z","iopub.status.idle":"2023-05-05T09:42:04.960520Z","shell.execute_reply.started":"2023-05-05T09:42:04.802809Z","shell.execute_reply":"2023-05-05T09:42:04.959381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Reshape image in 3 dim","metadata":{}},{"cell_type":"code","source":"X = X.values.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:06.451772Z","iopub.execute_input":"2023-05-05T09:42:06.453113Z","iopub.status.idle":"2023-05-05T09:42:06.458665Z","shell.execute_reply.started":"2023-05-05T09:42:06.453007Z","shell.execute_reply":"2023-05-05T09:42:06.457475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = test.values.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:06.964756Z","iopub.execute_input":"2023-05-05T09:42:06.965185Z","iopub.status.idle":"2023-05-05T09:42:06.971621Z","shell.execute_reply.started":"2023-05-05T09:42:06.965153Z","shell.execute_reply":"2023-05-05T09:42:06.969977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# To categorical label ","metadata":{}},{"cell_type":"code","source":"y = tf.keras.utils.to_categorical(y, num_classes=10)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:07.925093Z","iopub.execute_input":"2023-05-05T09:42:07.925596Z","iopub.status.idle":"2023-05-05T09:42:07.931424Z","shell.execute_reply.started":"2023-05-05T09:42:07.925557Z","shell.execute_reply":"2023-05-05T09:42:07.930263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y[0]","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:08.749254Z","iopub.execute_input":"2023-05-05T09:42:08.749687Z","iopub.status.idle":"2023-05-05T09:42:08.757623Z","shell.execute_reply.started":"2023-05-05T09:42:08.749652Z","shell.execute_reply":"2023-05-05T09:42:08.756306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split training and validation set ","metadata":{}},{"cell_type":"code","source":"random_seed = 100\nX_train, X_val, Y_train, y_val = train_test_split(X, y, test_size=0.33, random_state=random_seed)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:42:24.799704Z","iopub.execute_input":"2023-05-05T09:42:24.800514Z","iopub.status.idle":"2023-05-05T09:42:25.414487Z","shell.execute_reply.started":"2023-05-05T09:42:24.800470Z","shell.execute_reply":"2023-05-05T09:42:25.413110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visulation sample from dataset","metadata":{}},{"cell_type":"code","source":"len(X_train)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:47:54.709319Z","iopub.execute_input":"2023-05-05T09:47:54.709750Z","iopub.status.idle":"2023-05-05T09:47:54.717566Z","shell.execute_reply.started":"2023-05-05T09:47:54.709712Z","shell.execute_reply":"2023-05-05T09:47:54.716374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['label']","metadata":{"execution":{"iopub.status.busy":"2023-05-05T09:58:23.966663Z","iopub.execute_input":"2023-05-05T09:58:23.967488Z","iopub.status.idle":"2023-05-05T09:58:23.978479Z","shell.execute_reply.started":"2023-05-05T09:58:23.967434Z","shell.execute_reply":"2023-05-05T09:58:23.977051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n in range(1,10):\n    \n    plt.subplot(3, 3, n)\n    plt.imshow(X_train[n], cmap='Greys')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:09:12.464678Z","iopub.execute_input":"2023-05-05T10:09:12.465151Z","iopub.status.idle":"2023-05-05T10:09:13.144899Z","shell.execute_reply.started":"2023-05-05T10:09:12.465095Z","shell.execute_reply":"2023-05-05T10:09:13.143678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CNN - LeNet-5","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:07:48.958297Z","iopub.execute_input":"2023-05-05T10:07:48.958745Z","iopub.status.idle":"2023-05-05T10:07:48.965438Z","shell.execute_reply.started":"2023-05-05T10:07:48.958700Z","shell.execute_reply":"2023-05-05T10:07:48.964446Z"}}},{"cell_type":"markdown","source":"The LeNet-5 architecture consists of two sets of convolutional and average pooling layers, followed by a flattening convolutional layer, then two fully-connected layers and finally a softmax classifier.","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.layers import Conv2D, MaxPool2D, AveragePooling2D, Dense, Flatten\nfrom tensorflow.keras.activations import tanh, softmax\nfrom tensorflow.keras.metrics import categorical_crossentropy\n\n\nmodel = tf.keras.Sequential()\n# Convolution layer  \nmodel.add(Conv2D(6, kernel_size=(5, 5), strides=(1,1), activation=tanh, input_shape=(28, 28, 1), padding='same'))\n# Pooling layer \nmodel.add(AveragePooling2D(pool_size=(2,2), strides=(1,1), padding='valid'))\n# Convolution layer \nmodel.add(Conv2D(16, kernel_size=(5, 5), strides=(1,1), activation=tanh, padding='valid'))\n# Pooling layer \nmodel.add(AveragePooling2D(pool_size=(2,2), strides=(2,2), padding='valid'))\n\n# Full connected convolution layer\nmodel.add(Conv2D(120, kernel_size=(5, 5), strides=(1,1), activation=tanh, padding='valid'))\n# Flatten the CNN outputs result of this it is fully connected layers\nmodel.add(Flatten())\n\n# Full connected layer \nmodel.add(Dense(84, activation=tanh))\n\n# Output layer with softmax func activation\nmodel.add(Dense(10, activation=softmax))\n","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:37:38.572338Z","iopub.execute_input":"2023-05-05T10:37:38.573554Z","iopub.status.idle":"2023-05-05T10:37:38.667041Z","shell.execute_reply.started":"2023-05-05T10:37:38.573495Z","shell.execute_reply":"2023-05-05T10:37:38.665900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(loss = categorical_crossentropy, optimizer='SGD', metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:38:19.756626Z","iopub.execute_input":"2023-05-05T10:38:19.757175Z","iopub.status.idle":"2023-05-05T10:38:19.787699Z","shell.execute_reply.started":"2023-05-05T10:38:19.757116Z","shell.execute_reply":"2023-05-05T10:38:19.786630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hist = model.fit(X_train, Y_train, epochs=10, batch_size=128, validation_data=(X_val, y_val), verbose=1)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:39:29.286074Z","iopub.execute_input":"2023-05-05T10:39:29.286533Z","iopub.status.idle":"2023-05-05T10:43:17.172598Z","shell.execute_reply.started":"2023-05-05T10:39:29.286496Z","shell.execute_reply":"2023-05-05T10:43:17.171581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = model.predict(test)\n\nresults = np.argmax(results, axis=1)\n\nresults = pd.Series(results, name='Label')","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:46:35.285602Z","iopub.execute_input":"2023-05-05T10:46:35.286026Z","iopub.status.idle":"2023-05-05T10:46:46.020584Z","shell.execute_reply.started":"2023-05-05T10:46:35.285990Z","shell.execute_reply":"2023-05-05T10:46:46.019385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.Series(range(1, 28001), name='ImageId'), results], axis=1)\n\nsubmission.to_csv('cnn_mnist.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-05-05T10:48:12.963981Z","iopub.execute_input":"2023-05-05T10:48:12.964427Z","iopub.status.idle":"2023-05-05T10:48:13.012819Z","shell.execute_reply.started":"2023-05-05T10:48:12.964389Z","shell.execute_reply":"2023-05-05T10:48:13.011588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}