{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction:\n\n## What is a convolutional neural network (CNN)?\nA CNN  it's a neural network generaly used to analyse image data and recognize patterns.\nIt can recognize something in images even if these images are distorted, scaled and even rotated.\nA basic CNN is based in 4 steps:\n1. Conlutional layer:\n  This one test some filters and choose the better one\n\n2. MaxPooling:\n  This layer chooses the most important parts of the image to have the best information gain and reduce the processing\n\n3. FLattening:\n  Transforms the image in a array/vector in order to make it be fitted in a common Neural Network\n\n4. Neural network:\n  The last part is training it like a common neural networks.\n  \nYou can take a look in [this very well explained post on Towards data science medium blog](https://miro.medium.com/max/1838/1*uAeANQIOQPqWZnnuH-VEyw.jpeg)\n  \n![Image of a CNN structure by towards data science](https://miro.medium.com/max/1838/1*uAeANQIOQPqWZnnuH-VEyw.jpeg)","metadata":{}},{"cell_type":"markdown","source":"# Importing our libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom keras.callbacks import EarlyStopping\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-09-03T21:02:45.619991Z","iopub.execute_input":"2021-09-03T21:02:45.620499Z","iopub.status.idle":"2021-09-03T21:02:47.442024Z","shell.execute_reply.started":"2021-09-03T21:02:45.620414Z","shell.execute_reply":"2021-09-03T21:02:47.440840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory analysis:\n\nLet's take a look on our data and see if we can extract any insight","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:47.444479Z","iopub.execute_input":"2021-09-03T21:02:47.444840Z","iopub.status.idle":"2021-09-03T21:02:50.273330Z","shell.execute_reply.started":"2021-09-03T21:02:47.444802Z","shell.execute_reply":"2021-09-03T21:02:50.272483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:50.274915Z","iopub.execute_input":"2021-09-03T21:02:50.275250Z","iopub.status.idle":"2021-09-03T21:02:50.311295Z","shell.execute_reply.started":"2021-09-03T21:02:50.275214Z","shell.execute_reply":"2021-09-03T21:02:50.310566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So, as we can see this dataset has labeled data and each columns represents one of the pixels in the image, let's split our trainning data in X and y, X being the inputs and y being the outputs.","metadata":{}},{"cell_type":"code","source":"y = df['label'] #expected output\nX = df.drop('label', axis=1) #respective input","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:50.312532Z","iopub.execute_input":"2021-09-03T21:02:50.313015Z","iopub.status.idle":"2021-09-03T21:02:50.413759Z","shell.execute_reply.started":"2021-09-03T21:02:50.312977Z","shell.execute_reply":"2021-09-03T21:02:50.412527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.head()","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:50.415208Z","iopub.execute_input":"2021-09-03T21:02:50.415777Z","iopub.status.idle":"2021-09-03T21:02:50.452013Z","shell.execute_reply.started":"2021-09-03T21:02:50.415736Z","shell.execute_reply":"2021-09-03T21:02:50.450387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Is our data well distributed? what digits do we have and how many of each? let's take a look now","metadata":{}},{"cell_type":"code","source":"sns.countplot(y)\nplt.title('Numbers in trainning data')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:52.705127Z","iopub.execute_input":"2021-09-03T21:02:52.705493Z","iopub.status.idle":"2021-09-03T21:02:52.855936Z","shell.execute_reply.started":"2021-09-03T21:02:52.705456Z","shell.execute_reply":"2021-09-03T21:02:52.854968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nlist = []\nfor i in range(10):\n    number = np.array(df[df['label'] == i].drop('label', axis=1).iloc[0]).reshape(28, 28)\n    nlist.append(number)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:52.858940Z","iopub.execute_input":"2021-09-03T21:02:52.859309Z","iopub.status.idle":"2021-09-03T21:02:53.159572Z","shell.execute_reply.started":"2021-09-03T21:02:52.859268Z","shell.execute_reply":"2021-09-03T21:02:53.158683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualizing some example digits\nfig, axs = plt.subplots(2, 5)\na, b = 0, 0\nfor element in nlist:\n    axs[a, b].imshow(element)\n    b = b + 1\n    if b == 5:\n        b = 0\n        a = 1","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:53.161502Z","iopub.execute_input":"2021-09-03T21:02:53.161766Z","iopub.status.idle":"2021-09-03T21:02:53.906107Z","shell.execute_reply.started":"2021-09-03T21:02:53.161739Z","shell.execute_reply":"2021-09-03T21:02:53.905110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating CNN model:","metadata":{}},{"cell_type":"code","source":"X = np.array(X)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:53.907589Z","iopub.execute_input":"2021-09-03T21:02:53.907937Z","iopub.status.idle":"2021-09-03T21:02:54.005235Z","shell.execute_reply.started":"2021-09-03T21:02:53.907900Z","shell.execute_reply":"2021-09-03T21:02:54.004377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:54.006528Z","iopub.execute_input":"2021-09-03T21:02:54.006886Z","iopub.status.idle":"2021-09-03T21:02:54.012232Z","shell.execute_reply.started":"2021-09-03T21:02:54.006849Z","shell.execute_reply":"2021-09-03T21:02:54.011479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.max()\n\n#is easier for the computer to use normalized numbers, let's normalize it\n# dividing the whole dataset by 255\nX = X / 255","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:54.013372Z","iopub.execute_input":"2021-09-03T21:02:54.013857Z","iopub.status.idle":"2021-09-03T21:02:54.151114Z","shell.execute_reply.started":"2021-09-03T21:02:54.013821Z","shell.execute_reply":"2021-09-03T21:02:54.150231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Since our data is in only one row we need to transform each image into a 28x28 matrix\nX = X.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:54.152423Z","iopub.execute_input":"2021-09-03T21:02:54.152800Z","iopub.status.idle":"2021-09-03T21:02:54.157171Z","shell.execute_reply.started":"2021-09-03T21:02:54.152761Z","shell.execute_reply":"2021-09-03T21:02:54.156149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:54.158464Z","iopub.execute_input":"2021-09-03T21:02:54.158835Z","iopub.status.idle":"2021-09-03T21:02:54.169307Z","shell.execute_reply.started":"2021-09-03T21:02:54.158798Z","shell.execute_reply":"2021-09-03T21:02:54.168487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential([\n    layers.Conv2D(16, kernel_size=(3, 3), padding='same', activation='relu', input_shape=(28, 28, 1)),\n    layers.MaxPooling2D((3, 3)),\n    layers.Conv2D(32,  kernel_size=(3, 3), padding='same', activation='relu'),\n    layers.MaxPooling2D((3, 3)),\n    layers.Conv2D(32,  kernel_size=(3, 3), padding='same', activation='relu'),\n    layers.MaxPooling2D((3, 3)),\n    layers.Conv2D(64,  kernel_size=(3, 3), padding='same', activation='relu'),\n    \n    layers.Flatten(),\n    layers.Dense(128, activation='relu'),\n    layers.Dense(128, activation='relu'),\n    layers.Dropout(0.2, seed=42),\n    layers.Dense(64, activation='relu'),\n    layers.Dense(32, activation='relu'),\n    layers.Dense(16, activation='relu'),\n    layers.Dense(10, activation='softmax')\n])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:54.170659Z","iopub.execute_input":"2021-09-03T21:02:54.171019Z","iopub.status.idle":"2021-09-03T21:02:55.045064Z","shell.execute_reply.started":"2021-09-03T21:02:54.170983Z","shell.execute_reply":"2021-09-03T21:02:55.044210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer=\"Adam\", loss=\"SparseCategoricalCrossentropy\", metrics=[\"sparse_categorical_accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:55.046453Z","iopub.execute_input":"2021-09-03T21:02:55.046806Z","iopub.status.idle":"2021-09-03T21:02:55.061358Z","shell.execute_reply.started":"2021-09-03T21:02:55.046768Z","shell.execute_reply":"2021-09-03T21:02:55.060287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Early stopping system:\nWe will create an early stopping system that decides when to stop trainning with the data to prevent overfitting, this system will be based in the validation accuracy and the validation loss.\nIf the accuracy stops raising during 50 epochs or the loss stops decreasing the model will stop fitting.\n* Because we have this system we can set as many epochs as we want, the model itself will know when it's the time to stop.","metadata":{}},{"cell_type":"code","source":"es1 = EarlyStopping(patience=50, monitor='val_acc', mode='auto')\nes2 = EarlyStopping(patience=50, monitor='val_loss', mode='auto')","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:55.062923Z","iopub.execute_input":"2021-09-03T21:02:55.063295Z","iopub.status.idle":"2021-09-03T21:02:55.068842Z","shell.execute_reply.started":"2021-09-03T21:02:55.063258Z","shell.execute_reply":"2021-09-03T21:02:55.067749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(x=X,\n                    y=y,\n                    epochs=10000,\n                    validation_split=0.3,\n                    callbacks=[es1, es2])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:02:55.070210Z","iopub.execute_input":"2021-09-03T21:02:55.070731Z","iopub.status.idle":"2021-09-03T21:07:03.925317Z","shell.execute_reply.started":"2021-09-03T21:02:55.070692Z","shell.execute_reply":"2021-09-03T21:07:03.924214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualizing model perfomance:","metadata":{}},{"cell_type":"code","source":"time = np.arange(1, len(history.history['loss'])+1)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:03.928398Z","iopub.execute_input":"2021-09-03T21:07:03.928704Z","iopub.status.idle":"2021-09-03T21:07:03.935077Z","shell.execute_reply.started":"2021-09-03T21:07:03.928671Z","shell.execute_reply":"2021-09-03T21:07:03.934158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=history.history, x=time, y='loss')\nsns.lineplot(data=history.history, x=time, y='val_loss')\nplt.title('Loss fitting history')\nplt.legend(labels=['Loss', 'Validation loss'])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:03.937927Z","iopub.execute_input":"2021-09-03T21:07:03.938187Z","iopub.status.idle":"2021-09-03T21:07:04.148338Z","shell.execute_reply.started":"2021-09-03T21:07:03.938151Z","shell.execute_reply":"2021-09-03T21:07:04.147542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.lineplot(data=history.history, x=time, y='val_sparse_categorical_accuracy')\nsns.lineplot(data=history.history, x=time, y='sparse_categorical_accuracy')\nplt.title('Accuracy fitting history')\nplt.legend(labels=['Accuracy', 'Validation accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:04.149622Z","iopub.execute_input":"2021-09-03T21:07:04.149967Z","iopub.status.idle":"2021-09-03T21:07:04.380003Z","shell.execute_reply.started":"2021-09-03T21:07:04.149929Z","shell.execute_reply":"2021-09-03T21:07:04.379213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predicting values:","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:04.381212Z","iopub.execute_input":"2021-09-03T21:07:04.381554Z","iopub.status.idle":"2021-09-03T21:07:06.013931Z","shell.execute_reply.started":"2021-09-03T21:07:04.381518Z","shell.execute_reply":"2021-09-03T21:07:06.010205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X = np.array(test_df)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:06.019221Z","iopub.execute_input":"2021-09-03T21:07:06.019629Z","iopub.status.idle":"2021-09-03T21:07:06.102289Z","shell.execute_reply.started":"2021-09-03T21:07:06.019585Z","shell.execute_reply":"2021-09-03T21:07:06.101351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Normalizing the data\ntest_X = test_X / 255","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:06.103588Z","iopub.execute_input":"2021-09-03T21:07:06.104086Z","iopub.status.idle":"2021-09-03T21:07:06.188308Z","shell.execute_reply.started":"2021-09-03T21:07:06.104046Z","shell.execute_reply":"2021-09-03T21:07:06.187355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:06.189651Z","iopub.execute_input":"2021-09-03T21:07:06.190134Z","iopub.status.idle":"2021-09-03T21:07:06.196819Z","shell.execute_reply.started":"2021-09-03T21:07:06.190094Z","shell.execute_reply":"2021-09-03T21:07:06.196045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X = test_X.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:06.206265Z","iopub.execute_input":"2021-09-03T21:07:06.206790Z","iopub.status.idle":"2021-09-03T21:07:06.213276Z","shell.execute_reply.started":"2021-09-03T21:07:06.206755Z","shell.execute_reply":"2021-09-03T21:07:06.212553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_X.shape","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:06.214867Z","iopub.execute_input":"2021-09-03T21:07:06.216588Z","iopub.status.idle":"2021-09-03T21:07:06.225868Z","shell.execute_reply.started":"2021-09-03T21:07:06.216549Z","shell.execute_reply":"2021-09-03T21:07:06.224971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_pred = model.predict(test_X)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:06.227207Z","iopub.execute_input":"2021-09-03T21:07:06.227790Z","iopub.status.idle":"2021-09-03T21:07:07.772983Z","shell.execute_reply.started":"2021-09-03T21:07:06.227752Z","shell.execute_reply":"2021-09-03T21:07:07.771905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Saving predicted values:","metadata":{}},{"cell_type":"code","source":"numbers = final_pred.argmax(axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:07.776188Z","iopub.execute_input":"2021-09-03T21:07:07.776489Z","iopub.status.idle":"2021-09-03T21:07:07.782974Z","shell.execute_reply.started":"2021-09-03T21:07:07.776455Z","shell.execute_reply":"2021-09-03T21:07:07.782017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numbers","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:07.784473Z","iopub.execute_input":"2021-09-03T21:07:07.784919Z","iopub.status.idle":"2021-09-03T21:07:07.793557Z","shell.execute_reply.started":"2021-09-03T21:07:07.784876Z","shell.execute_reply":"2021-09-03T21:07:07.792472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df = pd.DataFrame(numbers, columns=['Label'])","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:07.794824Z","iopub.execute_input":"2021-09-03T21:07:07.795302Z","iopub.status.idle":"2021-09-03T21:07:07.803527Z","shell.execute_reply.started":"2021-09-03T21:07:07.795267Z","shell.execute_reply":"2021-09-03T21:07:07.802722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ImageId = np.array(final_df.index) + 1","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:07.804745Z","iopub.execute_input":"2021-09-03T21:07:07.805121Z","iopub.status.idle":"2021-09-03T21:07:07.816026Z","shell.execute_reply.started":"2021-09-03T21:07:07.805074Z","shell.execute_reply":"2021-09-03T21:07:07.815123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df['ImageId'] = ImageId","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:07.817461Z","iopub.execute_input":"2021-09-03T21:07:07.817855Z","iopub.status.idle":"2021-09-03T21:07:07.827047Z","shell.execute_reply.started":"2021-09-03T21:07:07.817816Z","shell.execute_reply":"2021-09-03T21:07:07.826163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:07.828182Z","iopub.execute_input":"2021-09-03T21:07:07.828604Z","iopub.status.idle":"2021-09-03T21:07:07.845625Z","shell.execute_reply.started":"2021-09-03T21:07:07.828565Z","shell.execute_reply":"2021-09-03T21:07:07.844704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_df.to_csv('final.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-03T21:07:07.847103Z","iopub.execute_input":"2021-09-03T21:07:07.847524Z","iopub.status.idle":"2021-09-03T21:07:07.899029Z","shell.execute_reply.started":"2021-09-03T21:07:07.847490Z","shell.execute_reply":"2021-09-03T21:07:07.898262Z"},"trusted":true},"execution_count":null,"outputs":[]}]}