{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# INTRODUCTION TO CNN- 99.6% ACCURACY (TOP 10 %)\n\n# TABLE OF CONTENT\n\n1. Introduction\n2. Data preparation\n3. CNN\n\n    3.1.  Define the model\n    \n    \n    3.2.  Set the optimizer \n4. Evaluate the model\n\n5. Prediction and submition","metadata":{}},{"cell_type":"markdown","source":"# 1. Introduction\n\nThis is a Sequential Convolutional Neural Network for digits recognition trained on MNIST dataset. We will build it with keras API. Firstly, I will prepare the data (handwritten digits images) then i will focus on the CNN modeling and evaluation. And finally i will do prediction and submission.\n\nI achieved Acuracy of more then 99.5% with this CNN model . \n\n**For computational reasons, i have set the epochs to 5 only as of now however  if you want to achieve further better accuracy set it to 50 but do remeber it will increase computational time too**.\n\n> # Please do UPVOTE to help me remain motivated in this journey of learning.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:18:23.615286Z","iopub.execute_input":"2023-06-05T12:18:23.615680Z","iopub.status.idle":"2023-06-05T12:18:23.628339Z","shell.execute_reply.started":"2023-06-05T12:18:23.615652Z","shell.execute_reply":"2023-06-05T12:18:23.627173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Importing minimum necessary libraries. We will import libraries as and when required.\nimport pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:19:25.796628Z","iopub.execute_input":"2023-06-05T12:19:25.797056Z","iopub.status.idle":"2023-06-05T12:19:25.803117Z","shell.execute_reply.started":"2023-06-05T12:19:25.797023Z","shell.execute_reply":"2023-06-05T12:19:25.801892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Load the data and prepare it","metadata":{}},{"cell_type":"code","source":"data=pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:42:11.341502Z","iopub.execute_input":"2023-06-05T11:42:11.342298Z","iopub.status.idle":"2023-06-05T11:42:17.072970Z","shell.execute_reply.started":"2023-06-05T11:42:11.342262Z","shell.execute_reply":"2023-06-05T11:42:17.071335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_test= pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:42:17.075514Z","iopub.execute_input":"2023-06-05T11:42:17.077196Z","iopub.status.idle":"2023-06-05T11:42:20.201482Z","shell.execute_reply.started":"2023-06-05T11:42:17.077128Z","shell.execute_reply":"2023-06-05T11:42:20.199713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.info()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:42:20.205033Z","iopub.execute_input":"2023-06-05T11:42:20.205480Z","iopub.status.idle":"2023-06-05T11:42:20.280917Z","shell.execute_reply.started":"2023-06-05T11:42:20.205445Z","shell.execute_reply":"2023-06-05T11:42:20.279324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:42:20.282862Z","iopub.execute_input":"2023-06-05T11:42:20.283401Z","iopub.status.idle":"2023-06-05T11:42:20.333198Z","shell.execute_reply.started":"2023-06-05T11:42:20.283352Z","shell.execute_reply":"2023-06-05T11:42:20.331526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=data.drop('label', axis  =1 )\ny=data['label']\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T11:42:20.335612Z","iopub.execute_input":"2023-06-05T11:42:20.336059Z","iopub.status.idle":"2023-06-05T11:42:20.558179Z","shell.execute_reply.started":"2023-06-05T11:42:20.336015Z","shell.execute_reply":"2023-06-05T11:42:20.556843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Split the data into training and testing data\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.2 , random_state= 42)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:21:53.985493Z","iopub.execute_input":"2023-06-05T12:21:53.985928Z","iopub.status.idle":"2023-06-05T12:21:54.530136Z","shell.execute_reply.started":"2023-06-05T12:21:53.985895Z","shell.execute_reply":"2023-06-05T12:21:54.528599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***Part 1 ---- Lets first Try simple logistic regression to identify it***","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nmodel= LogisticRegression(max_iter=100)\nmodel.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T00:54:26.491816Z","iopub.execute_input":"2023-06-03T00:54:26.492596Z","iopub.status.idle":"2023-06-03T00:54:43.348935Z","shell.execute_reply.started":"2023-06-03T00:54:26.492533Z","shell.execute_reply":"2023-06-03T00:54:43.347427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred= model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T00:54:43.352094Z","iopub.execute_input":"2023-06-03T00:54:43.352911Z","iopub.status.idle":"2023-06-03T00:54:43.448662Z","shell.execute_reply.started":"2023-06-03T00:54:43.352856Z","shell.execute_reply":"2023-06-03T00:54:43.447034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy= accuracy_score(y_pred, y_test)\nprint(f\"Validation Accuracy: {accuracy}\")\n","metadata":{"execution":{"iopub.status.busy":"2023-06-03T00:54:43.451016Z","iopub.execute_input":"2023-06-03T00:54:43.451958Z","iopub.status.idle":"2023-06-03T00:54:43.465712Z","shell.execute_reply.started":"2023-06-03T00:54:43.451898Z","shell.execute_reply":"2023-06-03T00:54:43.463852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_pred=model.predict(data_test)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T00:54:43.469720Z","iopub.execute_input":"2023-06-03T00:54:43.470636Z","iopub.status.idle":"2023-06-03T00:54:43.681155Z","shell.execute_reply.started":"2023-06-03T00:54:43.470565Z","shell.execute_reply":"2023-06-03T00:54:43.679252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare submission file\nsubmission = pd.DataFrame({'ImageId': range(1, len(data_test)+1), 'Label': test_pred})\n\n# Save the submission file\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-03T00:54:43.683475Z","iopub.execute_input":"2023-06-03T00:54:43.684385Z","iopub.status.idle":"2023-06-03T00:54:43.814790Z","shell.execute_reply.started":"2023-06-03T00:54:43.684328Z","shell.execute_reply":"2023-06-03T00:54:43.813911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We found that that accuracy is around 91%. Not Bad. Lets try improving further with NN","metadata":{}},{"cell_type":"markdown","source":"# **Part2 --- Now Lets try convolutional neural network (CNN) architecture****","metadata":{}},{"cell_type":"code","source":"#Importing libraries for CNN\nfrom sklearn.preprocessing import LabelBinarizer\nfrom keras.models import Sequential\nfrom keras.layers import Conv2D, MaxPooling2D, Flatten, Dense\nfrom keras.optimizers import Adam\nfrom keras.callbacks import EarlyStopping\nfrom keras.utils import to_categorical","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:26:51.164170Z","iopub.execute_input":"2023-06-05T12:26:51.164593Z","iopub.status.idle":"2023-06-05T12:26:51.171865Z","shell.execute_reply.started":"2023-06-05T12:26:51.164558Z","shell.execute_reply":"2023-06-05T12:26:51.170991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split the data into training and validation sets\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# Reshape and normalize the input data\nX_train = X_train.values.reshape(-1, 28, 28, 1) / 255.0\nX_val = X_val.values.reshape(-1, 28, 28, 1) / 255.0\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:26:51.860233Z","iopub.execute_input":"2023-06-05T12:26:51.861064Z","iopub.status.idle":"2023-06-05T12:26:52.240515Z","shell.execute_reply.started":"2023-06-05T12:26:51.861025Z","shell.execute_reply":"2023-06-05T12:26:52.239329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert labels to categorical format\nlabel_binarizer = LabelBinarizer()\ny_train = label_binarizer.fit_transform(y_train)\ny_val = label_binarizer.transform(y_val)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:26:52.384806Z","iopub.execute_input":"2023-06-05T12:26:52.385573Z","iopub.status.idle":"2023-06-05T12:26:52.401960Z","shell.execute_reply.started":"2023-06-05T12:26:52.385535Z","shell.execute_reply":"2023-06-05T12:26:52.400948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. CNN MODEL AND OPTIMISATION\n\nI used the Keras Sequential API. We will add one layer at a time, starting from the input.\n\nThe first is the convolutional (Conv2D) layer. It is like a set of learnable filters. I choosed to set 32 filters for the two firsts conv2D layers and 64 filters for the two last ones. \n\nThe second important layer in CNN is the pooling (MaxPool2D) layer. This layer simply acts as a downsampling filter. \n\nCombining convolutional and pooling layers, CNN are able to combine local features and learn more global features of the image.\n\nThe Flatten layer is use to convert the final feature maps into a one single 1D vector. T\n","metadata":{}},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout\n\n# Define the CNN model\nmodel = Sequential()\nmodel.add(Conv2D(32, kernel_size=(5, 5), activation='relu', input_shape=(28, 28, 1)))\nmodel.add(Conv2D(32, kernel_size=(5, 5), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Dropout(0.25))\nmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\nmodel.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))\nmodel.add(MaxPooling2D(pool_size=(2, 2)))\nmodel.add(Flatten())\nmodel.add(Dense(128, activation='relu'))\nmodel.add(Dense(10, activation='softmax'))\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:39:50.366615Z","iopub.execute_input":"2023-06-05T12:39:50.367028Z","iopub.status.idle":"2023-06-05T12:39:50.501980Z","shell.execute_reply.started":"2023-06-05T12:39:50.366997Z","shell.execute_reply":"2023-06-05T12:39:50.500057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:39:50.749254Z","iopub.execute_input":"2023-06-05T12:39:50.749723Z","iopub.status.idle":"2023-06-05T12:39:50.760028Z","shell.execute_reply.started":"2023-06-05T12:39:50.749689Z","shell.execute_reply":"2023-06-05T12:39:50.757963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Set the optimizer**\nThe most important function is the optimizer. This function will iteratively improve parameters  in order to minimise the loss.\n\nI choosed RMSprop default values.  The RMSProp update adjusts the Adagrad method in a very simple way in an attempt to reduce \nits aggressive, monotonically decreasing learning rate. \n\nThe metric function \"accuracy\" is used is to evaluate the performance our model. This metric function is similar to the loss\nfunction, except that the results from the metric evaluation are not used when training the model (only for evaluation).","metadata":{}},{"cell_type":"code","source":"from keras.optimizers import RMSprop\n\noptimizer = RMSprop(lr=0.001, rho=0.9, epsilon=1e-08, decay=0.0)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:39:51.588166Z","iopub.execute_input":"2023-06-05T12:39:51.588800Z","iopub.status.idle":"2023-06-05T12:39:51.596082Z","shell.execute_reply.started":"2023-06-05T12:39:51.588705Z","shell.execute_reply":"2023-06-05T12:39:51.594755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Compile the model\nmodel.compile(optimizer = optimizer , loss = \"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:39:51.945262Z","iopub.execute_input":"2023-06-05T12:39:51.945887Z","iopub.status.idle":"2023-06-05T12:39:51.964363Z","shell.execute_reply.started":"2023-06-05T12:39:51.945838Z","shell.execute_reply":"2023-06-05T12:39:51.962289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set up early stopping\nearly_stopping = EarlyStopping(patience=3, restore_best_weights=True)","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:39:52.864183Z","iopub.execute_input":"2023-06-05T12:39:52.864605Z","iopub.status.idle":"2023-06-05T12:39:52.871204Z","shell.execute_reply.started":"2023-06-05T12:39:52.864573Z","shell.execute_reply":"2023-06-05T12:39:52.869408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train the model\nhistory = model.fit(X_train, y_train, batch_size=32, epochs=10, validation_data=(X_val, y_val), callbacks=[early_stopping])\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:39:53.461684Z","iopub.execute_input":"2023-06-05T12:39:53.462216Z","iopub.status.idle":"2023-06-05T12:47:16.922227Z","shell.execute_reply.started":"2023-06-05T12:39:53.462175Z","shell.execute_reply":"2023-06-05T12:47:16.920604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. LETS DO PREDICTION NOW","metadata":{}},{"cell_type":"code","source":"# Load the test dataset\n# We have already loaded it in data_test \n\n# Preprocess the test data\nX_test = data_test.values.reshape(-1, 28, 28, 1) / 255.0\n\n# Make predictions on the test set\ntest_predictions = model.predict(X_test)\n","metadata":{"execution":{"iopub.status.busy":"2023-06-05T12:25:28.410702Z","iopub.execute_input":"2023-06-05T12:25:28.411185Z","iopub.status.idle":"2023-06-05T12:25:41.512048Z","shell.execute_reply.started":"2023-06-05T12:25:28.411150Z","shell.execute_reply":"2023-06-05T12:25:41.510818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using argmax() to return maximum values along an axis.\ntest_preds = []\nfor i in range(len(test_predictions)):\n    test_preds.append(np.argmax(test_predictions[i]))","metadata":{"execution":{"iopub.status.busy":"2023-06-04T14:55:41.893606Z","iopub.execute_input":"2023-06-04T14:55:41.894322Z","iopub.status.idle":"2023-06-04T14:55:42.029245Z","shell.execute_reply.started":"2023-06-04T14:55:41.894261Z","shell.execute_reply":"2023-06-04T14:55:42.027839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize some predicted images\nf,ax = plt.subplots(3, 7, figsize=(21,9))\nfor i in range(7):\n    ax[0][i].imshow(X_test[i])\n    ax[0][i].set_xlabel(test_preds[i])\n    ax[1][i].imshow(X_test[i+100])\n    ax[1][i].set_xlabel(test_preds[i+100])\n    ax[2][i].imshow(X_test[i+500])\n    ax[2][i].set_xlabel(test_preds[i+500])\n    \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-06-04T14:55:42.974457Z","iopub.execute_input":"2023-06-04T14:55:42.974854Z","iopub.status.idle":"2023-06-04T14:55:46.395226Z","shell.execute_reply.started":"2023-06-04T14:55:42.974825Z","shell.execute_reply":"2023-06-04T14:55:46.393482Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. SUBMITTING FOR EVALUATION","metadata":{}},{"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-06-04T14:55:46.397422Z","iopub.execute_input":"2023-06-04T14:55:46.398723Z","iopub.status.idle":"2023-06-04T14:55:46.422414Z","shell.execute_reply.started":"2023-06-04T14:55:46.398669Z","shell.execute_reply":"2023-06-04T14:55:46.420523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.Label = test_preds","metadata":{"execution":{"iopub.status.busy":"2023-06-04T14:55:47.001802Z","iopub.execute_input":"2023-06-04T14:55:47.002178Z","iopub.status.idle":"2023-06-04T14:55:47.159616Z","shell.execute_reply.started":"2023-06-04T14:55:47.002150Z","shell.execute_reply":"2023-06-04T14:55:47.158389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-04T14:55:52.053207Z","iopub.execute_input":"2023-06-04T14:55:52.053860Z","iopub.status.idle":"2023-06-04T14:55:52.133207Z","shell.execute_reply.started":"2023-06-04T14:55:52.053821Z","shell.execute_reply":"2023-06-04T14:55:52.131887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ***If you found this notebook helpful or you just liked it , some upvotes would be very much appreciated - That will keep me motivated :)***","metadata":{}}]}