{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30528,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Importing necessary libraries\n\n# Data preprocessing\nimport pandas as pd\nimport numpy as np\n\n# Visualization\nimport matplotlib.pyplot as plt\n\n# Model\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow import keras\nfrom keras.utils import to_categorical\nfrom keras import layers, Sequential\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import EarlyStopping, ReduceLROnPlateau\n\n# Suppressing warnings\nfrom warnings import filterwarnings\nfilterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-02-09T14:01:22.981084Z","iopub.execute_input":"2024-02-09T14:01:22.981489Z","iopub.status.idle":"2024-02-09T14:01:22.987869Z","shell.execute_reply.started":"2024-02-09T14:01:22.981460Z","shell.execute_reply":"2024-02-09T14:01:22.986891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring Data","metadata":{}},{"cell_type":"code","source":"# Reading data\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:28:49.684282Z","iopub.execute_input":"2024-02-09T14:28:49.684649Z","iopub.status.idle":"2024-02-09T14:28:54.127259Z","shell.execute_reply.started":"2024-02-09T14:28:49.684620Z","shell.execute_reply":"2024-02-09T14:28:54.126199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Separating images and labels\nimages, labels = train.iloc[:, 1:], train['label']","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:29:42.258912Z","iopub.execute_input":"2024-02-09T14:29:42.259615Z","iopub.status.idle":"2024-02-09T14:29:42.264380Z","shell.execute_reply.started":"2024-02-09T14:29:42.259583Z","shell.execute_reply":"2024-02-09T14:29:42.263405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Displaying 16 random images\ndef displayImg(images, title):\n    plt.figure(figsize = (15, 10))\n    \n    for i in range(len(images)):\n        plt.subplot(4, 4, i + 1)\n        plt.title(title[i])\n        plt.imshow(np.reshape(images[i], (28, 28)), cmap = 'gray')\n        plt.axis('off')\n    plt.show()\n    \nrand = np.random.randint(0, images.shape[0] - 16)\ndisplayImg(images.iloc[rand:rand + 16].values, labels.iloc[rand:rand + 16].values)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:29:49.065191Z","iopub.execute_input":"2024-02-09T14:29:49.065571Z","iopub.status.idle":"2024-02-09T14:29:50.106875Z","shell.execute_reply.started":"2024-02-09T14:29:49.065542Z","shell.execute_reply":"2024-02-09T14:29:50.105913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocessing Data","metadata":{}},{"cell_type":"code","source":"labels = to_categorical(labels, num_classes = 10)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:01:28.603769Z","iopub.execute_input":"2024-02-09T14:01:28.604431Z","iopub.status.idle":"2024-02-09T14:01:28.609087Z","shell.execute_reply.started":"2024-02-09T14:01:28.604400Z","shell.execute_reply":"2024-02-09T14:01:28.608195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshaping data and scaling it from 0 to 1    \nimages = images.to_numpy().reshape(-1, 28, 28, 1) / 255","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:01:28.610164Z","iopub.execute_input":"2024-02-09T14:01:28.610424Z","iopub.status.idle":"2024-02-09T14:01:28.721702Z","shell.execute_reply.started":"2024-02-09T14:01:28.610401Z","shell.execute_reply":"2024-02-09T14:01:28.720842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining Model","metadata":{}},{"cell_type":"code","source":"# Adding early stopping to avoid overfitting\nearlyStopping = EarlyStopping(\n    monitor = 'val_accuracy',\n    min_delta = 5e-4,\n    patience = 7,\n    restore_best_weights = True\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:01:28.728855Z","iopub.execute_input":"2024-02-09T14:01:28.729097Z","iopub.status.idle":"2024-02-09T14:01:28.739799Z","shell.execute_reply.started":"2024-02-09T14:01:28.729076Z","shell.execute_reply":"2024-02-09T14:01:28.738969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Adding a learning rate annealer\nreduceLR = ReduceLROnPlateau(\n    monitor = 'val_accuracy',\n    patience = 3,\n    factor = 0.5,\n    min_lr = 1e-5\n)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:01:28.740731Z","iopub.execute_input":"2024-02-09T14:01:28.741024Z","iopub.status.idle":"2024-02-09T14:01:28.751276Z","shell.execute_reply.started":"2024-02-09T14:01:28.740999Z","shell.execute_reply":"2024-02-09T14:01:28.750252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Augmentation","metadata":{}},{"cell_type":"code","source":"datagen = ImageDataGenerator(\n    rotation_range = 10,\n    zoom_range = 0.1,\n    width_shift_range = 0.1,\n    height_shift_range = 0.1\n)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Ensemble of 10 CNNs","metadata":{}},{"cell_type":"code","source":"nets = 10\nmodel = [0] * nets\n\nfor i in range(nets):\n    model[i] = Sequential([\n        layers.Conv2D(32, (5, 5), activation = 'relu', padding = 'same', input_shape = (28, 28, 1)),\n        layers.Conv2D(32, (5, 5), activation = 'relu', padding = 'same'),\n        layers.MaxPool2D((2, 2)),\n\n        layers.Dropout(0.1),\n\n        layers.Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n        layers.Conv2D(64, (3,3), activation = 'relu', padding = 'same'),\n        layers.MaxPool2D((2,2), strides = (2,2)),\n\n        layers.Dropout(0.1),\n\n        layers.Flatten(),\n\n        layers.Dense(256, activation = 'relu'),\n\n        layers.Dropout(0.1),\n\n        layers.Dense(10, activation = 'softmax')\n    ])\n    \n    model[i].compile(optimizer = 'rmsprop', loss = 'categorical_crossentropy', metrics = ['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:01:28.753035Z","iopub.execute_input":"2024-02-09T14:01:28.753391Z","iopub.status.idle":"2024-02-09T14:01:30.084957Z","shell.execute_reply.started":"2024-02-09T14:01:28.753352Z","shell.execute_reply":"2024-02-09T14:01:30.084139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = [0] * nets\nepochs = 30\n\n# Creating a dataframe to store results\nresults = pd.DataFrame(columns = ['Model Number', 'Training Accuracy', 'Validation Accuracy'])\n\nfor i in range(nets):\n    trainX, testX, trainY, testY = train_test_split(images, labels, test_size = 0.1)\n    \n    # Training model\n    history[i] = model[i].fit(\n        datagen.flow(trainX, trainY, batch_size = 128),\n        validation_data = (testX, testY),\n        epochs = epochs,\n        callbacks = [earlyStopping, reduceLR],\n        workers = 4,\n        use_multiprocessing = True,\n        verbose = 0\n    )\n    \n    # Appending each iteration result to dataframe\n    results = results.append({\n        'Model Number' : i + 1,\n        'Training Accuracy' : max(history[i].history['accuracy']),\n        'Validation Accuracy' : max(history[i].history['val_accuracy'])\n    }, ignore_index = True)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:23:43.328783Z","iopub.execute_input":"2024-02-09T14:23:43.329449Z","iopub.status.idle":"2024-02-09T14:23:43.334168Z","shell.execute_reply.started":"2024-02-09T14:23:43.329418Z","shell.execute_reply":"2024-02-09T14:23:43.333086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:21:51.396883Z","iopub.execute_input":"2024-02-09T14:21:51.397214Z","iopub.status.idle":"2024-02-09T14:21:51.413248Z","shell.execute_reply.started":"2024-02-09T14:21:51.397183Z","shell.execute_reply":"2024-02-09T14:21:51.412195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Creating Submission","metadata":{}},{"cell_type":"code","source":"# Reshaping and rescaling test data\ntest = test.to_numpy().reshape(-1, 28, 28) / 255","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:21:51.414610Z","iopub.execute_input":"2024-02-09T14:21:51.414956Z","iopub.status.idle":"2024-02-09T14:21:51.492425Z","shell.execute_reply.started":"2024-02-09T14:21:51.414922Z","shell.execute_reply":"2024-02-09T14:21:51.491390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Making predictions\npred = np.zeros((test.shape[0], 10))\nfor i in range(nets):\n    pred = pred + model[i].predict(test)\n\npred = np.argmax(pred, axis = 1)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:21:51.493816Z","iopub.execute_input":"2024-02-09T14:21:51.494579Z","iopub.status.idle":"2024-02-09T14:22:12.230161Z","shell.execute_reply.started":"2024-02-09T14:21:51.494538Z","shell.execute_reply":"2024-02-09T14:22:12.229077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"i = np.random.randint(0, 28000)\nprint('Label predicted :', pred[i], '\\n')\nplt.imshow(test[i], cmap = 'gray')\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:23:34.487199Z","iopub.execute_input":"2024-02-09T14:23:34.488088Z","iopub.status.idle":"2024-02-09T14:23:34.550026Z","shell.execute_reply.started":"2024-02-09T14:23:34.488055Z","shell.execute_reply":"2024-02-09T14:23:34.549047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating submission csv\noutput = pd.DataFrame({'ImageId' : range(1, 28001), 'Label' : pred})\noutput.to_csv('Digit Recognizer.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2024-02-09T14:22:12.322562Z","iopub.execute_input":"2024-02-09T14:22:12.325488Z","iopub.status.idle":"2024-02-09T14:22:12.434220Z","shell.execute_reply.started":"2024-02-09T14:22:12.325443Z","shell.execute_reply":"2024-02-09T14:22:12.433540Z"},"trusted":true},"execution_count":null,"outputs":[]}]}