{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"markdown","source":"## 2.1 Import Dependencies","metadata":{}},{"cell_type":"code","source":"# Data analysis and wrangling\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nprint(os.listdir(\"../input/digit-recognizer\"))\n\nfrom sklearn.model_selection import train_test_split\n\n# Use tensorflow.keras exclusively to avoid compatibility issues\nfrom tensorflow.keras.utils import to_categorical\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping, ModelCheckpoint, TensorBoard\n\n# Set seaborn style\nsns.set(style='white', context='notebook', palette='deep')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:20:30.003452Z","iopub.execute_input":"2024-12-25T20:20:30.003751Z","iopub.status.idle":"2024-12-25T20:20:30.010431Z","shell.execute_reply.started":"2024-12-25T20:20:30.003728Z","shell.execute_reply":"2024-12-25T20:20:30.009604Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2.2 Load Data","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:01:41.727922Z","iopub.execute_input":"2024-12-25T20:01:41.728495Z","iopub.status.idle":"2024-12-25T20:01:44.495176Z","shell.execute_reply.started":"2024-12-25T20:01:41.728466Z","shell.execute_reply":"2024-12-25T20:01:44.494221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# PREPARE DATA FOR NEURAL NETWORK\nY_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"],axis = 1)\nX_train = X_train / 255.0\nX_test = test / 255.0\nX_train = X_train.values.reshape(-1,28,28,1)\nX_test = X_test.values.reshape(-1,28,28,1)\nY_train = to_categorical(Y_train, num_classes = 10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:01:46.453128Z","iopub.execute_input":"2024-12-25T20:01:46.453562Z","iopub.status.idle":"2024-12-25T20:01:46.624101Z","shell.execute_reply.started":"2024-12-25T20:01:46.453529Z","shell.execute_reply":"2024-12-25T20:01:46.623423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n# PREVIEW IMAGES\nplt.figure(figsize=(15,4.5))\nfor i in range(30):  \n    plt.subplot(3, 10, i+1)\n    plt.imshow(X_train[i].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:01:49.882572Z","iopub.execute_input":"2024-12-25T20:01:49.882844Z","iopub.status.idle":"2024-12-25T20:01:51.797212Z","shell.execute_reply.started":"2024-12-25T20:01:49.882823Z","shell.execute_reply":"2024-12-25T20:01:51.796342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"g = sns.countplot(Y_train, palette='hsv')\ng.set_xlabel('Label')\ng.set_ylabel('Count')\ng.set_title('Y_train Labels')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:01:54.697963Z","iopub.execute_input":"2024-12-25T20:01:54.698318Z","iopub.status.idle":"2024-12-25T20:01:55.054147Z","shell.execute_reply.started":"2024-12-25T20:01:54.698292Z","shell.execute_reply":"2024-12-25T20:01:55.053260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CREATE MORE IMAGES VIA DATA AUGMENTATION\ndatagen = ImageDataGenerator(\n        rotation_range=10,  \n        zoom_range = 0.10,  \n        width_shift_range=0.1, \n        height_shift_range=0.1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:02:01.127786Z","iopub.execute_input":"2024-12-25T20:02:01.128195Z","iopub.status.idle":"2024-12-25T20:02:01.131942Z","shell.execute_reply.started":"2024-12-25T20:02:01.128162Z","shell.execute_reply":"2024-12-25T20:02:01.131063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize the ImageDataGenerator with desired augmentations\ndatagen = ImageDataGenerator(\n    rotation_range=10,      # Rotate images by up to 10 degrees\n    zoom_range=0.1,         # Zoom in by 10%\n    width_shift_range=0.1,  # Shift images horizontally by 10%\n    height_shift_range=0.1, # Shift images vertically by 10%\n    shear_range=0.1,        # Shear transformations\n    horizontal_flip=False,  # No horizontal flip for digit images\n    fill_mode='nearest'     # Fill in new pixels after transformation\n)\n\n# Function to plot augmented images\ndef plot_augmented_images(X, Y, indices, num_augmented=10):\n    plt.figure(figsize=(15, 4.5))\n    for i in range(num_augmented * len(indices)):\n        plt.subplot(len(indices), num_augmented, i + 1)\n        \n        # Select the appropriate index based on the current subplot\n        current_index = indices[i // num_augmented]\n        X_sample = X[current_index].reshape((1, 28, 28, 1))\n        Y_sample = Y[current_index].reshape((1, -1))  # Adjust based on Y's shape\n        \n        # Create a generator for the current sample\n        generator = datagen.flow(X_sample, Y_sample, batch_size=1)\n        \n        # Get the next augmented sample\n        X_augmented, Y_augmented = next(generator)\n        \n        # Display the image\n        plt.imshow(X_augmented[0].reshape((28, 28)), cmap=plt.cm.binary)\n        plt.axis('off')\n    \n    plt.subplots_adjust(wspace=-0.1, hspace=-0.1)\n    plt.show()\n\n# Define the indices of the samples you want to augment\nsample_indices = [9, 11, 18]\n\n# Plot augmented images for the selected samples\nplot_augmented_images(X_train, Y_train, sample_indices, num_augmented=10)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:02:04.122787Z","iopub.execute_input":"2024-12-25T20:02:04.123145Z","iopub.status.idle":"2024-12-25T20:02:06.257781Z","shell.execute_reply.started":"2024-12-25T20:02:04.123115Z","shell.execute_reply":"2024-12-25T20:02:06.256924Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Build 15 Convolutional Neural Networks","metadata":{}},{"cell_type":"code","source":"import datetime\n\n# Define the Learning Rate Scheduler Callback\nannealer = LearningRateScheduler(lambda epoch: 1e-3 * 0.95 ** epoch)\n\n# Define Additional Callbacks (Optional but Recommended)\nearly_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)\n# ModelCheckpoint will save the best model based on validation loss\ncheckpoint = ModelCheckpoint('best_model.keras', monitor='val_loss', save_best_only=True)\n# TensorBoard for visualization\nlog_dir = \"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\ntensorboard_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)\n\n# Combine Callbacks\ncallbacks = [annealer, early_stop, checkpoint, tensorboard_callback]\n\n# Number of models to build\nnets = 15\nmodels = [0] * nets  # Initialize a list to hold the models\n\n# Build the models (assuming you've already defined this part)\nfor j in range(nets):\n    models[j] = Sequential()\n    # Add layers as defined previously\n    models[j].add(Conv2D(32, kernel_size=3, activation='relu', input_shape=(28, 28, 1)))\n    models[j].add(BatchNormalization())\n    models[j].add(Conv2D(32, kernel_size=3, activation='relu'))\n    models[j].add(BatchNormalization())\n    models[j].add(Conv2D(32, kernel_size=5, strides=2, padding='same', activation='relu'))\n    models[j].add(BatchNormalization())\n    models[j].add(Dropout(0.4))\n\n    models[j].add(Conv2D(64, kernel_size=3, activation='relu'))\n    models[j].add(BatchNormalization())\n    models[j].add(Conv2D(64, kernel_size=3, activation='relu'))\n    models[j].add(BatchNormalization())\n    models[j].add(Conv2D(64, kernel_size=5, strides=2, padding='same', activation='relu'))\n    models[j].add(BatchNormalization())\n    models[j].add(Dropout(0.4))\n\n    models[j].add(Conv2D(128, kernel_size=4, activation='relu'))\n    models[j].add(BatchNormalization())\n    models[j].add(Flatten())\n    models[j].add(Dropout(0.4))\n    models[j].add(Dense(10, activation='softmax'))\n\n    # Compile the model\n    models[j].compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:21:59.568685Z","iopub.execute_input":"2024-12-25T20:21:59.569030Z","iopub.status.idle":"2024-12-25T20:22:02.825107Z","shell.execute_reply.started":"2024-12-25T20:21:59.569001Z","shell.execute_reply":"2024-12-25T20:22:02.824417Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train CNN","metadata":{}},{"cell_type":"code","source":"# TRAIN NETWORKS\nhistory = [0] * nets\nepochs = 45\nbatch_size = 64\n# TRAIN NETWORKS\nfor j in range(nets):\n    print(f\"\\nStarting training for CNN {j+1}/{nets}\")\n    \n    # Split the Data into Training and Validation Sets with Unique Random State\n    X_train2, X_val2, Y_train2, Y_val2 = train_test_split(\n        X_train, Y_train, test_size=0.1, random_state=42 + j\n    )\n    \n    # Calculate Steps Per Epoch Using np.ceil to Ensure All Data is Covered\n    steps_per_epoch = int(np.ceil(X_train2.shape[0] / batch_size))\n    \n    # Train the Model Using fit with verbose=1 to Show Progress\n    history[j] = models[j].fit(\n        datagen.flow(X_train2, Y_train2, batch_size=batch_size),\n        epochs=epochs,\n        steps_per_epoch=steps_per_epoch,\n        validation_data=(X_val2, Y_val2),\n        callbacks=callbacks,\n        verbose=1  # Set to 1 for progress bar, 2 for one line per epoch\n    )\n    \n    # Retrieve the Highest Accuracy Achieved During Training\n    train_acc = max(history[j].history.get('accuracy', [0]))\n    val_acc = max(history[j].history.get('val_accuracy', [0]))\n    \n    print(f\"CNN {j+1}: Epochs={epochs}, Train accuracy={train_acc:.5f}, Validation accuracy={val_acc:.5f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:22:12.938409Z","iopub.execute_input":"2024-12-25T20:22:12.938717Z","iopub.status.idle":"2024-12-25T20:38:17.497269Z","shell.execute_reply.started":"2024-12-25T20:22:12.938693Z","shell.execute_reply":"2024-12-25T20:38:17.496358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ENSEMBLE PREDICTIONS AND SUBMIT\nresults = np.zeros( (X_test.shape[0],10) ) \nfor j in range(nets):\n    results = results + models[j].predict(X_test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"MNIST-CNN-ENSEMBLE.csv\",index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:39:34.193490Z","iopub.execute_input":"2024-12-25T20:39:34.193786Z","iopub.status.idle":"2024-12-25T20:40:08.801064Z","shell.execute_reply.started":"2024-12-25T20:39:34.193763Z","shell.execute_reply":"2024-12-25T20:40:08.800390Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# PREVIEW PREDICTIONS\nplt.figure(figsize=(15,6))\nfor i in range(40):  \n    plt.subplot(4, 10, i+1)\n    plt.imshow(X_test[i].reshape((28,28)),cmap=plt.cm.binary)\n    plt.title(\"predict=%d\" % results[i],y=0.9)\n    plt.axis('off')\nplt.subplots_adjust(wspace=0.3, hspace=-0.1)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-25T20:40:17.253027Z","iopub.execute_input":"2024-12-25T20:40:17.253326Z","iopub.status.idle":"2024-12-25T20:40:18.053871Z","shell.execute_reply.started":"2024-12-25T20:40:17.253303Z","shell.execute_reply":"2024-12-25T20:40:18.052874Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Credits\nThe code here was composed from the following outstanding Kaggler(s) notebooks\n\n* [Chris Deotte] - [https://www.kaggle.com/code/cdeotte/25-million-images-0-99757-mnist]","metadata":{}}]}