{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import Libraries\nimport pandas as pd\nimport numpy as np\nfrom tensorflow.keras import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.optimizers import Adam\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:05.527530Z","iopub.execute_input":"2023-10-25T18:04:05.527928Z","iopub.status.idle":"2023-10-25T18:04:05.534115Z","shell.execute_reply.started":"2023-10-25T18:04:05.527890Z","shell.execute_reply":"2023-10-25T18:04:05.533091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Hyperparameters\nENSEMBLE_MODELS = 15\nEPOCHS = 50\nLEARNING_RATE = 0.001","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:05.538869Z","iopub.execute_input":"2023-10-25T18:04:05.539927Z","iopub.status.idle":"2023-10-25T18:04:05.547505Z","shell.execute_reply.started":"2023-10-25T18:04:05.539893Z","shell.execute_reply":"2023-10-25T18:04:05.546570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Data\ntraining_data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest_data = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\n# Split X and y in the training set\ny = training_data['label']\nX = training_data.drop('label', axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:05.549431Z","iopub.execute_input":"2023-10-25T18:04:05.550006Z","iopub.status.idle":"2023-10-25T18:04:09.856348Z","shell.execute_reply.started":"2023-10-25T18:04:05.549975Z","shell.execute_reply":"2023-10-25T18:04:09.855540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize X\nX = X / 255.0\ntest_data = test_data / 255.0","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:09.857565Z","iopub.execute_input":"2023-10-25T18:04:09.857948Z","iopub.status.idle":"2023-10-25T18:04:09.950169Z","shell.execute_reply.started":"2023-10-25T18:04:09.857915Z","shell.execute_reply":"2023-10-25T18:04:09.949382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# One-hot encode y\nencoder = OneHotEncoder()\ny = encoder.fit_transform(y.values.reshape(-1, 1)).toarray()","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:09.951429Z","iopub.execute_input":"2023-10-25T18:04:09.951801Z","iopub.status.idle":"2023-10-25T18:04:09.966599Z","shell.execute_reply.started":"2023-10-25T18:04:09.951769Z","shell.execute_reply":"2023-10-25T18:04:09.965687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshape X and test_data to 28x28x1 3D matrices\nX = X.values.reshape(-1, 28, 28, 1)\ntest_data = test_data.values.reshape(-1, 28, 28, 1)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:09.968590Z","iopub.execute_input":"2023-10-25T18:04:09.968885Z","iopub.status.idle":"2023-10-25T18:04:09.973458Z","shell.execute_reply.started":"2023-10-25T18:04:09.968862Z","shell.execute_reply":"2023-10-25T18:04:09.972539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model():\n    # Create a ConvNet model\n    model = Sequential()\n    # Conv2D use filters to view segments of the image one at at time to extract features\n    model.add(Conv2D(32, (5, 5), activation='relu', input_shape=(28, 28, 1)))\n    # Batch Normalization layers act as a normalizer, speeding convergence and protecting against vanishing gradients\n    model.add(BatchNormalization())\n    model.add(Dropout(0.4))\n    model.add(Conv2D(32, (5, 5), activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.4))\n    model.add(MaxPooling2D((2, 2)))\n    model.add(Conv2D(32, (3, 3), strides=2, padding='same', activation='relu'))\n    model.add(BatchNormalization())\n    model.add(Dropout(0.4))\n    model.add(Flatten())\n    model.add(Dense(256, activation='relu'))\n    model.add(Dense(256, activation='relu'))\n    model.add(Dense(10, activation='softmax'))\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:09.974608Z","iopub.execute_input":"2023-10-25T18:04:09.974962Z","iopub.status.idle":"2023-10-25T18:04:09.985466Z","shell.execute_reply.started":"2023-10-25T18:04:09.974922Z","shell.execute_reply":"2023-10-25T18:04:09.984662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = []\n\nfor i in range(ENSEMBLE_MODELS):\n    print(f\"Training model {i+1} of {ENSEMBLE_MODELS}...\")\n    model = build_model()\n    adam = Adam(learning_rate=LEARNING_RATE)\n    model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy'])\n    \n\n    # Split the training data into training and validation sets\n    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2)\n\n    # Create an image data generator\n    datagen = ImageDataGenerator(rotation_range=10, zoom_range=0.1, width_shift_range=0.1, height_shift_range=0.1, vertical_flip=False, horizontal_flip=False)\n\n    # Create an early stopping callback\n    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n    lr_reduction = ReduceLROnPlateau(monitor='val_loss', patience=2, verbose=1, factor=0.5, min_lr=0.00001)\n\n    # Train the model\n    model.fit(X_train, y_train, epochs=EPOCHS, batch_size=64, callbacks=[early_stopping, lr_reduction], validation_data=(X_val, y_val))\n\n    models.append(model)","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:04:09.986617Z","iopub.execute_input":"2023-10-25T18:04:09.987231Z","iopub.status.idle":"2023-10-25T18:27:27.320048Z","shell.execute_reply.started":"2023-10-25T18:04:09.987200Z","shell.execute_reply":"2023-10-25T18:27:27.319223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predict the test data\nall_predictions = []\nprint(f\"Classifying test data over {ENSEMBLE_MODELS} models...\")\nfor model in models:\n    # Predict the test data\n    predictions = model.predict(test_data)\n    all_predictions.append(predictions)\n\n# Average the predictions from all models and convert to labels\npredictions = np.argmax(np.mean(all_predictions, axis=0), axis=1)\n\n# Create a submission file\nsubmission = pd.DataFrame({'ImageId': range(1, len(predictions) + 1), 'Label': predictions})\nsubmission.to_csv('/kaggle/working/submission.csv', index=False)\nprint(\"Submission created and saved.\")","metadata":{"execution":{"iopub.status.busy":"2023-10-25T18:33:35.881252Z","iopub.execute_input":"2023-10-25T18:33:35.881618Z","iopub.status.idle":"2023-10-25T18:34:06.768948Z","shell.execute_reply.started":"2023-10-25T18:33:35.881590Z","shell.execute_reply":"2023-10-25T18:34:06.767995Z"},"trusted":true},"execution_count":null,"outputs":[]}]}