{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30616,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.layers import Input, Conv2D, BatchNormalization, ReLU, Dropout, Concatenate, Flatten, Dense, Add, Activation\nfrom tensorflow.keras.models import Model\nimport tensorflow_addons as tfa","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nTRAIN_CSV = \"/kaggle/input/digit-recognizer/train.csv\"\nTEST_CSV = \"/kaggle/input/digit-recognizer/test.csv\"\nSUBMISSION_FILE = \"/kaggle/working/submission.csv\"\nMODEL_PATH = \"/kaggle/working/best_model.h5\"\nIMAGE_WIDTH, IMAGE_HEIGHT = 28, 28\nBATCH_SIZE = 32\nEPOCHS = 40\nINITIAL_LR = 1e-4\nWEIGHT_DECAY = 1e-4\nDROPOUT_RATE = 0.5\nDECAY_STEPS = 512\nDECAY_RATE = 0.98","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data_from_csv(file_path, is_train=True):\n    df = pd.read_csv(file_path)\n    if is_train:\n        images = df.iloc[:, 1:].values.reshape(-1, IMAGE_WIDTH, IMAGE_HEIGHT, 1).astype(np.float32) / 255.0\n        labels = pd.get_dummies(df['label']).values\n        return images, labels\n    else:\n        images = df.values.reshape(-1, IMAGE_WIDTH, IMAGE_HEIGHT, 1).astype(np.float32) / 255.0\n        return images, None","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_generators():\n    train_images, train_labels = load_data_from_csv(TRAIN_CSV, is_train=True)\n    test_images, _ = load_data_from_csv(TEST_CSV, is_train=False)\n\n    # Data augmentation\n    datagen_args = {\n        'width_shift_range': 0.15,\n        'height_shift_range': 0.15,\n        'shear_range': 0.15,\n        'zoom_range': 0.15,\n        'rotation_range': 18,\n        'fill_mode': 'nearest'\n    }\n    \n    datagen_args['validation_split'] = 0.025\n\n    datagen = ImageDataGenerator(**datagen_args)\n    train_generator_args = {\n        'x': train_images,\n        'y': train_labels,\n        'batch_size': BATCH_SIZE,\n        'shuffle': True\n    }\n\n    train_generator_args['subset'] = 'training'\n\n    train_generator = datagen.flow(**train_generator_args)\n    validation_generator = datagen.flow(\n        x=train_images,\n        y=train_labels,\n        batch_size=BATCH_SIZE,\n        subset='validation'\n    )\n\n    return train_generator, validation_generator, test_images","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n    def count_conv2d_layers(model):\n        conv_layer_types = (Conv2D,)\n        return sum([1 for layer in model.layers if isinstance(layer, conv_layer_types)])\n    \n    def count_dense_layers(model):\n        return sum([1 for layer in model.layers if isinstance(layer, Dense)])\n    \n    def ResidualConvBlock(x, num_filters, kernel_size=3, strides=1, padding='same'):\n        # Projection layer is needed if the number of filters changes\n        identity = x\n        identity_shape = identity.shape[-1]\n        if identity_shape != num_filters:\n            identity = Conv2D(num_filters, 1, strides=strides, padding=padding)(identity)\n            identity = BatchNormalization()(identity)\n\n        # Convolutional block\n        x = Conv2D(num_filters, kernel_size, strides=strides, padding=padding)(x)\n        x = BatchNormalization()(x)\n        x = ReLU()(x)\n        x = Dropout(DROPOUT_RATE / 2)(x)\n\n        # Add the residual connection\n        x = Add()([x, identity])\n        return x\n    \n    def ResidualDenseBlock(input_tensor, units):\n        # Projection layer to match the input tensor shape to the output tensor shape\n        projection = Dense(units, activation=None)(input_tensor)\n        projection = BatchNormalization()(projection)\n\n        # Dense block\n        x = Dense(units, activation=None)(input_tensor)\n        x = BatchNormalization()(x)\n        x = Activation('relu')(x)\n        x = Dropout(DROPOUT_RATE)(x)\n\n        x = Dense(units, activation=None)(x)\n        x = BatchNormalization()(x)\n        x = Activation('relu')(x)\n\n        # Add the projection and the output of the dense block\n        x = Add()([x, projection])\n        x = Activation('relu')(x)\n        return x\n\n    a = Input(shape=(IMAGE_WIDTH, IMAGE_HEIGHT, 1))\n\n    # Adding ConvBlock\n    x = ResidualConvBlock(a, 384, padding='same')\n    x = ResidualConvBlock(x, 512, padding='same')\n\n    # Flatten layer to reshape the tensor for dense layers\n    x = Flatten()(x)\n\n    # Top layers\n    x = ResidualDenseBlock(x, 256)\n    x = ResidualDenseBlock(x, 192)\n    x = ResidualDenseBlock(x, 128)\n    \n    x = Dense(10, activation=\"softmax\")(x)\n\n    model = Model(inputs=a, outputs=x)\n\n    print(\"Model conv2d layers:\", count_conv2d_layers(model))\n    print(\"Model dense layers:\", count_dense_layers(model))\n\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n\n    train_generator, validation_generator, test_images = create_generators()\n    model = create_model()\n\n    # Define the ExponentialDecay learning rate scheduler\n    scheduler = ExponentialDecay(\n        initial_learning_rate=INITIAL_LR,\n        decay_steps=DECAY_STEPS,\n        decay_rate=DECAY_RATE,\n        staircase=True\n    )\n\n    # Pass the scheduler directly to the optimizer\n    optimizer = tfa.optimizers.AdamW(learning_rate=scheduler, weight_decay=WEIGHT_DECAY)\n\n    # Compile the model\n    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n\n    model_checkpoint = ModelCheckpoint(\n        MODEL_PATH,\n        monitor='val_loss',\n        save_best_only=True,\n        mode='min',\n        verbose=1\n    )\n\n    model.fit(\n        train_generator,\n        epochs=EPOCHS,\n        validation_data=validation_generator,\n        callbacks=[model_checkpoint]\n    )\n\n    best_model = load_model(MODEL_PATH)\n    predictions = best_model.predict(test_images, batch_size=BATCH_SIZE)\n    class_predictions = np.argmax(predictions, axis=1)\n    submission = pd.DataFrame({\"ImageId\": range(1, len(class_predictions) + 1), \"Label\": class_predictions})\n    submission.to_csv(SUBMISSION_FILE, index=False)\n    \nif __name__ == \"__main__\":\n    main()","metadata":{},"execution_count":null,"outputs":[]}]}