{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30616,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import load_model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.optimizers.schedules import ExponentialDecay\nfrom tensorflow.keras.preprocessing.image import ImageDataGenerator\nimport tensorflow_addons as tfa","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Constants\nTRAIN_CSV = \"/kaggle/input/digit-recognizer/train.csv\"\nTEST_CSV = \"/kaggle/input/digit-recognizer/test.csv\"\nSUBMISSION_FILE = \"/kaggle/working/submission.csv\"\nMODEL_PATH = \"/kaggle/working/best_model.h5\"\nIMAGE_WIDTH, IMAGE_HEIGHT = 28, 28\nBATCH_SIZE = 96\nEPOCHS = 120\nINITIAL_LR = 1e-4\nWEIGHT_DECAY = 2.5e-4\nDROPOUT_RATE = 0.5\nDECAY_STEPS = 512\nDECAY_RATE = 0.98","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data_from_csv(file_path, is_train=True):\n    df = pd.read_csv(file_path)\n    if is_train:\n        images = df.iloc[:, 1:].values.reshape(-1, IMAGE_WIDTH, IMAGE_HEIGHT, 1).astype(np.float32) / 255.0\n        labels = pd.get_dummies(df['label']).values\n        return images, labels\n    else:\n        images = df.values.reshape(-1, IMAGE_WIDTH, IMAGE_HEIGHT, 1).astype(np.float32) / 255.0\n        return images, None","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_generators():\n    train_images, train_labels = load_data_from_csv(TRAIN_CSV, is_train=True)\n    test_images, _ = load_data_from_csv(TEST_CSV, is_train=False)\n\n    # Data augmentation\n    datagen_args = {\n        'width_shift_range': 0.1,\n        'height_shift_range': 0.1,\n        'shear_range': 0.1,\n        'zoom_range': 0.15,\n        'rotation_range': 15,\n        'fill_mode': 'nearest'\n    }\n    \n    datagen_args['validation_split'] = 0.025\n\n    datagen = ImageDataGenerator(**datagen_args)\n    train_generator_args = {\n        'x': train_images,\n        'y': train_labels,\n        'batch_size': BATCH_SIZE,\n        'shuffle': True\n    }\n\n    train_generator_args['subset'] = 'training'\n\n    train_generator = datagen.flow(**train_generator_args)\n    validation_generator = datagen.flow(\n        x=train_images,\n        y=train_labels,\n        batch_size=BATCH_SIZE,\n        subset='validation'\n    )\n\n    return train_generator, validation_generator, test_images","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_model():\n\n    def count_conv2d_layers(model):\n        conv_layer_types = (tf.keras.layers.Conv2D, tf.keras.layers.DepthwiseConv2D)\n        return sum([1 for layer in model.layers if isinstance(layer, conv_layer_types)])\n\n    def EnhancedResidualBlock(inputs, out_channels, kernel_size, stride, expansion_factor, activation=tf.nn.relu):\n        shortcut = inputs\n\n        # Expansion phase\n        x = layers.Conv2D(inputs.shape[-1] * expansion_factor, 1, padding='valid', use_bias=False)(inputs)\n        x = layers.BatchNormalization()(x)\n        x = activation(x)\n\n        # Depthwise convolution\n        x = layers.DepthwiseConv2D(kernel_size, strides=stride, padding='same', use_bias=False)(x)\n        x = layers.BatchNormalization()(x)\n        x = activation(x)\n\n        # Dropout layer\n        x = layers.Dropout(DROPOUT_RATE / 2)(x)\n\n        # Output phase\n        x = layers.Conv2D(out_channels, 1, padding='same', use_bias=False)(x)\n        x = layers.BatchNormalization()(x)\n\n        # Dropout layer before the shortcut connection\n        x = layers.Dropout(DROPOUT_RATE / 2)(x)\n\n        # Adjust the shortcut path with conditional 1x1 convolution\n        if stride != 1 or inputs.shape[-1] != out_channels:\n            shortcut = layers.Conv2D(out_channels, (1, 1), strides=stride, padding='same', use_bias=False)(shortcut)\n            shortcut = layers.BatchNormalization()(shortcut)\n            if stride != 1:\n                shortcut = layers.MaxPooling2D(pool_size=(2, 2), strides=stride, padding='same')(shortcut)\n\n        x = layers.Add()([x, shortcut])  # Residual connection\n        return x\n    \n    input_tensor = layers.Input(shape=(IMAGE_WIDTH, IMAGE_HEIGHT, 1))\n\n    x = layers.Conv2D(128, 3, strides=1, padding='valid')(input_tensor)\n    x = layers.BatchNormalization()(x)\n    x = tf.nn.relu(x)\n    x = layers.Dropout(DROPOUT_RATE / 2)(x)\n\n    x = EnhancedResidualBlock(x, 128, 3, 1, 1)\n    x = EnhancedResidualBlock(x, 192, 3, 1, 1)\n    \n    # Flatten layer to reshape the tensor for dense layers\n    x = layers.Flatten()(x)\n\n    # Top layers\n    x = layers.Dense(256, activation=\"relu\")(x)\n    x = layers.Dropout(DROPOUT_RATE)(x)\n    x = layers.Dense(128, activation=\"relu\")(x)\n    x = layers.Dropout(DROPOUT_RATE)(x)\n    x = layers.Dense(10, activation=\"softmax\")(x)\n\n    model = tf.keras.models.Model(inputs=input_tensor, outputs=x)\n\n    print(\"Model depth:\", count_conv2d_layers(model))\n    print(\"Model Output Shape:\", model.output_shape)\n\n    return model","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def main():\n\n    train_generator, validation_generator, test_images = create_generators()\n    model = create_model()\n\n    # Define the ExponentialDecay learning rate scheduler\n    scheduler = ExponentialDecay(\n        initial_learning_rate=INITIAL_LR,\n        decay_steps=DECAY_STEPS,\n        decay_rate=DECAY_RATE,\n        staircase=True\n    )\n\n    # Pass the scheduler directly to the optimizer\n    optimizer = tfa.optimizers.AdamW(learning_rate=scheduler, weight_decay=WEIGHT_DECAY)\n\n    # Compile the model\n    model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])\n\n    model_checkpoint = ModelCheckpoint(\n        MODEL_PATH,\n        monitor='val_loss',\n        save_best_only=True,\n        mode='min',\n        verbose=1\n    )\n\n    model.fit(\n        train_generator,\n        epochs=EPOCHS,\n        validation_data=validation_generator,\n        callbacks=[model_checkpoint]\n    )\n\n    best_model = load_model(MODEL_PATH)\n    predictions = best_model.predict(test_images, batch_size=BATCH_SIZE)\n    class_predictions = np.argmax(predictions, axis=1)\n    submission = pd.DataFrame({\"ImageId\": range(1, len(class_predictions) + 1), \"Label\": class_predictions})\n    submission.to_csv(SUBMISSION_FILE, index=False)\n    \nif __name__ == \"__main__\":\n    main()","metadata":{},"execution_count":null,"outputs":[]}]}