{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\n# Load the data from the .csv files\ntrain = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ntest = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\n\n# Break the data into X_train, y_train, and X_test\nX_train = (train.iloc[:,1:].values).astype('float32') # all pixel values\ny_train = train.iloc[:,0].values.astype('int32') # only labels i.e targets digits\nX_test = test.values.astype('float32')\n\n# Reshape the data for training\nX_train_scaled = X_train.reshape(X_train.shape[0], 28, 28,1) / 255\nX_test_scaled = X_test.reshape(X_test.shape[0], 28, 28,1) / 255\n\n# Since softmax regression is being used, then convert the y to categorical \nfrom tensorflow.keras.utils import to_categorical\ny_train = to_categorical(y_train)\n\nX_train_mean = train.iloc[:, 1:].sum().sum() / (train.shape[0] * train.shape[1])\nX_test_mean = test.iloc[:, 1:].sum().sum() / (test.shape[0] * test.shape[1])\n# Using a normalized data makes the process faster and more accurate\nX_train_normalized = ((X_train - X_train_mean) / X_train_mean).reshape(train.shape[0], 28, 28,1)\nX_test_normalized = ((X_test - X_test_mean) / X_test_mean).reshape(test.shape[0], 28, 28,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Upgrading pip\n!/opt/conda/bin/python3.7 -m pip install --upgrade pip\n\n# Importing the needed modules\nimport tensorflow as tf\ntf.random.set_seed(123) # Set the see for all random stuff in the code\n\nfrom tensorflow.keras.models import Sequential # Keras model\nfrom tensorflow.keras.activations import relu\nfrom tensorflow.keras.callbacks import EarlyStopping, LearningRateScheduler, ReduceLROnPlateau\nfrom tensorflow.keras.layers import BatchNormalization, Add,Conv2D , MaxPooling2D, ReLU, Dropout, Flatten, Dense, InputLayer, Concatenate, SeparableConv2D\nfrom tensorflow.keras.optimizers import Adam, RMSprop\nfrom tensorflow.keras.losses import CategoricalCrossentropy\nfrom tensorflow.keras.regularizers import l1, l2\nfrom tensorflow.keras.initializers import TruncatedNormal, he_uniform, he_normal\n\n# Use functions from tensorflow_docs\n!pip install -q git+https://github.com/tensorflow/docs\nimport tensorflow_docs as tfdocs\nimport tensorflow_docs.plots\nimport tensorflow_docs.modeling","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Some global entities\ndef format_predictions(model, test_data=X_test_scaled):\n    \"\"\" Formats the predictions according to what is demanded in the submission. \"\"\"\n    preds = model.predict(test_data)\n\n    preds_test = []\n\n    for i in preds:\n        preds_test.append(np.argmax(i))\n    \n    return preds_test\n\n# Callback function to stop the training early given that there have been no sign of improve\nearly_stop = EarlyStopping(monitor='val_loss', \n                           patience=20, \n                           mode='min',\n                           restore_best_weights=True)\n\n# Decreasing the learning rate incrementally\ndef scheduler(epoch, lr):\n    if epoch < 20:\n        return lr \n    return lr * (0.9 ** ((epoch - 19) // 10))\n    \nrate_scheduler = LearningRateScheduler(scheduler)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Ideas:\n- Combining pretrained models with functional API\n- Use Data Augmentation"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conv_block1(In, std):\n    out = SeparableConv2D(16, (3, 3), kernel_initializer=TruncatedNormal(0, std, 1))(In)\n    out = BatchNormalization()(out)\n    out = SeparableConv2D(16, (1, 1), kernel_initializer=TruncatedNormal(0, std-1e-5, 1))(out)\n    \n    return out\n\ndef conv_block2(In, std):\n    out = SeparableConv2D(16, (3, 1), kernel_initializer=TruncatedNormal(0, std, 1), padding='same')(In)\n    out = SeparableConv2D(16, (1, 3), kernel_initializer=TruncatedNormal(0, std+1e-4, 1), padding='same')(out)\n    out = MaxPooling2D(2)(out)\n    \n    return out\n\ndef conv_block3(In, std):\n    out = SeparableConv2D(16, (5, 3), kernel_initializer=TruncatedNormal(0, std, 1), padding='same')(In)\n    out = BatchNormalization()(out)\n    out = SeparableConv2D(16, (3, 5), kernel_initializer=TruncatedNormal(0, std+1e-4, 1))(out)\n    out = MaxPooling2D(2)(out)\n    \n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Main input\nscaled_input = tf.keras.Input(shape=(28,28,1))\nnorm_in = BatchNormalization(name='norm_in')(scaled_input)\n\nout1 = []\n\nfor i in range(0, 8):\n    out1.append(conv_block1(norm_in, 1e3/10**(i)))\n\nout2 = []\nfor i in range(0, 8, 2):\n    out2.append(Add()([out1[i], out1[i+1]]))\n\nfor i in range(0, len(out2)):\n    out2[i] = conv_block2(out2[i], 1e4/10**(i))\n    out2[i] = conv_block3(out2[i], 10/10**(i // 2))\n\n\nflats = Concatenate()([Flatten()(out) for out in out2])\n\n# Classification\nd1 = Dense(1024, \n        kernel_initializer=TruncatedNormal(mean=10, stddev=2, seed=11), \n        bias_initializer=TruncatedNormal(mean=0, stddev=0.5, seed=11),\n        kernel_regularizer=l1(0.01)\n)(flats)\n\nd1 = BatchNormalization()(d1)\n\nd2 = Dense(1024, \n        kernel_initializer=TruncatedNormal(mean=1, stddev=0.02, seed=11), \n        bias_initializer=TruncatedNormal(mean=0, stddev=0.5, seed=11),\n        kernel_regularizer=l1(0.01)\n)(d1)\n\nd2 = BatchNormalization()(d2)\n\nfinal_prediction = Dense(10, activation='softmax')(d2)\n\nmodel = tf.keras.Model(inputs=[scaled_input], outputs=[final_prediction])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.count_params() / 1e6","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.utils.plot_model(model, show_shapes=1, show_layer_names=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(optimizer=Adam(0.0075),\n              loss=CategoricalCrossentropy(),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Changed the Dense three to ReLU 3\n# Chnaged the scheduler function\ndef scheduler(epoch, lr):\n    if epoch < 20:\n        return lr\n    elif lr > 5e-5:\n        return lr * (0.95 ** (epoch // 10 - 1))\n    \n    return lr\n    \nhist = model.fit(X_train_scaled, y_train, epochs= 2000, batch_size=128,\n          callbacks=[\n              LearningRateScheduler(scheduler), \n              tfdocs.modeling.EpochDots(),\n#               EarlyStopping(monitor='val_accuracy',\n#                            patience=400,\n#                            mode='min',\n#                            restore_best_weights=True\n#               ),\n              ReduceLROnPlateau(\n                            monitor='val_accuracy', factor=0.6, patience=20,\n                            verbose=1, mode='auto', min_delta=1e-4, cooldown=0, min_lr=1e-15\n              )\n          ], \n          verbose=True,\n          validation_split=0.33,\n          shuffle=True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission =  pd.DataFrame({\n        \"ImageId\": [i+1 for i in range(0, 28000)],\n        \"Label\": format_predictions(model)\n    })\n\nsubmission.to_csv('s.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"# def scheduler(epoch, lr):\n#     if epoch < 20:\n#         return lr\n#     elif epoch < 100:\n#         return lr * (0.8 ** (epoch / 80))\n    \n#     return lr * (0.8 ** (epoch / 50))\n\n# model.fit(X_train_scaled, y_train, epochs= 1500, batch_size=200,\n#           callbacks=[\n#               LearningRateScheduler(scheduler), \n#               tfdocs.modeling.EpochDots(),\n#               EarlyStopping(monitor='val_loss', \n#                            patience=100,\n#                            mode='min',\n#                            restore_best_weights=True\n#               ),\n#               ReduceLROnPlateau(\n#                             monitor='val_loss', factor=0.8, patience=20,\n#                             verbose=1, mode='auto', min_delta=1e-4, cooldown=0, min_lr=1e-6\n#               )\n#           ], \n#           verbose=True,\n#           validation_split=0.2,\n#           shuffle=True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from tf.keras import layers\n\n# inputs = keras.Input(shape=(32, 32, 3), name=\"img\")\n# x = layers.Conv2D(32, 3, activation=\"relu\")(inputs)\n# x = layers.Conv2D(64, 3, activation=\"relu\")(x)\n# block_1_output = layers.MaxPooling2D(3)(x)\n\n# x = layers.Conv2D(64, 3, activation=\"relu\", padding=\"same\")(block_1_output)\n# x = layers.Conv2D(64, 3, activation=\"relu\", padding=\"same\")(x)\n# block_2_output = layers.add([x, block_1_output])\n\n# x = layers.Conv2D(64, 3, activation=\"relu\", padding=\"same\")(block_2_output)\n# x = layers.Conv2D(64, 3, activation=\"relu\", padding=\"same\")(x)\n# block_3_output = layers.add([x, block_2_output])\n\n# x = layers.Conv2D(64, 3, activation=\"relu\")(block_3_output)\n# x = layers.GlobalAveragePooling2D()(x)\n# x = layers.Dense(256, activation=\"relu\")(x)\n# x = layers.Dropout(0.5)(x)\n# outputs = layers.Dense(10)(x)\n\n# model = keras.Model(inputs, outputs, name=\"toy_resnet\")\n# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# THIS BLOCK SHOULD BE RUN ONLY WHEN USING A TPU\n# detect and init the TPU\ntry :\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n\n    # instantiate a distribution strategy\n    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\nexcept:\n    print('No TPU being used!')\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# layers01 = [\n#     # Block 1\n#     Conv2D(32, (3, 3), kernel_initializer='he_normal', activation='relu',input_shape=(28, 28, 1)),\n#     Conv2D(32, (3, 3), kernel_initializer='he_normal', activation='relu'),\n    \n#     MaxPooling2D((2,2)),\n#     BatchNormalization(),\n    \n#     # Block 2\n#     Conv2D(128, (3, 3), kernel_initializer='he_normal', activation='relu'),\n#     Conv2D(128, (3, 3), kernel_initializer='he_normal', activation='relu'),\n    \n#     MaxPooling2D((2,2)),\n#     BatchNormalization(),\n    \n#     # Block 3\n#     Conv2D(512, (3, 3), kernel_initializer='he_normal', activation='relu'),\n    \n#     MaxPooling2D((2,2)),\n#     BatchNormalization(),\n    \n#     Flatten(),\n#     Dense(4000, activation='relu', kernel_initializer='he_normal', kernel_regularizer=l2(0.001)),\n#     Dense(4000, activation='relu', kernel_initializer='he_normal', kernel_regularizer=l2(0.001)),\n#     Dense(1000, activation='relu', kernel_initializer='he_normal', kernel_regularizer=l2(0.001)),\n    \n#     Dense(10, activation='softmax')\n# ]\n\n# # Model based on layer 1\n# model01 = Sequential(layers01)\n# model01.compile(optimizer=Adam(0.009),\n#                 loss=CategoricalCrossentropy(),\n#                 metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# \n# model01.fit(X_train_normalized, y_train, epochs=50, \n#           batch_size=128, validation_split=0.3,\n#           callbacks=[rate_scheduler, tfdocs.modeling.EpochDots(), early_stop], shuffle=True)\n\n# submission =  pd.DataFrame({\n#         \"ImageId\": [i+1 for i in range(0, 28000)],\n#         \"Label\": format_predictions(model01)\n#     })\n\n# submission.to_csv('submission01.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# def scheduler(epoch, lr):\n#     if epoch < 100:\n#         return lr\n#     elif epoch < 700:\n#         return lr * (0.99 ** (epoch + 1))\n#     elif epoch < 1000:\n#         return lr * (0.99 ** (epoch / 5))\n#     else:\n#         return lr * (0.99 ** (epoch / 50))\n    \n# rate_scheduler = LearningRateScheduler(scheduler)\n\n# model01.fit(X_train_normalized, y_train, epochs=100, \n#           batch_size=128, validation_split=0.3,\n#           callbacks=[rate_scheduler, tfdocs.modeling.EpochDots(), early_stop], shuffle=True)\n\n# submission =  pd.DataFrame({\n#         \"ImageId\": [i+1 for i in range(0, 28000)],\n#         \"Label\": format_predictions(model01)\n#     })\n\n# submission.to_csv('submission01_2.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# layers02 = [\n#     # Block 1\n#     Conv2D(64, (3, 3), kernel_initializer='he_normal', activation='relu',input_shape=(28, 28, 1)),\n#     Conv2D(64, (3, 3), kernel_initializer='he_normal', activation='relu', padding='same'),\n#     Conv2D(64, (3, 3), kernel_initializer='he_normal', activation='relu', padding='same'),\n#     BatchNormalization(),\n#     MaxPooling2D((2,2), padding='same'),\n    \n#     # Block 2\n#     Conv2D(256, (3, 3), kernel_initializer='he_normal', activation='relu', padding='same'),\n#     Conv2D(256, (3, 3), kernel_initializer='he_normal', activation='relu', padding='same'),\n#     BatchNormalization(),\n#     Conv2D(256, (3, 3), kernel_initializer='he_normal', activation='relu', padding='same'),\n#     Conv2D(256, (3, 3), kernel_initializer='he_normal', activation='relu', padding='same'),\n    \n#     BatchNormalization(),\n#     MaxPooling2D((2,2)),\n    \n#     # Block 3\n#     Conv2D(1024, (3, 3), kernel_initializer='he_normal', activation='relu'),\n#     BatchNormalization(),\n    \n#     Conv2D(1024, (3, 3), kernel_initializer='he_normal', activation='relu'),\n    \n#     BatchNormalization(),\n#     MaxPooling2D((2,2)),\n    \n#     # Classification base\n#     Flatten(),\n#     Dense(15, activation='relu', kernel_initializer='he_normal'),\n#     Dense(15, activation='relu', kernel_initializer='he_normal'),\n    \n#     BatchNormalization(),\n#     Dense(10, activation='softmax', kernel_initializer='he_normal', kernel_regularizer=l2(0.005)),\n    \n#     Dense(15, activation='relu', kernel_initializer='he_normal'),\n#     Dense(15, activation='relu', kernel_initializer='he_normal'),\n    \n#     BatchNormalization(),\n#     Dense(10, activation='softmax', kernel_initializer='he_normal', kernel_regularizer=l2(0.005)),\n    \n#     Dense(15, activation='relu', kernel_initializer='he_normal'),\n#     Dense(15, activation='relu', kernel_initializer='he_normal'),\n    \n#     BatchNormalization(),\n#     Dense(10, activation='softmax', kernel_initializer='he_normal', kernel_regularizer=l2(0.01))\n# ]\n\n# # Model based on layer 1\n# model02 = Sequential(layers02)\n# model02.compile(optimizer=Adam(0.01),\n#                 loss=CategoricalCrossentropy(),\n#                 metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.8, patience=3, min_lr=1e-10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model02.fit(X_train_scaled, y_train, epochs=500, \n#           batch_size=256, validation_split=0.2, verbose=1,\n#           callbacks=[reduce_lr, tfdocs.modeling.EpochDots(), early_stop], shuffle=True)\n\n# submission =  pd.DataFrame({\n#         \"ImageId\": [i+1 for i in range(0, 28000)],\n#         \"Label\": format_predictions(model02)\n#     })\n\n# submission.to_csv('submission02.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model02.fit(X_train_scaled, y_train, epochs=500, \n#           batch_size=256, validation_split=0.2, verbose=1,\n#           callbacks=[reduce_lr, tfdocs.modeling.EpochDots(), early_stop], shuffle=True)\n\n# submission =  pd.DataFrame({\n#         \"ImageId\": [i+1 for i in range(0, 28000)],\n#         \"Label\": format_predictions(model02)\n#     })\n\n# submission.to_csv('submission02.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}