{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":11307,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# LOAD LIBRARIES\nimport pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import LearningRateScheduler","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-10-08T10:09:11.818549Z","iopub.execute_input":"2024-10-08T10:09:11.818856Z","iopub.status.idle":"2024-10-08T10:09:12.803059Z","shell.execute_reply.started":"2024-10-08T10:09:11.818793Z","shell.execute_reply":"2024-10-08T10:09:12.802380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOAD THE DATA\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","metadata":{"_uuid":"d71b3fa2b10620dc8870352fc18d4548f824a88a","execution":{"iopub.status.busy":"2024-10-08T10:09:12.804512Z","iopub.execute_input":"2024-10-08T10:09:12.804765Z","iopub.status.idle":"2024-10-08T10:09:21.097107Z","shell.execute_reply.started":"2024-10-08T10:09:12.804716Z","shell.execute_reply":"2024-10-08T10:09:21.096071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PREPARE DATA FOR NEURAL NETWORK\nY_train = train[\"label\"]\nX_train = train.drop(labels = [\"label\"],axis = 1)\nX_train = X_train / 255.0\nX_test = test / 255.0\nX_train = X_train.values.reshape(-1,28,28,1)\nX_test = X_test.values.reshape(-1,28,28,1)\nY_train = to_categorical(Y_train, num_classes = 10)","metadata":{"_kg_hide-input":true,"_uuid":"b3c56055d1ba56d28d982f9647c33439c46753ff","execution":{"iopub.status.busy":"2024-10-08T10:09:21.098380Z","iopub.execute_input":"2024-10-08T10:09:21.098683Z","iopub.status.idle":"2024-10-08T10:09:21.845318Z","shell.execute_reply.started":"2024-10-08T10:09:21.098627Z","shell.execute_reply":"2024-10-08T10:09:21.844229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n# PREVIEW IMAGES\nplt.figure(figsize=(15,4.5))\nfor i in range(30):  \n    plt.subplot(3, 10, i+1)\n    plt.imshow(X_train[i].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","metadata":{"_kg_hide-input":true,"_uuid":"b95ca2c1e71cb5457684eff3c35bb8d68b4a0f97","execution":{"iopub.status.busy":"2024-10-08T10:09:21.846865Z","iopub.execute_input":"2024-10-08T10:09:21.847148Z","iopub.status.idle":"2024-10-08T10:09:23.177292Z","shell.execute_reply.started":"2024-10-08T10:09:21.847084Z","shell.execute_reply":"2024-10-08T10:09:23.176641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CREATE MORE IMAGES VIA DATA AUGMENTATION\ndatagen = ImageDataGenerator(\n        rotation_range=10,  \n        zoom_range = 0.10,  \n        width_shift_range=0.1, \n        height_shift_range=0.1)","metadata":{"_uuid":"1e61e07d14b9b012748fdaac9eaf02e5263a475e","execution":{"iopub.status.busy":"2024-10-08T10:09:23.178593Z","iopub.execute_input":"2024-10-08T10:09:23.178826Z","iopub.status.idle":"2024-10-08T10:09:23.183052Z","shell.execute_reply.started":"2024-10-08T10:09:23.178787Z","shell.execute_reply":"2024-10-08T10:09:23.182290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PREVIEW AUGMENTED IMAGES\nX_train3 = X_train[9,].reshape((1,28,28,1))\nY_train3 = Y_train[9,].reshape((1,10))\nplt.figure(figsize=(15,4.5))\nfor i in range(30):  \n    plt.subplot(3, 10, i+1)\n    X_train2, Y_train2 = datagen.flow(X_train3,Y_train3).next()\n    plt.imshow(X_train2[0].reshape((28,28)),cmap=plt.cm.binary)\n    plt.axis('off')\n    if i==9: X_train3 = X_train[11,].reshape((1,28,28,1))\n    if i==19: X_train3 = X_train[18,].reshape((1,28,28,1))\nplt.subplots_adjust(wspace=-0.1, hspace=-0.1)\nplt.show()","metadata":{"_kg_hide-input":true,"_uuid":"fcf6daaae4424b95978856d7e75271c97b971c71","execution":{"iopub.status.busy":"2024-10-08T10:09:23.184602Z","iopub.execute_input":"2024-10-08T10:09:23.184902Z","iopub.status.idle":"2024-10-08T10:09:24.408070Z","shell.execute_reply.started":"2024-10-08T10:09:23.184845Z","shell.execute_reply":"2024-10-08T10:09:24.407274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# BUILD CONVOLUTIONAL NEURAL NETWORKS\nnets = 15\nmodel = [0] *nets\nfor j in range(nets):\n    model[j] = Sequential()\n\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = (28, 28, 1)))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n\n    model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Flatten())\n    model[j].add(Dropout(0.4))\n    model[j].add(Dense(10, activation='softmax'))\n\n    # COMPILE WITH ADAM OPTIMIZER AND CROSS ENTROPY COST\n    model[j].compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","metadata":{"_uuid":"f6703f3f53c659e95579122755454899d842722a","execution":{"iopub.status.busy":"2024-10-08T10:09:24.409543Z","iopub.execute_input":"2024-10-08T10:09:24.409843Z","iopub.status.idle":"2024-10-08T10:09:40.679588Z","shell.execute_reply.started":"2024-10-08T10:09:24.409786Z","shell.execute_reply":"2024-10-08T10:09:40.678837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# DECREASE LEARNING RATE EACH EPOCH\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\n# TRAIN NETWORKS\nhistory = [0] * nets\nepochs = 45\nfor j in range(nets):\n    X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)\n    history[j] = model[j].fit_generator(datagen.flow(X_train2,Y_train2, batch_size=64),\n        epochs = epochs, steps_per_epoch = X_train2.shape[0]//64,  \n        validation_data = (X_val2,Y_val2), callbacks=[annealer], verbose=0)\n    print(\"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n        j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc']) ))","metadata":{"_uuid":"9f1dd8a54aa0fab8530c0095f7d4c4b35984ea6d","execution":{"iopub.status.busy":"2024-10-08T10:09:40.680973Z","iopub.execute_input":"2024-10-08T10:09:40.681398Z","iopub.status.idle":"2024-10-08T10:41:27.678440Z","shell.execute_reply.started":"2024-10-08T10:09:40.681239Z","shell.execute_reply":"2024-10-08T10:41:27.623687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble 15 CNN predictions and submit","metadata":{"_uuid":"28b78b6502d2d1c993555725383f3e30728fa5be"}},{"cell_type":"code","source":"# ENSEMBLE PREDICTIONS AND SUBMIT\nresults = np.zeros( (X_test.shape[0],10) ) \nfor j in range(nets):\n    results = results + model[j].predict(X_test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"MNIST-CNN-ENSEMBLE.csv\",index=False)","metadata":{"_uuid":"6e4e01ffe692c34c555bdbf5d606611f9a128b9c","execution":{"iopub.status.busy":"2024-10-08T10:41:27.624709Z","iopub.status.idle":"2024-10-08T10:41:27.625293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PREVIEW PREDICTIONS\nplt.figure(figsize=(15,6))\nfor i in range(40):  \n    plt.subplot(4, 10, i+1)\n    plt.imshow(X_test[i].reshape((28,28)),cmap=plt.cm.binary)\n    plt.title(\"predict=%d\" % results[i],y=0.9)\n    plt.axis('off')\nplt.subplots_adjust(wspace=0.3, hspace=-0.1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-08T10:41:27.626186Z","iopub.status.idle":"2024-10-08T10:41:27.626892Z"},"trusted":true},"execution_count":null,"outputs":[]}]}