{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Accuracy=99.75% using 25 Million Training Images!!\n#### Copied with minor edits from: https://www.kaggle.com/cdeotte/25-million-images-0-99757-mnist","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true}},{"cell_type":"code","source":"'''Install Libraries'''\n# Data Handling & Model Creation\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils.np_utils import to_categorical\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\n# Data augmentation\nfrom keras.preprocessing.image import ImageDataGenerator\n# Learning rate decay\nfrom keras.callbacks import LearningRateScheduler","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:13.797397Z","iopub.execute_input":"2021-12-25T10:16:13.797634Z","iopub.status.idle":"2021-12-25T10:16:14.734471Z","shell.execute_reply.started":"2021-12-25T10:16:13.797583Z","shell.execute_reply":"2021-12-25T10:16:14.733655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Kaggle's 42,000 training images","metadata":{"_uuid":"cd31c62c12088bfa6f2b26dcecc714182627c767"}},{"cell_type":"code","source":"# Load training and test data\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:14.738400Z","iopub.execute_input":"2021-12-25T10:16:14.740509Z","iopub.status.idle":"2021-12-25T10:16:21.808268Z","shell.execute_reply.started":"2021-12-25T10:16:14.740451Z","shell.execute_reply":"2021-12-25T10:16:21.807354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data for model\nY_train = train[\"label\"]\nX_train = train.drop(columns = [\"label\"])\nX_train = X_train / 255.0\nX_test = test / 255.0\nX_train = X_train.values.reshape(-1, 28, 28, 1)\nX_test = X_test.values.reshape(-1, 28, 28, 1)\nY_train = to_categorical(Y_train, num_classes = 10)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:21.809961Z","iopub.execute_input":"2021-12-25T10:16:21.810404Z","iopub.status.idle":"2021-12-25T10:16:22.579221Z","shell.execute_reply.started":"2021-12-25T10:16:21.810305Z","shell.execute_reply":"2021-12-25T10:16:22.578251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Vizualize training images\nimport matplotlib.pyplot as plt\nplt.figure(figsize = (10, 2))\nfor i in range(20):\n    plt.subplot(2, 10, i+1)\n    plt.imshow(X_train[i].reshape((28, 28)))\n    plt.axis('off')\nplt.subplots_adjust(wspace = 0.1, hspace = 0.1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:22.580623Z","iopub.execute_input":"2021-12-25T10:16:22.580894Z","iopub.status.idle":"2021-12-25T10:16:23.229483Z","shell.execute_reply.started":"2021-12-25T10:16:22.580848Z","shell.execute_reply":"2021-12-25T10:16:23.228636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Generate 25 million more images!!\nby randomly rotating, scaling, and shifting Kaggle's 42,000 images.","metadata":{"_uuid":"cfcb89d7d2dab632986e80d9f68d194c3c1c9e9f"}},{"cell_type":"code","source":"imagen = ImageDataGenerator(\n        rotation_range=10, \n        zoom_range = 0.10, \n        width_shift_range=0.1, \n        height_shift_range=0.1) ","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:23.233779Z","iopub.execute_input":"2021-12-25T10:16:23.234232Z","iopub.status.idle":"2021-12-25T10:16:23.242566Z","shell.execute_reply.started":"2021-12-25T10:16:23.234065Z","shell.execute_reply":"2021-12-25T10:16:23.241802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preview generated images\nX_train3 = X_train[9,].reshape((1, 28, 28, 1))\nY_train3 = Y_train[9,].reshape((1, 10))\nplt.figure(figsize = (15, 4.5))\nfor i in range(30):\n    plt.subplot(3, 10, i+1)\n    X_train2, Y_train2 = imagen.flow(X_train3, Y_train3).next()\n    plt.imshow(X_train2[0].reshape((28, 28)), cmap = plt.cm.binary)\n    plt.axis('off')\n    if i == 9: X_train3 = X_train[11,].reshape((1, 28, 28, 1))\n    if i == 19: X_train3 = X_train[18,].reshape((1, 28, 28, 1))\nplt.subplots_adjust(wspace = -0.1, hspace = -0.1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:23.244004Z","iopub.execute_input":"2021-12-25T10:16:23.244499Z","iopub.status.idle":"2021-12-25T10:16:24.346276Z","shell.execute_reply.started":"2021-12-25T10:16:23.244439Z","shell.execute_reply":"2021-12-25T10:16:24.345530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build 15 Convolutional Neural Networks!","metadata":{"_uuid":"9ea116cd3688cb26ac79b9fecc7309a1aebf3b63"}},{"cell_type":"code","source":"cnns = 15\nmodel = [0] *cnns\nfor j in range(cnns):\n    model[j] = Sequential()\n    \n    model[j].add(Conv2D(32, kernel_size = 3, activation = 'relu', input_shape = (28, 28, 1)))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 3, activation = 'relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(32, kernel_size = 5, strides = 2, padding = 'same', activation = 'relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n        \n    model[j].add(Conv2D(64, kernel_size = 3, activation = 'relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 3, activation = 'relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Conv2D(64, kernel_size = 5, strides = 2, padding = 'same', activation = 'relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Dropout(0.4))\n        \n    model[j].add(Conv2D(128, kernel_size = 4, activation = 'relu'))\n    model[j].add(BatchNormalization())\n    model[j].add(Flatten())\n    model[j].add(Dropout(0.4))\n    model[j].add(Dense(10, activation = 'softmax'))\n    \n    #Compile with adam optimizer and cross entropy cost\n    model[j].compile(optimizer = \"adam\", loss = \"categorical_crossentropy\", metrics = [\"accuracy\"])\n    ","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:24.347718Z","iopub.execute_input":"2021-12-25T10:16:24.347998Z","iopub.status.idle":"2021-12-25T10:16:38.357186Z","shell.execute_reply.started":"2021-12-25T10:16:24.347933Z","shell.execute_reply":"2021-12-25T10:16:38.356446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train 15 CNNs","metadata":{"_uuid":"e433661c7762b947c0fbfc4ad3f5e1d2e056312c"}},{"cell_type":"code","source":"# Iterative learning rate decrease\nannealer = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\n# Train networks\nhistory = [0] * cnns\nepochs = 45\nfor j in range(cnns):\n    X_train2, X_val2, Y_train2, Y_val2 = train_test_split(X_train, Y_train, test_size = 0.1)\n    history[j] = model[j].fit_generator(imagen.flow(X_train2, Y_train2, batch_size = 64), \n        epochs = epochs, steps_per_epoch = X_train2.shape[0]//64, \n        validation_data = (X_val2,Y_val2), callbacks=[annealer], verbose = 0)\n    print(\"CNN {0:d}: Epochs={1:d}, Training accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n        j+1,epochs,max(history[j].history['acc']),max(history[j].history['val_acc']) ))","metadata":{"execution":{"iopub.status.busy":"2021-12-25T10:16:38.359545Z","iopub.execute_input":"2021-12-25T10:16:38.360086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble 15 CNN predictions and submit","metadata":{"_uuid":"28b78b6502d2d1c993555725383f3e30728fa5be"}},{"cell_type":"code","source":"# Ensemble predictions\nresults = np.zeros( (X_test.shape[0],10) )\nfor j in range(cnns):\n    results = results + model[j].predict(X_test)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name = \"Label\")\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\nsubmission.to_csv(\"MNIST-15-CNNs-ENSEMBLE.csv\",index = False)  ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preview prediction set\nplt.figure(figsize = (20, 8))\nfor fig in range(50):\n    plt.subplot(5, 10, fig+1)\n    plt.imshow(X_test[fig].reshape((28, 28)), cmap = plt.cm.binary)\n    plt.title(\"prediction = %d\" % results[fig], y = 0.9)\n    plt.axis(\"off\")\nplt.subplots_adjust(wspace = 0.3, hspace = -0.1)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}