{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceType":"competition","sourceId":3004,"databundleVersionId":861823}],"dockerImageVersionId":438,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"* MNIST with Convoluted NN Keras\n* Train set is made of Kaggle 42k + Keras 60k = 102k\n* CV on train and then test on 10k from Keras\n* Final model is trained on train+test = 112k\n* CNN with adam ( vs rmsprop) and dropout against overfitting\n\n* It's a quick intro to the capabilities of CNN and Keras","metadata":{"_uuid":"aa621f76195962da026e03055468e0d5ae6fca66"}},{"cell_type":"code","source":"# IMPORT modules\n# TURN ON the GPU !!!\n# If importing dataset from outside - like the Keras dataset - Internet must be \"connected\"\n\nimport os\nfrom operator import itemgetter    \nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nget_ipython().magic(u'matplotlib inline')\nplt.style.use('ggplot')\n\nimport tensorflow as tf\n\nfrom keras import models, regularizers, layers, optimizers, losses, metrics\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.utils import np_utils, to_categorical\n\nfrom keras.datasets import mnist\n\nprint(os.getcwd())\nprint(\"Modules imported \\n\")\nprint(\"Files in current directory:\")\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\")) #check the files available in the directory","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LOAD DATA from Kaggle\n\ntrainRaw = pd.read_csv('../input/train.csv')\ntestRaw = pd.read_csv('../input/test.csv')","metadata":{"trusted":true,"_uuid":"dd21d55df8216b41b6f62a6df369246a5c860149","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = trainRaw.copy()\ntest_imagesKaggle = testRaw.copy()\ntrain_labelsKaggle = trainRaw['label']\n\nprint(\"train with Labels  \", train.shape)\nprint(\"train_labelsKaggle \", train_labelsKaggle.shape)\nprint(\"_\"*50)\ntrain.drop(['label'],axis=1, inplace=True)\ntrain_imagesKaggle = train\nprint(\"train_imagesKaggle without Labels \", train_imagesKaggle.shape)\nprint(\"_\"*50)\nprint(\"test_imagesKaggle  \", test_imagesKaggle.shape)","metadata":{"trusted":true,"_uuid":"62137573d06bff5fd352ef7b2f79b88a527ed97d","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# RESHAPE to 28 X 28 (Height, Width) which Kaggle has flattened in their file\n\ntrain4Display = np.array(train_imagesKaggle).reshape(42000,28,28)\ntest4Display = np.array(test_imagesKaggle).reshape(28000,28,28)\n\nz = 4056\n\nprint(\"train image\")\nprint(train_labelsKaggle[z])\ndigit = train4Display[z]\nplt.imshow(digit, cmap=plt.cm.binary)\nplt.show()\n\nprint(\"test image\")\ndigit = test4Display[z]\nplt.imshow(digit, cmap=plt.cm.binary)\nplt.show()\n","metadata":{"trusted":true,"_uuid":"08c17320b4df5efce432e399afdd0b58a9272493","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NORMALIZE / SCALE and Prep for CNN in terms of number dimensions expected\n\ntrain_imagesKaggle = train4Display.reshape(42000,28,28,1)\ntest_imagesKaggle = test4Display.reshape(28000,28,28,1)\n\ntrain_imagesKaggle = train_imagesKaggle.astype('float32') / 255\ntest_imagesKaggle = test_imagesKaggle.astype('float32') / 255\nprint(\"train_imagesKaggle \",train_imagesKaggle.shape)\nprint(\"test_imagesKaggle \", test_imagesKaggle.shape)\nprint(\"_\"*50)\n\n# ONE HOT ENCODER for the labels\ntrain_labelsKaggle = to_categorical(train_labelsKaggle)\nprint(\"train_labelsKaggle \",train_labelsKaggle.shape)","metadata":{"trusted":true,"_uuid":"c6f6371bd2c5400a5cb4a735ef8a1e61b2093d66","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load Data from Keras MNIST\n\n(train_imagesRaw, train_labelsRaw), (test_imagesRaw, test_labelsRaw) = mnist.load_data()","metadata":{"trusted":true,"_uuid":"efd02eff42dd1d81d3408b3e10d8aa9769c97197","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Normalize / Scale and One Hot encoder for the Keras dataset & Reshape for CNN\n\ntrain_imagesKeras = train_imagesRaw.copy()\ntrain_labelsKeras = train_labelsRaw.copy()\ntest_imagesKeras = test_imagesRaw.copy()\ntest_labelsKeras = test_labelsRaw.copy()\n\ntrain_imagesKeras = train_imagesKeras.reshape(60000,28,28,1)\ntest_imagesKeras = test_imagesKeras.reshape(10000,28,28,1)\n\nprint(\"train_imagesKeras \",train_imagesKeras.shape)\nprint(\"train_labelsKeras \",train_labelsKeras.shape)\nprint(\"test_imagesKeras \", test_imagesKeras.shape)\nprint(\"test_labelsKeras \", test_labelsKeras.shape)\n\n# NORMALIZE 0-255 to 0-1\ntrain_imagesKeras = train_imagesKeras.astype('float32') / 255\ntest_imagesKeras = test_imagesKeras.astype('float32') / 255\nprint(\"_\"*50)\n\n# ONE HOT ENCODER for the labels\ntrain_labelsKeras = to_categorical(train_labelsKeras)\ntest_labelsKeras = to_categorical(test_labelsKeras)\nprint(\"train_labelsKeras \",train_labelsKeras.shape)\nprint(\"test_labelsKeras \", test_labelsKeras.shape)","metadata":{"trusted":true,"_uuid":"ed31acc871c37d7386fe4aa7e791d25ae3217e5d","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CONCATENATE the training sets of Kaggle and Keras into final TRAIN and leave the test for CV\n\ntrain_images = np.concatenate((train_imagesKeras,train_imagesKaggle), axis=0)\nprint(\"new Concatenated train_images \", train_images.shape)\nprint(\"_\"*50)\n\ntrain_labels = np.concatenate((train_labelsKeras,train_labelsKaggle), axis=0)\nprint(\"new Concatenated train_labels \", train_labels.shape)","metadata":{"trusted":true,"_uuid":"76394ab509a426d75385ebf6e2f66c8c4c116edf","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initial model\n\nmodel = models.Sequential()\nmodel.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu'))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu'))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\nmodel.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy'])\nprint(model.summary())","metadata":{"trusted":true,"_uuid":"15ce294fa6aee518d0def20afa1b489cda189e3c","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initial fIT & Evaluate initial model\n\nnum_epochs = 30\nBatchSize = 2048\n\nmodel.fit(train_images, train_labels, epochs=num_epochs, batch_size=BatchSize)\ntest_loss, test_acc = model.evaluate(test_imagesKeras, test_labelsKeras)\nprint(\"_\"*80)\nprint(\"Accuracy on test \", test_acc)","metadata":{"trusted":true,"_uuid":"0361e316830d909e82a8e56fef09bca13f51bcd7","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# NN MODEL\n\ndef build_model():    \n    model = models.Sequential()\n    model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n    model.add(layers.Dropout(0.5))\n    model.add(layers.MaxPooling2D((2, 2)))\n    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n    model.add(layers.Dropout(0.5))\n    model.add(layers.MaxPooling2D((2, 2)))\n    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n    model.add(layers.Dropout(0.5))\n    model.add(layers.Flatten())\n    model.add(layers.Dense(64, activation='relu'))\n    model.add(layers.Dense(10, activation='softmax'))\n    model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model","metadata":{"trusted":true,"_uuid":"4231c20fc6b109202029e4743dd51adeffcefcb9","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check some test vs pred\n#TestNum = 10\n#for t in range(100, 100+TestNum):\n#    print(predictions[t])\n#    digit = test_imagesRaw[t]\n#    plt.imshow(digit, cmap=plt.cm.binary)\n#    plt.show()","metadata":{"trusted":true,"_uuid":"0b17d62c2cb73a3f159bee74ac7f9df41434993d","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CHECK ALL the ERRORS\n#TestNum = test_labels.shape[0]\n#ErrCount = 0\n#for t in range(TestNum):\n#        if test_labelsRaw[t] != predictions[t]:\n#            ErrCount = ErrCount +1\n#            #print(\"True \", test_labelsRaw[t], \"Predicted \",predictions[t])\n#            #digit = test_imagesRaw[t]\n#            #plt.imshow(digit, cmap=plt.cm.binary)\n#            #plt.show()\n\n#print(\"Errors \", ErrCount, \" out of \", TestNum, \" = \", 100 * ErrCount/TestNum)","metadata":{"trusted":true,"_uuid":"0de13e6aae7e7fe0a8c64720a76d79e485109763","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CROSS VALIDATION k-fold\ntrain_data = train_images\ntrain_targets = train_labels\nk = 4\nnum_val_samples = len(train_data) // k\nall_mae_histories = []\nfor i in range(k):\n    print('processing fold #', i)\n    val_data = train_data[i * num_val_samples: (i + 1) * num_val_samples]\n    val_targets = train_targets[i * num_val_samples: (i + 1) * num_val_samples]\n    partial_train_data = np.concatenate(\n    [train_data[:i * num_val_samples],\n    train_data[(i + 1) * num_val_samples:]],\n    axis=0)\n    partial_train_targets = np.concatenate(\n    [train_targets[:i * num_val_samples],\n    train_targets[(i + 1) * num_val_samples:]],\n    axis=0)\n    \n    model = build_model()\n    history = model.fit(partial_train_data, partial_train_targets,\n    validation_data=(val_data, val_targets),\n    epochs=num_epochs, batch_size=BatchSize, verbose=0)\n    \n    mae_history = history.history['acc']\n    all_mae_histories.append(mae_history)\n    \nprint(\"Done CV k-fold\")","metadata":{"trusted":true,"_uuid":"c476f0390000552510340ea839032218045c9fff","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LOSS Learning curves\n\nhistory_dict = history.history\nloss_values = history_dict['loss']\nval_loss_values = history_dict['val_loss']\nepochs = range(1, (len(history.history['acc']) + 1))\nplt.plot(epochs, loss_values, 'bo', label='Training loss')\nplt.plot(epochs, val_loss_values, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"trusted":true,"_uuid":"3eafffaf34507a6651bd18de5cd6cdd2ecf45cca","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ACCURACY Learning Curves\n\nhistory_dict = history.history\nloss_values = history_dict['acc']\nval_loss_values = history_dict['val_acc']\nepochs = range(1, (len(history.history['acc']) + 1))\nplt.plot(epochs, loss_values, 'bo', label='Training Acc')\nplt.plot(epochs, val_loss_values, 'b', label='Validation Acc')\nplt.title('Training and validation Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"trusted":true,"_uuid":"1703b47d39fd67b0e5ebdf28f1e5a830c7670e0c","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# CONCATENATE the train with test for FINAL FIT\n\ntrain_imagesFin = np.concatenate((train_images,test_imagesKeras), axis=0)\nprint(\"train_imagesFin \", train_imagesFin.shape)\nprint(\"_\"*50)\n\ntrain_labelsFin = np.concatenate((train_labels,test_labelsKeras), axis=0)\nprint(\"train_labelsFin \", train_labelsFin.shape)","metadata":{"trusted":true,"_uuid":"0da5dcfec2cb66b08d8a9789787a9e38eac30fa9","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# FINAL FIT according to the above charts\n\nmodel = build_model()\nmodel.fit(train_imagesFin, train_labelsFin, epochs=num_epochs, batch_size=BatchSize)","metadata":{"trusted":true,"_uuid":"653bd6719155ce7c0ca50bb247fc7bfdecadace2","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# PREDICT & ARGMAX to get the digit from the probability of softmax layer\n\nRawPred = model.predict(test_imagesKaggle)\npred = []\nnumTest = RawPred.shape[0]\nfor i in range(numTest):\n    pred.append(np.argmax(RawPred[i])) \npredictions = np.array(pred)  ","metadata":{"trusted":true,"collapsed":true,"_uuid":"273ab79269b12b940eb247a91d4d55d68cc6c3ad","jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# SUBMISSION\nsample_submission = pd.read_csv('../input/sample_submission.csv')\n#print(sample_submission.shape)\nresult=pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':predictions})\nresult.to_csv(\"submission.csv\",index=False)\nprint(result)","metadata":{"trusted":true,"_uuid":"35b495f34127d894d0bb1b94df9ec3ac881387ae","collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null}]}