{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# IMPORT modules\n# TURN ON the GPU !!!\n# If importing dataset from outside - like the Keras dataset - Internet must be \"connected\"\n\nimport os\nfrom operator import itemgetter    \nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.filterwarnings('ignore')\nget_ipython().magic(u'matplotlib inline')\nplt.style.use('ggplot')\n\nimport tensorflow as tf\n\nfrom keras import models, regularizers, layers, optimizers, losses, metrics\nfrom keras.models import Sequential\nfrom keras.layers import Dense\nfrom keras.utils import np_utils, to_categorical\n\nfrom keras.datasets import mnist\n\nprint(os.getcwd())\nprint(\"Modules imported \\n\")\nprint(\"Files in current directory:\")\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\")) #check the files available in the directory","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-28T10:01:04.177140Z","iopub.execute_input":"2023-03-28T10:01:04.177608Z","iopub.status.idle":"2023-03-28T10:01:04.311108Z","shell.execute_reply.started":"2023-03-28T10:01:04.177537Z","shell.execute_reply":"2023-03-28T10:01:04.308710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainRaw = pd.read_csv('../input/digit-recognizer/train.csv')\ntestRaw = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:04.314124Z","iopub.execute_input":"2023-03-28T10:01:04.315917Z","iopub.status.idle":"2023-03-28T10:01:08.921217Z","shell.execute_reply.started":"2023-03-28T10:01:04.315847Z","shell.execute_reply":"2023-03-28T10:01:08.919853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = trainRaw.copy()\ntest_imagesKaggle = testRaw.copy()\ntrain_labelsKaggle = trainRaw['label']\n\nprint(\"train with Labels  \", train.shape)\nprint(\"train_labelsKaggle \", train_labelsKaggle.shape)\nprint(\"_\"*50)\ntrain.drop(['label'],axis=1, inplace=True)\ntrain_imagesKaggle = train\nprint(\"train_imagesKaggle without Labels \", train_imagesKaggle.shape)\nprint(\"_\"*50)\nprint(\"test_imagesKaggle  \", test_imagesKaggle.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:08.923698Z","iopub.execute_input":"2023-03-28T10:01:08.924169Z","iopub.status.idle":"2023-03-28T10:01:09.188641Z","shell.execute_reply.started":"2023-03-28T10:01:08.924119Z","shell.execute_reply":"2023-03-28T10:01:09.187503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# RESHAPE to 28 X 28 (Height, Width) which Kaggle has flattened in their file\n\ntrain4Display = np.array(train_imagesKaggle).reshape(42000,28,28)\ntest4Display = np.array(test_imagesKaggle).reshape(28000,28,28)\n\nz = 4056\n\nprint(\"train image\")\nprint(train_labelsKaggle[z])\ndigit = train4Display[z]\nplt.imshow(digit, cmap=plt.cm.binary)\nplt.show()\n\nprint(\"test image\")\ndigit = test4Display[z]\nplt.imshow(digit, cmap=plt.cm.binary)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:09.191627Z","iopub.execute_input":"2023-03-28T10:01:09.192127Z","iopub.status.idle":"2023-03-28T10:01:09.804642Z","shell.execute_reply.started":"2023-03-28T10:01:09.192085Z","shell.execute_reply":"2023-03-28T10:01:09.803605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NORMALIZE / SCALE and Prep for CNN in terms of number dimensions expected\n\ntrain_imagesKaggle = train4Display.reshape(42000,28,28,1)\ntest_imagesKaggle = test4Display.reshape(28000,28,28,1)\n\ntrain_imagesKaggle = train_imagesKaggle.astype('float32') / 255\ntest_imagesKaggle = test_imagesKaggle.astype('float32') / 255\nprint(\"train_imagesKaggle \",train_imagesKaggle.shape)\nprint(\"test_imagesKaggle \", test_imagesKaggle.shape)\nprint(\"_\"*50)\n\n# ONE HOT ENCODER for the labels\ntrain_labelsKaggle = to_categorical(train_labelsKaggle)\nprint(\"train_labelsKaggle \",train_labelsKaggle.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:09.806381Z","iopub.execute_input":"2023-03-28T10:01:09.806730Z","iopub.status.idle":"2023-03-28T10:01:09.944471Z","shell.execute_reply.started":"2023-03-28T10:01:09.806701Z","shell.execute_reply":"2023-03-28T10:01:09.942306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load Data from Keras MNIST\n\n(train_imagesRaw, train_labelsRaw), (test_imagesRaw, test_labelsRaw) = mnist.load_data()","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:09.946582Z","iopub.execute_input":"2023-03-28T10:01:09.947574Z","iopub.status.idle":"2023-03-28T10:01:10.238337Z","shell.execute_reply.started":"2023-03-28T10:01:09.947524Z","shell.execute_reply":"2023-03-28T10:01:10.237170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Normalize / Scale and One Hot encoder for the Keras dataset & Reshape for CNN\n\ntrain_imagesKeras = train_imagesRaw.copy()\ntrain_labelsKeras = train_labelsRaw.copy()\ntest_imagesKeras = test_imagesRaw.copy()\ntest_labelsKeras = test_labelsRaw.copy()\n\ntrain_imagesKeras = train_imagesKeras.reshape(60000,28,28,1)\ntest_imagesKeras = test_imagesKeras.reshape(10000,28,28,1)\n\nprint(\"train_imagesKeras \",train_imagesKeras.shape)\nprint(\"train_labelsKeras \",train_labelsKeras.shape)\nprint(\"test_imagesKeras \", test_imagesKeras.shape)\nprint(\"test_labelsKeras \", test_labelsKeras.shape)\n\n# NORMALIZE 0-255 to 0-1\ntrain_imagesKeras = train_imagesKeras.astype('float32') / 255\ntest_imagesKeras = test_imagesKeras.astype('float32') / 255\nprint(\"_\"*50)\n\n# ONE HOT ENCODER for the labels\ntrain_labelsKeras = to_categorical(train_labelsKeras)\ntest_labelsKeras = to_categorical(test_labelsKeras)\nprint(\"train_labelsKeras \",train_labelsKeras.shape)\nprint(\"test_labelsKeras \", test_labelsKeras.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:10.239795Z","iopub.execute_input":"2023-03-28T10:01:10.240175Z","iopub.status.idle":"2023-03-28T10:01:10.341243Z","shell.execute_reply.started":"2023-03-28T10:01:10.240137Z","shell.execute_reply":"2023-03-28T10:01:10.339760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CONCATENATE the training sets of Kaggle and Keras into final TRAIN and leave the test for CV\n\ntrain_images = np.concatenate((train_imagesKeras,train_imagesKaggle), axis=0)\nprint(\"new Concatenated train_images \", train_images.shape)\nprint(\"_\"*50)\n\ntrain_labels = np.concatenate((train_labelsKeras,train_labelsKaggle), axis=0)\nprint(\"new Concatenated train_labels \", train_labels.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:10.343396Z","iopub.execute_input":"2023-03-28T10:01:10.344276Z","iopub.status.idle":"2023-03-28T10:01:10.493922Z","shell.execute_reply.started":"2023-03-28T10:01:10.344230Z","shell.execute_reply":"2023-03-28T10:01:10.492578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initial model\n\nmodel = models.Sequential()\nmodel.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu'))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.MaxPooling2D((2, 2)))\nmodel.add(layers.Conv2D(64, (3, 3), activation='relu'))\nmodel.add(layers.Dropout(0.5))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(64, activation='relu'))\nmodel.add(layers.Dense(10, activation='softmax'))\nmodel.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy'])\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:10.495852Z","iopub.execute_input":"2023-03-28T10:01:10.496680Z","iopub.status.idle":"2023-03-28T10:01:10.626955Z","shell.execute_reply.started":"2023-03-28T10:01:10.496635Z","shell.execute_reply":"2023-03-28T10:01:10.626050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initial fIT & Evaluate initial model\n\nnum_epochs = 30\nBatchSize = 2048\n\nmodel.fit(train_images, train_labels, epochs=num_epochs, batch_size=BatchSize)\ntest_loss, test_acc = model.evaluate(test_imagesKeras, test_labelsKeras)\nprint(\"_\"*80)\nprint(\"Accuracy on test \", test_acc)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:01:10.630971Z","iopub.execute_input":"2023-03-28T10:01:10.631880Z","iopub.status.idle":"2023-03-28T10:02:03.324903Z","shell.execute_reply.started":"2023-03-28T10:01:10.631836Z","shell.execute_reply":"2023-03-28T10:02:03.323684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# NN MODEL\n\ndef build_model():    \n    model = models.Sequential()\n    model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n    model.add(layers.Dropout(0.5))\n    model.add(layers.MaxPooling2D((2, 2)))\n    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n    model.add(layers.Dropout(0.5))\n    model.add(layers.MaxPooling2D((2, 2)))\n    model.add(layers.Conv2D(64, (3, 3), activation='relu'))\n    model.add(layers.Dropout(0.5))\n    model.add(layers.Flatten())\n    model.add(layers.Dense(64, activation='relu'))\n    model.add(layers.Dense(10, activation='softmax'))\n    model.compile(optimizer='rmsprop',loss='categorical_crossentropy',metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:02:03.326950Z","iopub.execute_input":"2023-03-28T10:02:03.327349Z","iopub.status.idle":"2023-03-28T10:02:03.337178Z","shell.execute_reply.started":"2023-03-28T10:02:03.327308Z","shell.execute_reply":"2023-03-28T10:02:03.335777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check some test vs pred\n#TestNum = 10\n#for t in range(100, 100+TestNum):\n#    print(predictions[t])\n#    digit = test_imagesRaw[t]\n#    plt.imshow(digit, cmap=plt.cm.binary)\n#    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:02:03.339394Z","iopub.execute_input":"2023-03-28T10:02:03.339909Z","iopub.status.idle":"2023-03-28T10:02:03.348576Z","shell.execute_reply.started":"2023-03-28T10:02:03.339868Z","shell.execute_reply":"2023-03-28T10:02:03.346589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CHECK ALL the ERRORS\n#TestNum = test_labels.shape[0]\n#ErrCount = 0\n#for t in range(TestNum):\n#        if test_labelsRaw[t] != predictions[t]:\n#            ErrCount = ErrCount +1\n#            #print(\"True \", test_labelsRaw[t], \"Predicted \",predictions[t])\n#            #digit = test_imagesRaw[t]\n#            #plt.imshow(digit, cmap=plt.cm.binary)\n#            #plt.show()\n\n#print(\"Errors \", ErrCount, \" out of \", TestNum, \" = \", 100 * ErrCount/TestNum)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:02:03.350137Z","iopub.execute_input":"2023-03-28T10:02:03.350934Z","iopub.status.idle":"2023-03-28T10:02:03.359794Z","shell.execute_reply.started":"2023-03-28T10:02:03.350865Z","shell.execute_reply":"2023-03-28T10:02:03.358517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CROSS VALIDATION k-fold\ntrain_data = train_images\ntrain_targets = train_labels\nk = 4\nnum_val_samples = len(train_data) // k\nall_mae_histories = []\nfor i in range(k):\n    print('processing fold #', i)\n    val_data = train_data[i * num_val_samples: (i + 1) * num_val_samples]\n    val_targets = train_targets[i * num_val_samples: (i + 1) * num_val_samples]\n    partial_train_data = np.concatenate(\n    [train_data[:i * num_val_samples],\n    train_data[(i + 1) * num_val_samples:]],\n    axis=0)\n    partial_train_targets = np.concatenate(\n    [train_targets[:i * num_val_samples],\n    train_targets[(i + 1) * num_val_samples:]],\n    axis=0)\n    \n    model = build_model()\n    history = model.fit(partial_train_data, partial_train_targets,\n    validation_data=(val_data, val_targets),\n    epochs=num_epochs, batch_size=BatchSize, verbose=0)\n    \n    mae_history = history.history['val_loss']  #'acc'\n    all_mae_histories.append(mae_history)\n    \nprint(\"Done CV k-fold\")","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:02:03.361670Z","iopub.execute_input":"2023-03-28T10:02:03.362235Z","iopub.status.idle":"2023-03-28T10:04:50.758643Z","shell.execute_reply.started":"2023-03-28T10:02:03.362108Z","shell.execute_reply":"2023-03-28T10:04:50.757310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# LOSS Learning curves\n\nhistory_dict = history.history\nloss_values = history_dict['loss']\nval_loss_values = history_dict['val_loss']\nepochs = range(1, (len(history.history['val_loss']) + 1)) #'acc'\nplt.plot(epochs, loss_values, 'bo', label='Training loss')\nplt.plot(epochs, val_loss_values, 'b', label='Validation loss')\nplt.title('Training and validation loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:04:50.760286Z","iopub.execute_input":"2023-03-28T10:04:50.761059Z","iopub.status.idle":"2023-03-28T10:04:51.020694Z","shell.execute_reply.started":"2023-03-28T10:04:50.761013Z","shell.execute_reply":"2023-03-28T10:04:51.019618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ACCURACY Learning Curves\n\nhistory_dict = history.history\nloss_values = history_dict['val_loss'] #'acc'\nval_loss_values = history_dict['val_loss'] #'val_acc'\nepochs = range(1, (len(history.history['val_loss']) + 1)) #'acc'\nplt.plot(epochs, loss_values, 'bo', label='Training Acc')\nplt.plot(epochs, val_loss_values, 'b', label='Validation Acc')\nplt.title('Training and validation Accuracy')\nplt.xlabel('Epochs')\nplt.ylabel('Accuracy')\nplt.legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:04:51.022272Z","iopub.execute_input":"2023-03-28T10:04:51.022791Z","iopub.status.idle":"2023-03-28T10:04:51.278786Z","shell.execute_reply.started":"2023-03-28T10:04:51.022748Z","shell.execute_reply":"2023-03-28T10:04:51.277559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# CONCATENATE the train with test for FINAL FIT\n\ntrain_imagesFin = np.concatenate((train_images,test_imagesKeras), axis=0)\nprint(\"train_imagesFin \", train_imagesFin.shape)\nprint(\"_\"*50)\n\ntrain_labelsFin = np.concatenate((train_labels,test_labelsKeras), axis=0)\nprint(\"train_labelsFin \", train_labelsFin.shape)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:05:42.935384Z","iopub.execute_input":"2023-03-28T10:05:42.936008Z","iopub.status.idle":"2023-03-28T10:05:43.098798Z","shell.execute_reply.started":"2023-03-28T10:05:42.935960Z","shell.execute_reply":"2023-03-28T10:05:43.097636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# FINAL FIT according to the above charts\n\nmodel = build_model()\nmodel.fit(train_imagesFin, train_labelsFin, epochs=num_epochs, batch_size=BatchSize)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:05:59.905539Z","iopub.execute_input":"2023-03-28T10:05:59.906288Z","iopub.status.idle":"2023-03-28T10:07:23.434443Z","shell.execute_reply.started":"2023-03-28T10:05:59.906249Z","shell.execute_reply":"2023-03-28T10:07:23.433274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PREDICT & ARGMAX to get the digit from the probability of softmax layer\n\nRawPred = model.predict(test_imagesKaggle)\npred = []\nnumTest = RawPred.shape[0]\nfor i in range(numTest):\n    pred.append(np.argmax(RawPred[i])) \npredictions = np.array(pred)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:10:53.664527Z","iopub.execute_input":"2023-03-28T10:10:53.665179Z","iopub.status.idle":"2023-03-28T10:10:56.236194Z","shell.execute_reply.started":"2023-03-28T10:10:53.665140Z","shell.execute_reply":"2023-03-28T10:10:56.234709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SUBMISSION\nsample_submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')\n#print(sample_submission.shape)\nresult=pd.DataFrame({'ImageId':sample_submission.ImageId, 'Label':predictions})\nresult.to_csv(\"submission.csv\",index=False)\nprint(result)","metadata":{"execution":{"iopub.status.busy":"2023-03-28T10:11:52.012151Z","iopub.execute_input":"2023-03-28T10:11:52.012986Z","iopub.status.idle":"2023-03-28T10:11:52.062209Z","shell.execute_reply.started":"2023-03-28T10:11:52.012947Z","shell.execute_reply":"2023-03-28T10:11:52.060987Z"},"trusted":true},"execution_count":null,"outputs":[]}]}