{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nsample_submission = pd.read_csv(\"../input/digit-recognizer/sample_submission.csv\")\ntest = pd.read_csv(\"../input/digit-recognizer/test.csv\")\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Lets just look at some of the training set\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n#Prep training data\nx       = np.ndarray(shape = (train.values.shape[0], 28, 28, 1))\nx_label = np.ndarray(shape = train.values.shape[0])\ni       = 0\nfor img in train.values:\n    x[i, :, :, 0] = np.reshape(img[1:], (28, 28))/255 #put data on 0 to 1\n    x_label[i]    = img[0]\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f = plt.figure(figsize=(10,10))\nfor j in range(64):\n\t# define subplot\n\tplt.subplot(8, 8, j + 1)\n\timage = x[j, :, :, 0]\n\t# plot raw pixel data\n\tplt.imshow(image)\n# show the figure\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#We're going to increase the test set a factor of 10 by doing some img augmentation\n#Flipping isn't good because that could easily take 9 -> 6 and the end test will be \n# numbers written left to right\n# It looks like rotations (below say 45 degrees), shears, and perhaps zooms \n# could work well to create new entries","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#rotation\nfrom keras.preprocessing.image import ImageDataGenerator\nrot_gen = ImageDataGenerator(rotation_range = 25, width_shift_range = 3, height_shift_range = 3,\n                            brightness_range = [0.65, 1.5], shear_range = 15, zoom_range = 0.15)\nx_aug   = rot_gen.flow(x, batch_size=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f = plt.figure(figsize=(10,10))\nfor j in range(64):\n\t# define subplot\n\tplt.subplot(8, 8, j + 1)\n\timage = x_aug.next()[0,:,:,0]\n\t# plot raw pixel data\n\tplt.imshow(image)\n# show the figure\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Ok, this looks like good augmentation here, lets now split the original dataset \n## and augment to get a larger training set","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train, x_test, y_train, y_test = train_test_split(x, x_label, test_size=0.2, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import keras as ks\nY_train = ks.utils.to_categorical(y_train)\nY_test  = ks.utils.to_categorical(y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new = rot_gen.flow(x_train, Y_train, batch_size = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"thin = new.next()\nthing = thin[0]\nlbl = thin[1]\nfor j in range(10):\n\t# define subplot\n\tplt.subplot(2, 5, j + 1)\n\timage = thing[j,:,:,0]\n\t# plot raw pixel data\n\tplt.imshow(image)\n# show the figure\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(10):\n    print(np.argmax(lbl[i]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"aug = rot_gen.flow(x_train, Y_train, batch_size = 40)\nx_train_aug = np.ndarray(shape = (x_train.shape[0]*41, 28, 28, 1))\nY_train_aug = np.ndarray(shape = (Y_train.shape[0]*41, 10))\nx_train_aug[: x_train.shape[0], :, :, :] = x_train\nY_train_aug[: x_train.shape[0], :] = Y_train\nfor i in range(x_train.shape[0] - 1):\n    nn = aug.next()\n    x_train_aug[x_train.shape[0] + i*40: x_train.shape[0] + (i + 1)*40, :, :, :] = nn[0]/255\n    Y_train_aug[x_train.shape[0] + i*40: x_train.shape[0] + (i + 1)*40, :]  = nn[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Dense, Flatten, Conv2D, MaxPooling2D, AveragePooling2D, Input, Dropout\nfrom keras.callbacks import History, ModelCheckpoint, Callback\nfrom keras.utils.vis_utils import plot_model\nfrom keras.models import Model\nfrom keras.optimizers import SGD\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\n\nclass makestats(Callback):\n    def on_epoch_end(self, batch, logs=None):\n        if batch % 5 == 0 and batch != 0:\n            \n            YY_test = np.argmax(Y_test, axis=1)\n            y_pred = model.predict(x_test)\n            print(classification_report(YY_test, np.argmax(y_pred, axis = 1)))\n            print(confusion_matrix((YY_test), y_pred.argmax(axis=1)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    inputA = Input(shape = (28, 28, 1))\n    x = Conv2D(32, kernel_size=(5, 5), strides=(1),\n                 activation='relu',\n                 padding = 'same')(inputA)\n    x = MaxPooling2D(pool_size=(2, 2), strides=(2))(x)\n    x = Dropout(0.3)(x)\n    x = Conv2D(64, (5, 5), activation='relu')(x)\n    x = MaxPooling2D(pool_size=(2, 2))(x)\n    x = Dropout(0.3)(x)\n    x = Flatten()(x)\n    x = Dense(1000, activation='relu')(x)\n    x = Dropout(0.3)(x)\n    z = Dense(500, activation=\"relu\")(x)\n    x = Dropout(0.3)(x)\n    z = Dense(10, activation=\"softmax\")(z)\n    model = Model(inputs=inputA, outputs=z)\n    epochs = 40\n    lrate = 0.0003\n    decay = lrate/epochs\n    sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)\n    model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = History()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(x_train_aug, Y_train_aug, validation_data=(x_test, Y_test), verbose = 1,\n          epochs=epochs, batch_size=32, callbacks = [history, makestats()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"        plt.plot(history.history['accuracy'])\n        plt.plot(history.history['val_accuracy'])\n        plt.title('model accuracy')\n        plt.ylabel('accuracy')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n        plt.close()\n        # summarize history for loss\n        plt.plot(history.history['loss'])\n        plt.plot(history.history['val_loss'])\n        plt.title('model loss')\n        plt.ylabel('loss')\n        plt.xlabel('epoch')\n        plt.legend(['train', 'test'], loc='upper left')\n        plt.show()\n        plt.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Seems to be leveled out, lets use this model to make a prediction to submit\nTEST       = np.ndarray(shape = (test.values.shape[0], 28, 28, 1))\ni          = 0\nfor img in test.values:\n    TEST[i, :, :, 0] = np.reshape(img, (28, 28))/255 #put data on 0 to 1\n    i += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(TEST)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor yy in y_pred:\n    preds.append(np.argmax(yy))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_file = open(\"predictions.csv\", \"w\")\nout_file.write(\"ImageId,Label\\n\")\nfor i in range(len(preds)):\n    out_file.write(str(i+1) + \",\" + str(int(preds[i])) + \"\\n\")\nout_file.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}