{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#Mostly taken from the Deep Learning kaggle exercise, Deep Learning from scratch\n#https://www.kaggle.com/alexanderdbooth/exercise-deep-learning-from-scratch\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow import keras\n\n#image size\nimg_rows, img_cols = 28, 28\n\n#ten possible digits (classes), 0,1,2,3,4,5,6,7,8,9\nnum_classes = 10\n\n#training data has x and y (labels)\ndef prep_data(raw):\n    y = raw[:, 0]\n    out_y = keras.utils.to_categorical(y, num_classes)\n    \n    x = raw[:,1:]\n    num_images = raw.shape[0]\n    out_x = x.reshape(num_images, img_rows, img_cols, 1)\n    out_x = out_x / 255\n    return out_x, out_y\n\n#test data only has x\ndef prep_data_test(raw_test): #only has x, no labels\n    x = raw_test[:,0:]\n    num_images = raw_test.shape[0]\n    out_x = x.reshape(num_images, img_rows, img_cols, 1)\n    out_x = out_x / 255\n    return out_x\n\nnumber_file = \"../input/train.csv\"\nnumber_data = np.loadtxt(number_file, skiprows=1, delimiter=',')\nx, y = prep_data(number_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.shape(x) #42,000 training images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#read and prep test data\nnumber_file_test = \"../input/test.csv\"\nnumber_data_test = np.loadtxt(number_file_test, skiprows=1, delimiter=',')\nx_test = prep_data_test(number_data_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.shape(x_test) #28,000 test images","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n#view a couple of the training images\nfor i in range(1,13):\n    plt.subplot(3,4,i)\n    plt.imshow(x[i-1].reshape([28,28]),cmap=\"gray\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Flatten, Conv2D\n\n#build the model\nnumber_model = Sequential()\nnumber_model.add(Conv2D(12, kernel_size=(3, 3),\n                 activation='relu',\n                 input_shape=(img_rows, img_cols, 1))) #activation layer\n\n#additional learning layers\nnumber_model.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\nnumber_model.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\nnumber_model.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\nnumber_model.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\n\n#final prediction layers\nnumber_model.add(Flatten())\nnumber_model.add(Dense(100, activation='relu'))\nnumber_model.add(Dense(num_classes, activation='softmax'))\n\n#compile the model\nnumber_model.compile(loss=keras.losses.categorical_crossentropy,\n              optimizer='adam',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#initial fit with validation\nnumber_model.fit(x, y,\n          batch_size=100,\n          epochs=3,\n          validation_split = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get predictions\npreds_test = number_model.predict(x_test)\n#the model returns a list of probabilities for each number the image could be. \n#argmax is a hack to get the most likely number\nrealPreds = [np.argmax(x) for x in preds_test]\nrealPreds[0:12]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print a few of the test numbers to look at our predictions\nfor i in range(1,13):\n    plt.subplot(3,4,i) #sub plots have to start at 1\n    plt.imshow(x_test[i-1].reshape([28,28]),cmap=\"gray\")\n    \n#looks like we are missing the fourth number","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Retrain using data augmentation for fun (again, Deep Learning Kaggle Course exercise)\n#https://www.kaggle.com/alexanderdbooth/exercise-data-augmentation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#have to manually get our validation data when dealing with generators\nX_train, X_val, Y_train, Y_val = train_test_split(x, y, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#build the model, same as above\nnumber_model_aug = Sequential()\nnumber_model_aug.add(Conv2D(12, kernel_size=(3, 3),\n                 activation='relu',\n                 input_shape=(img_rows, img_cols, 1))) #activation layer\n\n#additional learning layers\nnumber_model_aug.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\nnumber_model_aug.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\nnumber_model_aug.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\nnumber_model_aug.add(Conv2D(20, kernel_size=(3, 3), activation='relu'))\n\n#final prediction layers\nnumber_model_aug.add(Flatten())\nnumber_model_aug.add(Dense(100, activation='relu'))\nnumber_model_aug.add(Dense(num_classes, activation='softmax'))\n\n#compile the model\nnumber_model_aug.compile(loss=keras.losses.categorical_crossentropy,\n              optimizer='adam',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.preprocessing.image import ImageDataGenerator\ndatagen = ImageDataGenerator(featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\ndatagen.fit(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"number_model_aug.fit_generator(datagen.flow(X_train,Y_train),\n                              epochs = 15, validation_data = (X_val,Y_val), steps_per_epoch=20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get predictions\npreds_test_aug = number_model_aug.predict(x_test)\n#the model returns a list of probabilities for each number the image could be. \n#argmax is a hack to get the most likely number\nreal_preds_aug = [np.argmax(x) for x in preds_test_aug]\nreal_preds_aug[0:12]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print a few of the test numbers to look at our predictions\nfor i in range(1,13):\n    plt.subplot(3,4,i) #sub plots have to start at 1\n    plt.imshow(x_test[i-1].reshape([28,28]),cmap=\"gray\")\n    \n#looks like we are missing the 4th and 5th numbers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save test predictions to file\n# no aug performed better\noutput = pd.DataFrame({'ImageId': range(1,28001),\n                       'Label': realPreds})\noutput.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}