{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"This is my first submission and notebook on Kaggle. I'm still learning and any comments would be appreciated. Hope some may find this useful."},{"metadata":{"trusted":true,"_uuid":"75de0e3751902b63f8f56c1fe3d204028823c153"},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\n#load the libs\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow import keras\nimport matplotlib.pyplot as plt\nimport os\nprint(tf.__version__)\nprint(os.listdir(\"../input\"))\nprint(\"test\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"542eb6aa3808e23dd9dc29a1a419edb643df775b"},"cell_type":"markdown","source":"Import the MNIST data from input folder"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#import data and define the classes\ntrain_data = pd.read_csv(\"../input/train.csv\")\ntest_data = pd.read_csv(\"../input/test.csv\")\nclass_names = [0,1,2,3,4,5,6,7,8,9]\n\n#print out training data\nprint(train_data.shape)\nprint(train_data.head())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2d98713d22cc3131b19b1cd01b96aa0d20cab7b"},"cell_type":"markdown","source":"Process the data by splitting training data into features and labels. Then apply preprocessing to the data to get it ready for input into the model."},{"metadata":{"trusted":true,"_uuid":"a7ad2ed260a8d997efdb548a77e011630cd2ff23"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n#split out the data into features (pixel values) and categorical labels (digit values 0-9)\ntrain_x = train_data.iloc[:,1:].values.astype('float32') # all pixel values\ntrain_y = train_data.iloc[:,0].values.astype('int32') # only labels i.e targets digits\n\ntest_x = test_data.iloc[:,].values.astype('float32') # all pixel values\n\n#reshape the features to be 28x28\ntrain_x = train_x.reshape(train_x.shape[:1] + (28, 28, 1))\ntest_x = test_x.reshape(test_x.shape[:1] + (28, 28, 1))\n\n#change the labels to be one-hot encoded\ntrain_y = keras.utils.to_categorical(train_y)\nnum_classes = train_y.shape[1]\n\n\n#normalize pixel values using minmax (values between 0 and 1 inclusive)\ntrain_x = train_x / 255\ntest_x = test_x / 255","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6878a1cbee916ba6cc0597d7cee91b201c44df51"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9769424c0374bfef349702b9da46d6c77d352f20"},"cell_type":"code","source":"keras.callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, batch_size=32, write_graph=True, write_grads=True, write_images=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8adaf9b0ce1ebe09b46c262e18d216d1b869884e"},"cell_type":"markdown","source":"|Create Keras callbacks for use during training"},{"metadata":{"_uuid":"a1b31e234e968ee866c79a19dbb16825a189c889"},"cell_type":"markdown","source":"View an example of one feature"},{"metadata":{"trusted":true,"_uuid":"e8355637a5d928322fcd27f701c72aff75bd7f7d"},"cell_type":"code","source":"plt.figure()\nplt.imshow(train_x[0].reshape(28, 28))\nplt.colorbar()\nplt.grid(False)\nplt.show()\n\n#plot a group of features and labels to check data\nplt.figure(figsize=(10,10))\nfor i in range(25):\n    plt.subplot(5,5,i+1)\n    plt.xticks([])\n    plt.yticks([])\n    plt.grid(False)\n    plt.imshow(train_x[i].reshape(28, 28), cmap=plt.cm.binary)\n    plt.xlabel(class_names[np.argmax(train_y[i])])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"25fac73297c7c3b2843df009b0292d18dd9ede87"},"cell_type":"markdown","source":"Define the model using Keras and TensorFlow backend (channels first)"},{"metadata":{"trusted":true,"_uuid":"9a1bcf666409bc9e72dba96e7592c19a72307ab2"},"cell_type":"code","source":"from keras import regularizers\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten\nfrom keras.layers import Conv2D, MaxPooling2D\nfrom keras import backend as K\nfrom keras.layers.normalization import BatchNormalization\n\n#define the model and layers\n\n#first layer\nlayer1= tf.keras.layers.Conv2D(32,kernel_size=(3, 3),activation='relu',kernel_initializer='he_normal',\n                          input_shape=(28,28,1))\nlayer2= tf.keras.layers.Conv2D(32,kernel_size=(3,3), activation='relu',kernel_initializer='he_normal')\nlayer3= tf.keras.layers.MaxPooling2D(pool_size=(2,2))\nlayer4= tf.keras.layers.Dropout(0.20)\n\n#second layer\nlayer5= tf.keras.layers.Conv2D(64,(3, 3),activation='relu',padding='same')\nlayer6= tf.keras.layers.Conv2D(64,(3,3),activation='relu',padding='same')\nlayer7= tf.keras.layers.MaxPooling2D(pool_size=(2,2))\nlayer8= tf.keras.layers.Dropout(0.25)\n\n\n\n## third layer\nlayer9= tf.keras.layers.Conv2D(128,(3, 3),activation='relu',padding='same')\nlayer10= tf.keras.layers.Dropout(0.25)\n\n#output layer\nlayer11= tf.keras.layers.Flatten()\nlayer12= tf.keras.layers.Dense(128,activation='relu')\n# layer12 = tf.keras.layers.BatchNormalization()\nlayer13= tf.keras.layers.Dropout(0.3)\nlayer14= tf.keras.layers.Dense(10, activation=tf.nn.softmax)\nmodel = keras.models.Sequential()\nmodel.add(layer1)\nmodel.add(layer2)\nmodel.add(layer3)\nmodel.add(layer4)\nmodel.add(layer5)\nmodel.add(layer6)\nmodel.add(layer7)\nmodel.add(layer8)\nmodel.add(layer9)\nmodel.add(layer10)\nmodel.add(layer11)\n# model.add(layer12)\nmodel.add(layer13)\nmodel.add(layer14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41d0db6ff0d8b63d9df66e11347a0247f364754f"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7520471457012e5058b010574d9487f9a2e725d7","_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"## save model to image\n\nfrom keras.utils import plot_model\nplot_model(model, to_file='model.png')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6c323c5fbbe4c582db90bc0231db6c381b0823c0"},"cell_type":"markdown","source":"Compile and summarize the model. Use adam optimizer and categorical cross entropy for loss (takes input of one-hot encoded targets)"},{"metadata":{"trusted":true,"_uuid":"74ed627f9fe49932dec02fcc83bc877b7eac3960"},"cell_type":"code","source":"#compile the model\nmodel.compile(optimizer=tf.keras.optimizers.Adam(),\n              loss='categorical_crossentropy',\n              metrics=['accuracy'])\n\n#print a summary of the model\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10ae21041d921784c48d7c426db8a95755ddbad1"},"cell_type":"markdown","source":"|Train the model using batch_size of 32 and up to 30 epochs (depending on improvement of validation accuracy)."},{"metadata":{"trusted":true,"_uuid":"736eb21f9cc718a6edaf51263977fc2bb8925ec0","scrolled":true},"cell_type":"code","source":"#train the model\nhist = model.fit(x=train_x, \n            y=train_y,\n            batch_size=128,\n            epochs=20,\n            verbose=1,\n            validation_split=0.15,\n            shuffle=True)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57c92ba559f7397e221ddae4fd4dc3828e0cca43"},"cell_type":"markdown","source":"\n Make prediction on the test data"},{"metadata":{"trusted":true,"_uuid":"f57416500b158a18096b71ea3b8162dd666b085c"},"cell_type":"code","source":"test = layer1.get_weights()\ntest[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ca31005e77b5711d9a34904c2a67cdb87a6266b"},"cell_type":"code","source":"plt.imshow(train_x[10][:,:,0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"deb2837525efd3fe4406f27b8e201291644d737d"},"cell_type":"code","source":"from numpy import array\nimport matplotlib.pyplot as plt\n\ndef plot_conv_weights():\n    W = layer1.get_weights()[0]\n    if len(W.shape) == 4:\n        W = np.squeeze(W)\n        for i in range(W.shape[0]):\n            print(i)\n            for j in range(W.shape[1]):\n                print(j)\n                a = array(W[i][j])\n                b = a.reshape(4,8)\n                plt.imshow(b, cmap='magma', interpolation='nearest')\n                name = str(i) + str(j) + \".png\"\n                plt.savefig(name)\n        \n\n            \nplot_conv_weights()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2104067069b859a057d8de5ba503a30be852b56a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b3b8ba19f504762547e9006bb3e962e6c7baafb"},"cell_type":"code","source":"lay.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"813358feeb9e2ad58ce66347fdc3b76e0b7dcc9a"},"cell_type":"code","source":"#make predictions on the test features\npredictions = model.predict(test_x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5874c2f83b7431294d68a0fc9c7156ec13ca2b77"},"cell_type":"markdown","source":"Plot a table of test features (images) and the predicted targets (digits). Display the confidence via probability of the prediction under each image."},{"metadata":{"trusted":true,"_uuid":"3f7fe171919a0f3cb2a8cd3df780dc0d8c581245"},"cell_type":"code","source":"def plot_value_array(i, predictions_array):\n    predictions_array = predictions_array[i]\n    plt.grid(False)\n    plt.xticks([])\n    plt.yticks([])\n    thisplot = plt.bar(range(10), predictions_array, color=\"#777777\")\n    plt.ylim([0, 1]) \n    predicted_label = np.argmax(predictions_array)\n    thisplot[predicted_label].set_color('red')\n\ndef plot_image(i, predictions_array, img):\n    img = img.reshape(img.shape[0] ,28, 28)\n    predictions_array, img = predictions_array[i], img[i]\n    plt.grid(False)\n    plt.xticks([])\n    plt.yticks([])\n    plt.imshow(img, cmap=plt.cm.binary)\n    predicted_label = np.argmax(predictions_array)\n    plt.xlabel(\"{} - prob:{:2.0f}%\".format(class_names[predicted_label], 100*np.max(predictions_array)), color='red')\n\n# Plot the first X test images, their predicted label, and the true label\n# Color correct predictions in blue, incorrect predictions in red\nnum_rows = 5\nnum_cols = 3\nnum_images = num_rows*num_cols\nplt.figure(figsize=(2*2*num_cols, 2*num_rows))\nfor i in range(num_images):\n    plt.subplot(num_rows, 2*num_cols, 2*i+1)\n    plot_image(i, predictions, test_x)\n    plt.subplot(num_rows, 2*num_cols, 2*i+2)\n    plot_value_array(i, predictions)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cf07c10253f000dbcc1456b4fad4b8a93b2f994"},"cell_type":"markdown","source":"Create submission file for Kaggle"},{"metadata":{"trusted":true,"_uuid":"cdb4bc75164d700d276b00a1c27a75e425cd5da2"},"cell_type":"code","source":"print(hist.history['loss'])\nprint(hist.history['acc'])\nprint(hist.history['val_loss'])\nprint(hist.history['val_acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"654c64c5632e430b5d97aedc004b6c3aaf68319d","scrolled":true},"cell_type":"code","source":"\nimport matplotlib.pyplot as plt\n\nfig, loss_ax = plt.subplots()\nacc_ax = loss_ax.twinx()\n\nloss_ax.plot(hist.history['loss'], 'y', label='train loss')\nloss_ax.plot(hist.history['val_loss'], 'r', label='val loss')\nloss_ax.set_xlabel('epoch')\nloss_ax.set_ylabel('loss')\nloss_ax.legend(loc='upper right')\n\nacc_ax.plot(hist.history['acc'], 'b', label='train acc')\nacc_ax.plot(hist.history['val_acc'], 'g', label='val acc')\nacc_ax.set_ylabel('accuracy')\nacc_ax.legend(loc='upper left')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa23cad0fd3be650c456f429277e8f5705fd19ee"},"cell_type":"code","source":"#submissions for Kaggle\n#cat_predictions = np.argmax(predictions, axis=1)\nsubmissions=pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n                         \"Label\": np.argmax(predictions, axis=1)})\nsubmissions.to_csv(\"my_submissions.csv\", index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34e9d114cfff2d5aa0af7a0ec848107278ce40ab"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}