{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"markdown","source":"## import libraries\nimport numpy as np # linear algebra, matrix multiplications\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)"},{"metadata":{"_uuid":"5c4c2fa7eeffa51a4c2294cac109bed54fda7dca","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\ntrain = pd.read_csv(\"../input/digit-recognizer/train.csv\")\ntest = pd.read_csv('../input/hellohello/train_converted.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a7cb2975ffe4514b061c379a9bf8cc37c48f9ed","trusted":true},"cell_type":"code","source":"print(train.shape)\nntrain = train.shape[0]\n\nprint(test.shape)\nntest = test.shape[0]\n\ntrain.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ffd9d5ea1aa332b428fa668215c7c84c28f3dae","trusted":true},"cell_type":"code","source":"# check data type\nprint(train.dtypes[:5]) # all int64, otherwise do train = train.astype('int64')\nprint(train.dtypes[:5]) # all int64, otherwise do test = test.astype('int64')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f9d2f465a2abfab92687cd5c08404f7e8aef8ae6","trusted":true},"cell_type":"code","source":"# array containing labels of each image\nytrain = train[\"label\"]\nprint(\"Shape of ytrain: \", ytrain.shape)\n\n# dataframe containing all pixels (the label column is dropped)\nxtrain = train.drop(\"label\", axis=1)\n\n# the images are in square form, so dim*dim = 784\nfrom math import sqrt\ndim = int(sqrt(xtrain.shape[1]))\nprint(\"The images are {}x{} squares.\".format(dim, dim))\n\nprint(\"Shape of xtrain: \", xtrain.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6e5f24bd732ba81028c32812eccec43ce50db629","trusted":true},"cell_type":"code","source":"ytrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6eb1fc174dc86ce277e2b8fc858ea1067c8174ee","trusted":true},"cell_type":"code","source":"import seaborn as sns\nsns.set(style='white', context='notebook', palette='deep')\n\n# plot how many images there are in each class\nsns.countplot(ytrain)\n\nprint(ytrain.shape)\nprint(type(ytrain))\n\n# array with each class and its number of images\nvals_class = ytrain.value_counts()\nprint(vals_class)\n\n# mean and std\ncls_mean = np.mean(vals_class)\ncls_std = np.std(vals_class,ddof=1)\n\nprint(\"The mean amount of elements per class is\", cls_mean)\nprint(\"The standard deviation in the element per class distribution is\", cls_std)\n\n# 68% - 95% - 99% rule, the 68% of the data should be cls_std away from the mean and so on\n# https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule\nif cls_std > cls_mean * (0.6827 / 2):\n    print(\"The standard deviation is high\")\n    \n# if the data is skewed then we won't be able to use accurace as its results will be misleading and we may use F-beta score instead.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03c7f67d1575a25008ebd244872a262f4a4495ce","trusted":true},"cell_type":"code","source":"def check_nan(df):\n    print(df.isnull().any().describe())\n    print(\"There are missing values\" if df.isnull().any().any() else \"There are no missing values\")\n\n    if df.isnull().any().any():\n        print(df.isnull().sum(axis=0))\n        \n    print()\n        \ncheck_nan(xtrain)\ncheck_nan(test)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a22f34f81cb94af5787f06448e326058e61122d","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport matplotlib.image as mpimg\n%matplotlib inline\n\n# convert train dataset to (num_images, img_rows, img_cols) format in order to plot it\nxtrain_vis = xtrain.values.reshape(ntrain, dim, dim)\n\n# https://matplotlib.org/api/_as_gen/matplotlib.pyplot.subplot.html\n# subplot(2,3,3) = subplot(233)\n# a grid of 3x3 is created, then plots are inserted in some of these slots\nfor i in range(0,9): # how many imgs will show from the 3x3 grid\n    plt.subplot(330 + (i+1)) # open next subplot\n    plt.imshow(xtrain_vis[i], cmap=plt.get_cmap('gray'))\n    plt.title(ytrain[i]);","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76e6f0a067bb7cabf2cfa079a3917f2391693872","trusted":true},"cell_type":"code","source":"# Normalize the data\nxtrain = xtrain / 255.0\ntest = test / 255.0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9685eb7e78736876af2f032c53dd2f2d2ea5382b","trusted":true},"cell_type":"code","source":"# reshape of image data to (nimg, img_rows, img_cols, 1)\ndef df_reshape(df):\n    print(\"Previous shape, pixels are in 1D vector:\", df.shape)\n    df = df.values.reshape(-1, dim, dim, 1) \n    # -1 means the dimension doesn't change, so 42000 in the case of xtrain and 28000 in the case of test\n    print(\"After reshape, pixels are a 28x28x1 3D matrix:\", df.shape)\n    return df\n\nxtrain = df_reshape(xtrain) # numpy.ndarray type\ntest = df_reshape(test) # numpy.ndarray type","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"508a23dfec7dde6ea68e2904277687283a956327","trusted":true},"cell_type":"code","source":"from keras.utils.np_utils import to_categorical\n\nprint(type(ytrain))\n# number of classes, in this case 10\nnclasses = ytrain.max() - ytrain.min() + 1\n\nprint(\"Shape of ytrain before: \", ytrain.shape) # (42000,)\n\nytrain = to_categorical(ytrain, num_classes = nclasses)\n\nprint(\"Shape of ytrain after: \", ytrain.shape) # (42000, 10), also numpy.ndarray type\nprint(type(ytrain))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"117f9749a1511c0a238a0dece0f684bd27b41cef","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# fix random seed for reproducibility\nseed = 2\nnp.random.seed(seed)\n\n# percentage of xtrain which will be xval\nsplit_pct = 0.1\n\n# Split the train and the validation set\nxtrain, xval, ytrain, yval = train_test_split(xtrain,\n                                              ytrain, \n                                              test_size=split_pct,\n                                              random_state=seed,\n                                              shuffle=True,\n                                              stratify=ytrain\n                                             )\n\nprint(xtrain.shape, ytrain.shape, xval.shape, yval.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"162a9388b958cf25272870d290a616be47fd85ee","trusted":true},"cell_type":"code","source":"from keras import backend as K\n\n# for the architecture\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Lambda, Flatten, BatchNormalization\nfrom keras.layers import Conv2D, MaxPool2D, AvgPool2D\n\n# optimizer, data generator and learning rate reductor\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.callbacks import ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb1b773c6bf7baf33756dd265f3a2c1bc662b220","trusted":true},"cell_type":"code","source":"model = Sequential()\n\ndim = 28\nnclasses = 10\n\nmodel.add(Conv2D(filters=32, kernel_size=(5,5), padding='same', activation='relu', input_shape=(dim,dim,1)))\nmodel.add(Conv2D(filters=32, kernel_size=(5,5), padding='same', activation='relu',))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.2))\n\nmodel.add(Conv2D(filters=64, kernel_size=(5,5), padding='same', activation='relu'))\nmodel.add(Conv2D(filters=64, kernel_size=(5,5), padding='same', activation='relu'))\nmodel.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))\nmodel.add(Dropout(0.2))\n\nmodel.add(Flatten())\nmodel.add(Dense(120, activation='relu'))\nmodel.add(Dense(84, activation='relu'))\nmodel.add(Dense(nclasses, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7983fec4500a27ff54adc40535405ba71e9fd80a","trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5785f24f35271268d9616e45cdde37ddf4547921","trusted":true},"cell_type":"code","source":"model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8a38a240404358164f63020aceb591c3907ffdc","trusted":true},"cell_type":"code","source":"lr_reduction = ReduceLROnPlateau(monitor='val_acc', \n                                 patience=3, \n                                 verbose=1, \n                                 factor=0.5, \n                                 min_lr=0.00001)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1379cb5bd05278c3726b39df441a5eb1dcd9dc5","trusted":true},"cell_type":"code","source":"datagen = ImageDataGenerator(\n          featurewise_center=False,            # set input mean to 0 over the dataset\n          samplewise_center=False,             # set each sample mean to 0\n          featurewise_std_normalization=False, # divide inputs by std of the dataset\n          samplewise_std_normalization=False,  # divide each input by its std\n          zca_whitening=False,                 # apply ZCA whitening\n          rotation_range=30,                   # randomly rotate images in the range (degrees, 0 to 180)\n          zoom_range = 0.1,                    # Randomly zoom image \n          width_shift_range=0.1,               # randomly shift images horizontally (fraction of total width)\n          height_shift_range=0.1,              # randomly shift images vertically (fraction of total height)\n          horizontal_flip=False,               # randomly flip images\n          vertical_flip=False)                 # randomly flip images\n\ndatagen.fit(xtrain)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fd7356e907ad7001a84890191271f59975ff49e","trusted":true},"cell_type":"code","source":"epochs = 8\nbatch_size = 4096","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a87fb83b3d6654ee0b2a54e65a8b9c00ec48cf4","scrolled":false,"trusted":true},"cell_type":"code","source":"history = model.fit_generator(datagen.flow(xtrain,ytrain, batch_size=batch_size),\n                              epochs=epochs, \n                              validation_data=(xval,yval),\n                              verbose=1, \n                              steps_per_epoch=xtrain.shape[0] // batch_size, \n                              callbacks=[lr_reduction])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5876a7e97bfb916020e9c04a6a6056bc96f98669","trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"fig, ax = plt.subplots(2,1)\nax[0].plot(history.history['loss'], color='b', label=\"Training loss\")\nax[0].plot(history.history['val_loss'], color='r', label=\"Validation loss\",axes =ax[0])\nax[0].grid(color='black', linestyle='-', linewidth=0.25)\nlegend = ax[0].legend(loc='best', shadow=True)\n\nax[1].plot(history.history['acc'], color='b', label=\"Training accuracy\")\nax[1].plot(history.history['val_acc'], color='r',label=\"Validation accuracy\")\nax[1].grid(color='black', linestyle='-', linewidth=0.25)\nlegend = ax[1].legend(loc='best', shadow=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63a0bda9010ac311a6a9c845343c64ebcba8ddc3","trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\nimport itertools\n\ndef plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, cm[i, j],\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Predict the values from the validation dataset\nypred_onehot = model.predict(xval)\n# Convert predictions classes from one hot vectors to labels: [0 0 1 0 0 ...] --> 2\nypred = np.argmax(ypred_onehot,axis=1)\n# Convert validation observations from one hot vectors to labels\nytrue = np.argmax(yval,axis=1)\n# compute the confusion matrix\nconfusion_mtx = confusion_matrix(ytrue, ypred)\n# plot the confusion matrix\nplot_confusion_matrix(confusion_mtx, classes=range(nclasses))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f5f8df7abc5d6278caa4098f5d39ea7566a55ccb","trusted":true},"cell_type":"code","source":"errors = (ypred - ytrue != 0) # array of bools with true when there is an error or false when the image is cor\n\nypred_er = ypred_onehot[errors]\nypred_classes_er = ypred[errors]\nytrue_er = ytrue[errors]\nxval_er = xval[errors]\n\ndef display_errors(errors_index, img_errors, pred_errors, obs_errors):\n    \"\"\" This function shows 6 images with their predicted and real labels\"\"\"\n    n = 0\n    nrows = 2\n    ncols = 3\n    fig, ax = plt.subplots(nrows, ncols, sharex=True, sharey=True)\n    for row in range(nrows):\n        for col in range(ncols):\n            error = errors_index[n]\n            ax[row,col].imshow((img_errors[error]).reshape((28,28)))\n            ax[row,col].set_title(\"Predicted label :{}\\nTrue label :{}\".format(pred_errors[error],obs_errors[error]))\n            n += 1\n            \n# Probabilities of the wrong predicted numbers\nypred_er_prob = np.max(ypred_er,axis=1)\n\n# Predicted probabilities of the true values in the error set\ntrue_prob_er = np.diagonal(np.take(ypred_er, ytrue_er, axis=1))\n\n# Difference between the probability of the predicted label and the true label\ndelta_pred_true_er = ypred_er_prob - true_prob_er\n\n# Sorted list of the delta prob errors\nsorted_delta_er = np.argsort(delta_pred_true_er)\n\n# Top 6 errors. You can change the range to see other images\nmost_important_er = sorted_delta_er[-6:]\n\n# Show the top 6 errors\ndisplay_errors(most_important_er, xval_er, ypred_classes_er, ytrue_er)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f85a38f96f2fb9c26119f6a2a32ce84ee0751c91","trusted":true},"cell_type":"code","source":"predictions = model.predict_classes(test, verbose=1)\n\nsubmissions = pd.DataFrame({\"ImageId\": list(range(1,len(predictions)+1)),\n                            \"Recognized Number is : \": predictions})\n\nsubmissions.to_csv(\"mnist2908.csv\", index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10b85b336fec00b220b564d6ffb896956e685aaa"},"cell_type":"code","source":"%matplotlib inline\nfrom keras.preprocessing import image\nnum_2=0\nimport matplotlib.pyplot as plt\nimport numpy as np\nwhile(num_2<=20):\n df1 = pd.read_csv('../input/hellohello/train_converted.csv',sep=',') \n df2 = pd.read_csv(\"mnist2908.csv\")\n x = df1.loc[:, df1.columns != 'label'] # shape 42000, 785\n image0 = x.iloc[num_2]\n x2 = np.array(image0)\n # reshape into 28,28,1\n x3 = x2.reshape(28,28)\n plt.imshow(x3)\n plt.show()\n out_row_20 = df2.iloc[num_2]\n print(out_row_20)\n num_2=num_2+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31fff7f6d9dee150dbfb7c28c1e2b8a947c3c38f"},"cell_type":"code","source":"import csv\nwith open('mnist2908.csv', newline='') as csvfile:\n spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|')\n for row in spamreader:\n    print(', '.join(row))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}