{"cells":[{"metadata":{"_uuid":"2568464a3174a8733e67daaf4e7b6b0b330b61fa","trusted":false},"cell_type":"code","source":"# Load in Packages \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt, matplotlib.image as mpimg #plotting\nfrom sklearn.model_selection import train_test_split #machine Learning\nfrom sklearn import svm #support vector machines\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"037768a46e9edd8e70efe327930f9dfc64f094d5","scrolled":true,"trusted":false},"cell_type":"code","source":"#Load CSV file\nlabeled_images = pd.read_csv(\"../input/train.csv\")\n#create labels from first column\nlabels = labeled_images.iloc[:, 0]\n#create images from remaining columns\nimages = labeled_images.iloc[:, 1:]\n#split data into train and testing 80% for training\ntrain_images, test_images,train_labels, test_labels = train_test_split(images, labels, train_size=0.8, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5c705c206c4bed3505f62c1b8a5749c4aa7381c3","trusted":false},"cell_type":"code","source":"#save copies of training and testing image data before image processing steps\ntrain_images_original=train_images.copy()\ntest_images_original=test_images.copy()\n\n#scale image matrices\nmax_value = images.values.max()\ntrain_images_scaled = train_images.copy()/max_value\ntest_images_scaled= test_images.copy()/max_value\n\n#turn image matrices of pixel values into 0 and 1s only\ntrain_images.iloc[train_images>0] = 1\ntest_images.iloc[test_images>0] = 1\n\n#plot histogram of 0's compared to 1's for sample image\nplt.hist(train_images.iloc[5])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75cf91790232200cd90e357fb05c87046165f02a","trusted":false},"cell_type":"code","source":"#function to plot images on grid of m by m with labels\ndef plot_as_grid(images, labels, m):\n    n_pixels = len(images.columns)\n    dimension = int(np.sqrt(n_pixels))\n\n    # set up the figure\n    fig = plt.figure(figsize=(6, 6))  # figure size in inches\n    fig.subplots_adjust(left=0, right=1, bottom=0, top=1, hspace=0.5, wspace=0.05)\n\n    # plot the digits: each image is max mxm pixels\n    for i in range( min(m*m, len(images.index))):\n        ax = fig.add_subplot(m, m, i + 1, xticks=[], yticks=[])    \n    \n        img=images.iloc[i].values.reshape((dimension,dimension))\n    \n        ax.imshow(img, cmap=plt.cm.binary, interpolation='nearest')\n        plt.title(labels.iloc[i])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9239d04539a2cacd3171251683f5ce83befb85a7","trusted":false},"cell_type":"code","source":"#compare by eye differences between raw data, processed and scaled images\nplot_as_grid(test_images,test_labels,2)\nplot_as_grid(test_images_scaled,test_labels,2)\nplot_as_grid(test_images_original,test_labels,2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"782be1c6755f62c7396f1cc1352234d334ab4ba0","scrolled":true,"trusted":false},"cell_type":"code","source":"#create support vector machine classifier, fit to processed data and check score on test data \nclf = svm.SVC()\nclf.fit(train_images, train_labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"74d44cdfd2004f31d3bc50a5149588d9528a66ea","trusted":false},"cell_type":"code","source":"#create support vector machine classifier, fit to raw data\nclf_raw = svm.SVC()\nclf_raw.fit(train_images_original, train_labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7602f1a804f598c74cfc471a8a70f18dce65ac16","trusted":false},"cell_type":"code","source":"#create support vector machine classifier, fit to scaled data\nclf_scaled = svm.SVC()\nclf_scaled.fit(train_images_scaled, train_labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0020340a0b3486519cf9ec35b08d21c473afebdb","trusted":false},"cell_type":"code","source":"#check scores on training and testing data\ntrainingscore=clf.score(train_images,train_labels)\ntestingscore=clf.score(test_images,test_labels)\nprint(\"Training Score on 0&1 processed data:\"+str(trainingscore))\nprint(\"Testinging Score on 0&1 processed data:\"+str(testingscore))\n\ntestingscore_raw=clf_raw.score(test_images_original,test_labels)\nprint(\"Testing Score on raw data:\"+str(testingscore_raw))\n\ntestingscore_scaled=clf_scaled.score(test_images_scaled,test_labels)\nprint(\"Testing Score on scaled data:\"+str(testingscore_scaled))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d127d5f65da98b306b434fbb2aa1f33b5c07a3ee"},"cell_type":"code","source":"trainingscore_raw=clf_raw.score(train_images_original,train_labels)\nprint(\"Training Score on raw data:\"+str(trainingscore_raw))\n\ntrainingscore_scaled=clf_scaled.score(train_images_scaled,train_labels)\nprint(\"Training Score on scaled data:\"+str(trainingscore_scaled))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"65b19dd8c15f1533e60094f56b10ccc4923fca79"},"cell_type":"markdown","source":"**Processed data (where images are matrices of 1's and 0's) has the highest scores for Support Vector Machines**"},{"metadata":{"_uuid":"0fd5db9439368db391366ccf0e04d1a00a42e595","trusted":false},"cell_type":"code","source":"from sklearn import metrics\n#function to plot confusion matrix\ndef plot_confusion_matrix(labels, predictions):\n    print(\"Confusion matrix:\\n%s\" % metrics.confusion_matrix(labels, predictions))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd75a78615e11222f2ed15b9ad4ab1d47b8062d8"},"cell_type":"markdown","source":"Let us now use the Support Vector machine to predict values and create a figure to see the results versus images"},{"metadata":{"_uuid":"075e57b8a203b9f1e965745e981d6d863e4339af","trusted":false},"cell_type":"code","source":"#create results variable from test_images\nresults=clf.predict(test_images)\n#create dataframe and plot images with predictions as titles\nresults_dataframe = pd.DataFrame(results)\nresults_dataframe.columns=['predictions']\nplot_as_grid(test_images,results_dataframe['predictions'],10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"569b1cd849bd7766a110d94f14af3faad9a5824d","trusted":false},"cell_type":"code","source":"#create a confusion matrix to see which values have what issues\nplot_confusion_matrix(test_labels,results)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3cbbdc37a39b3c6e75fcda9a61a40182bdd82751"},"cell_type":"markdown","source":"We will now save a results file created from test data for the competition. Sample code to write to CSV"},{"metadata":{"trusted":false,"_uuid":"d3886f7a1021776ef84da4411ef9ee789c2026ef"},"cell_type":"code","source":"test_data=pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"43ad82d91c89806af4df6bd76dfd3638bcac6174"},"cell_type":"code","source":"results=clf.predict(test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"30de18640df81727c7841bc48c88a62684d56af2"},"cell_type":"code","source":"df = pd.DataFrame(results)\ndf.index.name='ImageId'\ndf.index+=1\ndf.columns=['Label']\ndf.to_csv('results.csv', header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"879a20c729944eead7b69dda03b6af4148d4326c"},"cell_type":"code","source":"def plot_incorrect_classifications(ypred, test_labels, test_images):\n    \"Plots incorrectly classified images and corresponding prediction\"\n    ypred = pd.DataFrame(ypred)\n    ypred = ypred.set_index(test_labels.index.values)\n    ypred.columns = ['prediction']\n    predict_df = pd.concat([ypred, test_labels], axis=1)\n    predict_df['Incorrect'] = predict_df.prediction != predict_df.label\n    idx = predict_df.index[predict_df['Incorrect']]\n\n    plot_as_grid(test_images.loc[idx], predict_df['prediction'].loc[idx], 5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d131a21c7b5ff95e152d7938c699d6f7a7b2b925"},"cell_type":"code","source":"results_processed=clf.predict(test_images)\nresults_raw=clf_raw.predict(test_images_original)\nresults_scaled=clf_scaled.predict(test_images_scaled)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b84379a7161223aa9f01734014b927b4c5f895b8"},"cell_type":"code","source":"plot_incorrect_classifications(results_processed, test_labels, test_images)\nplot_incorrect_classifications(results_raw, test_labels, test_images_original)\nplot_incorrect_classifications(results_scaled, test_labels, test_images_scaled)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a415570cf156841445aa40ca91d0e904fbc9b9c0"},"cell_type":"markdown","source":"We will now create Neural Networks to classify the data"},{"metadata":{"trusted":false,"_uuid":"8ea253377e11befabd3e5747e7ea3d30984a8e25"},"cell_type":"code","source":"# Create first network with Keras\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers import Dense","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0be53cb70ba77c7660432d1625a148b56328af7c"},"cell_type":"code","source":"# create model\nmodel_scaled = Sequential()\nmodel_scaled.add(Dense(820, input_dim=784, activation='relu')) #dense type, 820 neurons, 784 inputs ie the number of pixels, activation function relu\nmodel_scaled.add(Dense(410, activation='relu'))\nmodel_scaled.add(Dense(10, activation='softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f50f5b63f55a395e25bd96c4b583035176bd7dae"},"cell_type":"code","source":"# Compile model\nmodel_scaled.compile(optimizer='adam',  # Good default optimizer to start with\n              loss='sparse_categorical_crossentropy',  # how will we calculate our \"error.\" Neural network aims to minimize loss.\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d42bca91811e0a632a80ca65ddbc637e5d6fb495"},"cell_type":"code","source":"# Fit the model\nmodel_scaled.fit(train_images_scaled, train_labels, epochs=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e7a072b13d971be00d193a1be2433bc112c69bae"},"cell_type":"code","source":"# create model\nmodel_raw = Sequential()\nmodel_raw.add(Dense(820, input_dim=784, activation='relu'))\nmodel_raw.add(Dense(410, activation='relu'))\nmodel_raw.add(Dense(10, activation='softmax'))\n# Compile model\nmodel_raw.compile(optimizer='adam',  # Good default optimizer to start with\n              loss='sparse_categorical_crossentropy',  # how will we calculate our \"error.\" Neural network aims to minimize loss.\n              metrics=['accuracy'])\n# Fit the model\nmodel_raw.fit(train_images_original, train_labels, epochs=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8ca94d09003486d3071693820e82ac285d04a178"},"cell_type":"code","source":"# create model\nmodel_processed = Sequential()\nmodel_processed.add(Dense(820, input_dim=784, activation='relu'))\nmodel_processed.add(Dense(410, activation='relu'))\nmodel_processed.add(Dense(10, activation='softmax'))\n# Compile model\nmodel_processed.compile(optimizer='adam',  # Good default optimizer to start with\n              loss='sparse_categorical_crossentropy',  # how will we calculate our \"error.\" Neural network aims to minimize loss.\n              metrics=['accuracy'])\n# Fit the model\nmodel_processed.fit(train_images, train_labels, epochs=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"79dc0d27735112c8f1d2c2239a5b5f26fd9d8647"},"cell_type":"code","source":"predictions_scaled=model_scaled.predict(test_images_scaled)\npredictions_raw=model_raw.predict(test_images_original)\npredictions_processed=model_processed.predict(test_images)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"231b6717fd0c8471744fc36b9ec611212682153c"},"cell_type":"code","source":"def probability_matrix_to_classification(predictions):\n    return np.argmax(predictions,axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1b95264ed93c772cdc3d035de979b697333d001d"},"cell_type":"code","source":"predictions_scaled_class=probability_matrix_to_classification(predictions_scaled)\npredictions_raw_class=probability_matrix_to_classification(predictions_raw)\npredictions_processed_class=probability_matrix_to_classification(predictions_processed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3f8f689ebee6a19f672d49c0051bac8e81a6c99f"},"cell_type":"code","source":"def score_of_cnn(prediction_classes,testlabels):\n    op=prediction_classes==testlabels\n    return np.sum(op)/len(prediction_classes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c854c89a5cb5727bd1467010b5a351629396000a"},"cell_type":"code","source":"testingscore_scaled=score_of_cnn(predictions_scaled_class,test_labels)\nprint(\"Testing Score on scaled data:\"+str(testingscore_scaled))\n\ntestingscore_raw=score_of_cnn(predictions_raw_class,test_labels)\nprint(\"Testing Score on raw data:\"+str(testingscore_raw))\n\ntestingscore_processed=score_of_cnn(predictions_processed_class,test_labels)\nprint(\"Testing Score on processed data:\"+str(testingscore_processed))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a703fef269c8e2924ff7a639dbc40dd0ceea7f41"},"cell_type":"markdown","source":"**Scaled data have the highest score for CNN's**"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}