{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"<center>\n    <h1>Final Project - KDD</h1>\n    <h2>Digit Reconigzer</h2>\n    <h3>By Allison Fernández and Tatiana Ortiz - MUSS Program</h3>\n</center>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import the necessary packages\n# For Numerical Computing\nimport numpy as np\n\n# For visualize the data\nimport pandas as pd \nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\n\n# For images\nimport cv2 as cv\n\n# For Classifiers\n#from sklearn.model_selection import cross_val_score\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn import tree\nfrom sklearn.neighbors import KNeighborsClassifier\n\n# Preprocessing PCA and Accuracy Metric\nfrom sklearn.decomposition import PCA\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# For ANN tools  - Keras needs Tensorflow package. \nfrom keras.models import Sequential\nfrom keras.utils import to_categorical\nfrom keras.layers import Dense \nfrom keras.optimizers import Adam \nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data domain and description:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train dataset:\n# Load the train data.\ntrainData = pd.read_csv(\"../input/digit-recognizer/train.csv\")\nprint('The train dataset have',trainData.shape[0],'images.')\nprint('There are',trainData.shape[1],'columns. 784 for pixels of each images and 1 for label field.')\n\n#Split the data into Targets and Pixels  \n\n# Train dataset:\n# X-Axis : Pixels values - each images 28x28 pixels convert into to  28^2 = 784 array. \ntrainXAxis = (trainData.iloc[:,1:]).values.astype('float32')\n# Y-Axis : Target : Label\ntrainYAxis = trainData.iloc[:,0].values.astype('int32')\ntrainData.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trainDistribution = sns.countplot(trainYAxis)\nplt.show(trainDistribution)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test dataset:\n# Load the test data.\ntestData = pd.read_csv(\"../input/digit-recognizer/test.csv\")\nprint('The test dataset have',testData.shape[0],'images.')\nprint('There are',testData.shape[1],'columns. 784 for pixels of each images without label.')\n\n# X-Axis : Pixels values\ntestXAxis = (testData.iloc[:,0:]).values.astype('float32')\n# Y-Axis : Target : Label\n#testYAxis = trainData.iloc[:,0].values.astype('int32')\ntestData.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# In order to visualize the hand-drawn digits: "},{"metadata":{"trusted":true},"cell_type":"code","source":"trainXAxisImg = trainXAxis.reshape(trainXAxis.shape[0], 28, 28)\nfig = plt.figure();\nfor i in range(0, 15):\n    plt.subplot(3,5,(i+1))\n    plt.imshow(trainXAxisImg[i], cmap='gray')\n    plt.title(trainYAxis[i]);\n    plt.axis('off')\nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# In order to normalize the data, to have values [0,1].\ntrainXAxisN = (trainXAxis/255)\ntestXAxisN = (testXAxis/255)\n\n# Split the Train dataset. \ntrainImage, testImage, trainLabel, testLabel = train_test_split(trainXAxisN, trainYAxis, test_size=0.4, random_state=42)\nscaler = StandardScaler()\nscaler.fit(trainImage)\nscaler.fit(testImage)\n\ntrainImage = scaler.transform(trainImage)\ntestImage = scaler.transform(testImage)\n\n\nprint('New Train set Size: ',trainImage.shape[0])\nprint('New Test set Size: ',testImage.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Apply PCA For Classifier Algorithms \n# Test to select the best number of components for the PCA. \npca=PCA(784, whiten=True)  # We decided to mantain all the components\npca.fit(trainImage)\ntrainImage_pca=pd.DataFrame(pca.transform(trainImage))\ntestImage_pca=pd.DataFrame(pca.transform(testImage))\nexpl = pca.explained_variance_ratio_\nplt.plot(np.cumsum(pca.explained_variance_ratio_))\nplt.xlabel('Number of components.')\nplt.ylabel('Cumulative explained variance.')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PCA value.\npca=PCA(400, whiten=True)  # We decided to mantain 400 the components\npca.fit(trainImage)\ntrainImage_pca=pd.DataFrame(pca.transform(trainImage))\ntestImage_pca=pd.DataFrame(pca.transform(testImage))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.1. Data Mining: Classifiers Algorithms "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Algorithms\nGaussianModel = GaussianNB()\nDecisionTreeModel = tree.DecisionTreeClassifier()\nRandomForestModel = RandomForestClassifier()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the algorithms\nGaussianModel.fit(trainImage_pca,trainLabel)\nDecisionTreeModel.fit(trainImage_pca,trainLabel)\nRandomForestModel.fit(trainImage_pca,trainLabel) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test the algorithms with PCA data prepocessing.\nGAMPredict = GaussianModel.predict(testImage_pca)\nDTMPredict = DecisionTreeModel.predict(testImage_pca)\nRFMPredict = RandomForestModel.predict(testImage_pca)\n\nprint('Accuracy score for Gaussian Model - using PCA:' ,accuracy_score(GAMPredict, testLabel))\nprint('Accuracy score for Decision Tree Model - using PCA:' ,accuracy_score(DTMPredict, testLabel))\nprint('Accuracy score for Random Forest Model - using PCA:' ,accuracy_score(RFMPredict, testLabel))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### No PCA data preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the algorithms\nGaussianModel.fit(trainImage,trainLabel)\nDecisionTreeModel.fit(trainImage,trainLabel)\nRandomForestModel.fit(trainImage,trainLabel) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test the algorithms without PCA data prepocessing.\nGAMPredict = GaussianModel.predict(testImage)\nDTMPredict = DecisionTreeModel.predict(testImage)\nRFMPredict = RandomForestModel.predict(testImage)\n\nprint('Accuracy score for Gaussian Model:' ,accuracy_score(GAMPredict, testLabel))\nprint('Accuracy score for Decision Tree Model:' ,accuracy_score(DTMPredict, testLabel))\nprint('Accuracy score for Random Forest Model:' ,accuracy_score(RFMPredict, testLabel))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print ('Confusion Matrix for Random Forest Model:\\n\\n')\nprint (confusion_matrix(RFMPredict, testLabel))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.2. Data Mining: K-Neighbors Classifier"},{"metadata":{},"cell_type":"markdown","source":"#### Using PCA"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model\nKNCModel = KNeighborsClassifier()\n\n# Train the model\nKNCModel.fit(trainImage_pca,trainLabel)\n\n# Test the model\nKNCPredict = KNCModel.predict(testImage_pca)\n\nprint('Accuracy score for K.neighbors Model - PCA:' ,accuracy_score(KNCPredict, testLabel))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Not using PCA"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Model\nKNCModel = KNeighborsClassifier()\n\n# Train the model\nKNCModel.fit(trainImage,trainLabel)\n\n# Test the model\nKNCPredict = KNCModel.predict(testImage)\n\nprint('Accuracy score for K-neighbors Model:' ,accuracy_score(KNCPredict, testLabel))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print ('Metrics For K-neighbors Model:\\n\\n ')\nprint ('Confussion matrix for K-neighbors Model:\\n\\n ')\nprint (confusion_matrix(KNCPredict, testLabel))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predicted value:\ntestXAxisImg = testXAxis.reshape(testXAxis.shape[0], 784)\nprint(testXAxisImg.shape)\npredictionsKNC = KNCModel.predict(testXAxisImg)\nprint(predictionsKNC)\ntestXAxis = testXAxis.reshape(testXAxis.shape[0], 28, 28)\nfig = plt.figure();\nfor i in range(0, 40):\n    plt.subplot(5,8,(i+1))\n    plt.imshow(testXAxis[i], cmap='gray')\n    plt.title(predictionsKNC[i]);\n    plt.axis('off')\nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 1.3. Data Mining: Artificial Neural Network Model "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build the model\nmodel = Sequential()\nmodel.add( Dense(70, activation = 'relu', input_dim = 784))\nmodel.add( Dense(10, activation = 'softmax'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile the model\nmodel.compile(\n    optimizer = 'adam',\n        loss = 'categorical_crossentropy',\n        metrics = ['accuracy']\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\nmodel.fit(\n    trainImage,\n        to_categorical(trainLabel),\n        epochs = 2,\n        batch_size = 32\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Evaluate the model\nmodel.evaluate(\n    testImage,\n    to_categorical(testLabel)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Predicted value:\ntestXAxisImg = testXAxis.reshape(testXAxis.shape[0], 784)\npredictions = model.predict_classes(testXAxisImg, verbose=0)\nprint('The predicted values by the Model:')\ntestXAxis = testXAxis.reshape(testXAxis.shape[0], 28, 28)\nfig = plt.figure();\nfor i in range(0, 40):\n    plt.subplot(5,8,(i+1))\n    plt.imshow(testXAxis[i], cmap='gray')\n    plt.title(predictions[i]);\n    plt.axis('off')\nfig.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = model.predict_classes(testImage, verbose=0)\nmatrix = confusion_matrix(testLabel, prediction)\nprint('Confusion Matrix for ANN Model:\\n\\n',matrix)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Results summary\n\n<TABLE>\n    <TR>\n        <TH>Algorithm</TH> <TH>Accuracy</TH>\n    </TR>\n    <TR>\n        <TH>Random Forest Tree</TH> <TD>0.9605</TD>\n    </TR>\n    <TR>\n        <TH>ANN - 2 epochs</TH> <TD>0.9581</TD>\n    </TR>\n    <TR>\n        <TH>K-neighbors</TH> <TD>0.9335</TD>\n    </TR>\n    <TR>\n        <TH>Decision Tree</TH> <TD>0.8447</TD>\n    </TR>\n    <TR>\n        <TH>Gaussian</TH> <TD>0.5976</TD>\n    </TR>\n    <TR>\n        <TH>Random Forest Tree - PCA</TH> <TD>0.9234</TD>\n    </TR>\n    <TR>\n        <TH>Decision Tree - PCA</TH> <TD>0.7976</TD>\n    </TR>\n    <TR>\n        <TH>K-neighbors - PCA</TH> <TD>0.7730</TD>\n    </TR>\n    <TR>\n        <TH>Gaussian - PCA</TH> <TD>0.4938</TD>\n    </TR>\n</TABLE>"},{"metadata":{},"cell_type":"markdown","source":"#### We decided to select the ANN - 2 epochs model, based on the predictions results and in the accuracy. "},{"metadata":{},"cell_type":"markdown","source":"### Submit the results in Kaggle."},{"metadata":{"trusted":true},"cell_type":"code","source":"imgId = range(1, len(predictions) + 1)\nourSubmission = pd.DataFrame({'ImageId': imgId, 'Label': predictions})\n\nourSubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}