{"cells":[{"metadata":{},"cell_type":"markdown","source":"Introduction\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport pandas as pd \nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom keras.models import Sequential\nfrom keras.layers.core import Lambda, Dense, Flatten, Dropout\nfrom keras.callbacks import EarlyStopping\nfrom keras.layers import BatchNormalization, Conv2D  , MaxPooling2D\nfrom keras.utils import np_utils\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC  \nfrom sklearn import tree","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Pre-processing"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))\nprint(os.getcwd())\n#read the data and turn it into a dataframe\nX_data = pd.read_csv(\"../input/train.csv\") \nT_df = pd.read_csv(\"../input/test.csv\") \nY_df = X_data[\"label\"]\nX_df = X_data.drop(\"label\", axis=1)\n#%%\n#extract just the values so it works with numpy\nX = X_df.values\nX = X/255 #We also take this moment to normalize the data. \nY = Y_df.values\n\nnp.random.seed(100) #seet the seed so it is reproducible\n\n#split our data into testing and training, for validation\nX_train, X_test, Y_train, Y_test = train_test_split(X, Y)\n\n\n#Logistic regression and Bayes want Y to have a certain shape. \nY_train_r = np.ravel(Y_train) \nY_test_r = np.ravel(Y_test)\n\n#our neural network wants Y to be categorical. \nY_train_c = np_utils.to_categorical(Y_train, 10)\nY_test_c = np_utils.to_categorical(Y_test, 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"KNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"#create classifier\nKNN = KNeighborsClassifier(n_neighbors=1)\n#fit classifier\nKNN.fit(X_train,Y_train)\n#score classifier\nKNN.score(X_test, Y_test)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create classifier\nKNN_2 = KNeighborsClassifier(n_neighbors=2)\n#fit classifier\nKNN_2.fit(X_train,Y_train)\n#score classifier\nKNN_2.score(X_test, Y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"SVC"},{"metadata":{"trusted":true},"cell_type":"code","source":"#SVC\n\nsvclassifier = SVC(kernel=\"linear\")  \nsvclassifier.fit(X_train, Y_train)  \n\nY_pred = svclassifier.predict(X_test)\naccuracy_score(Y_test, Y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"svclassifier_rbf = SVC(kernel=\"rbf\", gamma = \"auto\")  \nsvclassifier_rbf.fit(X_train, Y_train)  \n\nY_pred = svclassifier_rbf.predict(X_test)\naccuracy_score(Y_test, Y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Logistic Regression"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Logistic regression. \n\n#first we create a list, that we will populate with fitted models \nLreg = list()\nfor i in range(-3,4):\n    c = 10**i #(0.001 ,0.01, 0.1, 1, 10, 100, 1000)\n    #create our model\n    dummy = LogisticRegression(multi_class=\"multinomial\",solver=\"lbfgs\", max_iter = 4000, C=c)\n    #fit our model and add it to the list\n    Lreg.append(dummy.fit(X_train,Y_train_r))    \n\n#create a list of scores\nLreg_scores = list()\nfor i in range(0,6):\n    #score our model and add it to the list. \n    Lreg_scores.append(Lreg[i].score(X_test, Y_test)) \n\n#print out list\nprint(Lreg_scores)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Neural Network"},{"metadata":{"trusted":true},"cell_type":"code","source":"neural1 = Sequential() \nneural1.add(Dense(16, input_dim=784 , activation='relu')) \nneural1.add(Dense(10, activation='softmax'))\n\nneural2 = Sequential() \nneural2.add(Dense(32, input_dim=784 , activation='relu')) \nneural2.add(Dense(10, activation='softmax'))\n\nneural3 = Sequential() \nneural3.add(Dense(64, input_dim=784 , activation='relu')) \nneural3.add(Dense(10, activation='softmax'))\n\nneural4 = Sequential() \nneural4.add(Dense(128, input_dim=784 , activation='relu')) \nneural4.add(Dense(10, activation='softmax'))\n\nneural5 = Sequential() \nneural5.add(Dense(256, input_dim=784 , activation='relu')) \nneural5.add(Dense(10, activation='softmax'))\n\nneural6 = Sequential() \nneural6.add(Dense(512, input_dim=784 , activation='relu')) \nneural6.add(Dense(10, activation='softmax'))\n\nneural7 = Sequential() \nneural7.add(Dense(1024, input_dim=784 , activation='relu')) \nneural7.add(Dense(10, activation='softmax'))\n\nneural8 = Sequential() \nneural8.add(Dense(2048, input_dim=784 , activation='relu')) \nneural8.add(Dense(10, activation='softmax'))\n\nneural1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nneural2.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nneural3.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nneural4.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nneural5.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nneural6.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nneural7.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nneural8.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nneural1.fit(X_train, Y_train_c, epochs=50, batch_size=200)\nneural2.fit(X_train, Y_train_c, epochs=50, batch_size=200)\nneural3.fit(X_train, Y_train_c, epochs=50, batch_size=200)\nneural4.fit(X_train, Y_train_c, epochs=50, batch_size=200)\nneural5.fit(X_train, Y_train_c, epochs=50, batch_size=200)\nneural6.fit(X_train, Y_train_c, epochs=50, batch_size=200)\nneural7.fit(X_train, Y_train_c, epochs=50, batch_size=200)\nneural8.fit(X_train, Y_train_c, epochs=50, batch_size=200)\n\nscores1 = neural1.evaluate(X_test, Y_test_c)\nscores2 = neural2.evaluate(X_test, Y_test_c)\nscores3 = neural3.evaluate(X_test, Y_test_c)\nscores4 = neural4.evaluate(X_test, Y_test_c)\nscores5 = neural5.evaluate(X_test, Y_test_c)\nscores6 = neural6.evaluate(X_test, Y_test_c)\nscores7 = neural7.evaluate(X_test, Y_test_c)\nscores8 = neural8.evaluate(X_test, Y_test_c)\n\nplt.plot([1,2,3,4,5,6,7,8],\n         [scores1[0],scores2[0],scores3[0],scores4[0],scores5[0],scores6[0],\n                            scores7[0],scores8[0]],'-o')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot([1,2,3,4,5,6,7,8],[scores1[1],scores2[1],scores3[1],scores4[1],scores5[1],scores6[1],\n                            scores7[1],scores8[1]],'-o')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"CNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nX_train_2d = X_train.reshape(X_train.shape[0], 28, 28,1)\nX_train_2d.shape\nX_test_2d = X_test.reshape(X_test.shape[0], 28, 28,1)\nX_test_2d.shape\n\n\nmodel_1= Sequential()\nmodel_1.add(Conv2D(filters = 32, kernel_size = (5,5),activation ='relu', input_shape=(28,28,1)))\nmodel_1.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_1.add(Flatten())\nmodel_1.add(Dense(512, activation='relu'))\nmodel_1.add(Dense(10, activation='softmax'))\n\nmodel_2= Sequential()\nmodel_2.add(Conv2D(filters = 32, kernel_size = (5,5),activation ='relu', input_shape=(28,28,1)))\nmodel_2.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_2.add(Conv2D(filters = 64, kernel_size = (5,5),activation ='relu'))\nmodel_2.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_2.add(Flatten())\nmodel_2.add(Dense(512, activation='relu'))\nmodel_2.add(Dense(10, activation='softmax'))\n\nmodel_3= Sequential()\nmodel_3.add(Conv2D(filters = 32, kernel_size = (5,5),activation ='relu', input_shape=(28,28,1)))\nmodel_3.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_3.add(Conv2D(filters = 64, kernel_size = (5,5),activation ='relu'))\nmodel_3.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_3.add(Conv2D(filters = 128, kernel_size = (3,3),activation ='relu'))\nmodel_3.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_3.add(Flatten())\nmodel_3.add(Dense(512, activation='relu'))\nmodel_3.add(Dense(10, activation='softmax'))\n\nmodel_1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel_1.fit(X_train_2d, Y_train_c, epochs=30, batch_size=200)\nmodel_2.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel_2.fit(X_train_2d, Y_train_c, epochs=30, batch_size=200)\nmodel_3.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel_3.fit(X_train_2d, Y_train_c, epochs=30, batch_size=200)\n\nscores1 = model_1.evaluate(X_test_2d, Y_test_c)\nscores2 = model_2.evaluate(X_test_2d, Y_test_c)\nscores3 = model_3.evaluate(X_test_2d, Y_test_c)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(scores1)\nprint(scores2)\nprint(scores3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_2= Sequential()\nmodel_2.add(Conv2D(filters = 32, kernel_size = (5,5),activation ='relu', input_shape=(28,28,1)))\nmodel_2.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_2.add(Dropout(0.4))\nmodel_2.add(Conv2D(filters = 64, kernel_size = (5,5),activation ='relu'))\nmodel_2.add(MaxPooling2D(pool_size=(2, 2)))\nmodel_2.add(Dropout(0.4))\nmodel_2.add(Flatten())\nmodel_2.add(Dense(128, activation='relu'))\nmodel_2.add(Dropout(0.4))\nmodel_2.add(Dense(10, activation='softmax'))\n\nmodel_2.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\nmodel_2.fit(X_train_2d, Y_train_c, epochs=30, batch_size=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.image import ImageDataGenerator\n\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\ndatagen.fit(X_train_2d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model_2.fit_generator(datagen.flow(X_train_2d,Y_train_c, batch_size=200),\n                              epochs = 25, validation_data = (X_test_2d,Y_test_c),\n                              verbose = 2, steps_per_epoch=X_train.shape[0]/200)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores2 = model_2.evaluate(X_test_2d, Y_test_c)\nprint(scores2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"T = T_df.values\nT = T/255\nT = T.reshape(T.shape[0], 28, 28,1)\n\nresults = model_2.predict(T)\nresults = np.argmax(results,axis = 1)\nresults = pd.Series(results,name=\"Label\")\n\n\n\nsubmission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),results],axis = 1)\n\nsubmission.to_csv(\"submission_kaggle_2D.csv\",index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}