{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-20T19:23:40.361992Z","iopub.execute_input":"2021-06-20T19:23:40.362348Z","iopub.status.idle":"2021-06-20T19:23:40.373071Z","shell.execute_reply.started":"2021-06-20T19:23:40.362318Z","shell.execute_reply":"2021-06-20T19:23:40.371999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=pd.read_csv(\"../input/digit-recognizer/train.csv\")\ndf.head()\n#Each image is 28 pixels in height and 28 pixels in width, for a total of 784 pixels in total.","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:40.394040Z","iopub.execute_input":"2021-06-20T19:23:40.394400Z","iopub.status.idle":"2021-06-20T19:23:43.215588Z","shell.execute_reply.started":"2021-06-20T19:23:40.394370Z","shell.execute_reply":"2021-06-20T19:23:43.214499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=pd.read_csv(\"../input/digit-recognizer/test.csv\")\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:43.217125Z","iopub.execute_input":"2021-06-20T19:23:43.217411Z","iopub.status.idle":"2021-06-20T19:23:45.028585Z","shell.execute_reply.started":"2021-06-20T19:23:43.217383Z","shell.execute_reply":"2021-06-20T19:23:45.027515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:45.030377Z","iopub.execute_input":"2021-06-20T19:23:45.030676Z","iopub.status.idle":"2021-06-20T19:23:45.035760Z","shell.execute_reply.started":"2021-06-20T19:23:45.030645Z","shell.execute_reply":"2021-06-20T19:23:45.034802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we split training data into features and target\nX=df.drop(\"label\",axis=1).values\ny=df[\"label\"].values\n#we normalize the data\n","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:45.037711Z","iopub.execute_input":"2021-06-20T19:23:45.038388Z","iopub.status.idle":"2021-06-20T19:23:45.134459Z","shell.execute_reply.started":"2021-06-20T19:23:45.038340Z","shell.execute_reply":"2021-06-20T19:23:45.133349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X.shape)\nprint(y.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:45.136179Z","iopub.execute_input":"2021-06-20T19:23:45.136616Z","iopub.status.idle":"2021-06-20T19:23:45.150404Z","shell.execute_reply.started":"2021-06-20T19:23:45.136564Z","shell.execute_reply":"2021-06-20T19:23:45.149136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nplt.figure(figsize=(15,10))\nsns.set_style(\"darkgrid\")\nsns.countplot(x=\"label\",data=df)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:45.152268Z","iopub.execute_input":"2021-06-20T19:23:45.152753Z","iopub.status.idle":"2021-06-20T19:23:45.400757Z","shell.execute_reply.started":"2021-06-20T19:23:45.152691Z","shell.execute_reply":"2021-06-20T19:23:45.399610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.05, random_state=42)\nprint(X_train.shape)\nprint(X_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:45.402380Z","iopub.execute_input":"2021-06-20T19:23:45.402717Z","iopub.status.idle":"2021-06-20T19:23:45.820893Z","shell.execute_reply.started":"2021-06-20T19:23:45.402681Z","shell.execute_reply":"2021-06-20T19:23:45.819876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nerror_rate=list()\n#here we iterate meny different k values and plot their error rates \n#and discover which one is better than others and has the lowest error rate\nfor i in range(1,40):\n    knn=KNeighborsClassifier(n_neighbors=i)\n    knn.fit(X_train,y_train)\n    prediction_i=knn.predict(X_test)\n    error_rate.append(np.mean(prediction_i != y_test))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:23:45.823314Z","iopub.execute_input":"2021-06-20T19:23:45.823626Z","iopub.status.idle":"2021-06-20T19:26:13.092127Z","shell.execute_reply.started":"2021-06-20T19:23:45.823595Z","shell.execute_reply":"2021-06-20T19:26:13.091054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Now we will plot the prediction error rates of different k values\nplt.figure(figsize=(15,10))\nplt.plot(range(1,40),error_rate, color=\"blue\", linestyle=\"--\",marker=\"o\",markerfacecolor=\"red\",markersize=10)\nplt.title(\"Error Rate vs K Value\")\nplt.xlabel=\"K Value\"\nplt.ylabel(\"Error Rate\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:13.094162Z","iopub.execute_input":"2021-06-20T19:26:13.094489Z","iopub.status.idle":"2021-06-20T19:26:13.346445Z","shell.execute_reply.started":"2021-06-20T19:26:13.094453Z","shell.execute_reply":"2021-06-20T19:26:13.345741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn=KNeighborsClassifier(n_neighbors=1) # we get the minimum error when n=1","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:13.347913Z","iopub.execute_input":"2021-06-20T19:26:13.348581Z","iopub.status.idle":"2021-06-20T19:26:13.353573Z","shell.execute_reply.started":"2021-06-20T19:26:13.348533Z","shell.execute_reply":"2021-06-20T19:26:13.352256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn.fit(X_train,y_train)\nknn_predictions = knn.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:13.355104Z","iopub.execute_input":"2021-06-20T19:26:13.355522Z","iopub.status.idle":"2021-06-20T19:26:16.652268Z","shell.execute_reply.started":"2021-06-20T19:26:13.355478Z","shell.execute_reply":"2021-06-20T19:26:16.651133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report, confusion_matrix,accuracy_score\nprint(confusion_matrix(y_test,knn_predictions))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:16.653634Z","iopub.execute_input":"2021-06-20T19:26:16.653970Z","iopub.status.idle":"2021-06-20T19:26:16.663741Z","shell.execute_reply.started":"2021-06-20T19:26:16.653939Z","shell.execute_reply":"2021-06-20T19:26:16.662582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test, knn_predictions))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:16.664947Z","iopub.execute_input":"2021-06-20T19:26:16.665258Z","iopub.status.idle":"2021-06-20T19:26:16.684392Z","shell.execute_reply.started":"2021-06-20T19:26:16.665220Z","shell.execute_reply":"2021-06-20T19:26:16.682915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(accuracy_score(y_test, knn_predictions))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:16.686611Z","iopub.execute_input":"2021-06-20T19:26:16.686991Z","iopub.status.idle":"2021-06-20T19:26:16.692978Z","shell.execute_reply.started":"2021-06-20T19:26:16.686954Z","shell.execute_reply":"2021-06-20T19:26:16.691427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nrandom=RandomForestClassifier()\nrandom.fit(X_train,y_train)\nrandom_predictions= random.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:16.694531Z","iopub.execute_input":"2021-06-20T19:26:16.694938Z","iopub.status.idle":"2021-06-20T19:26:42.573267Z","shell.execute_reply.started":"2021-06-20T19:26:16.694902Z","shell.execute_reply":"2021-06-20T19:26:42.572202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test, random_predictions))\nprint(confusion_matrix(y_test,random_predictions))\nprint(accuracy_score(y_test, random_predictions)) #Random forest performs better than KNN","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:42.574579Z","iopub.execute_input":"2021-06-20T19:26:42.574880Z","iopub.status.idle":"2021-06-20T19:26:42.595314Z","shell.execute_reply.started":"2021-06-20T19:26:42.574851Z","shell.execute_reply":"2021-06-20T19:26:42.594342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\nbayes=GaussianNB()\nbayes.fit(X_train, y_train)\nbayes_predictions=bayes.predict(X_test)\nprint(classification_report(y_test, bayes_predictions))\nprint(confusion_matrix(y_test, bayes_predictions))\nprint(accuracy_score(y_test, bayes_predictions)) #The predictions are not good","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:42.596573Z","iopub.execute_input":"2021-06-20T19:26:42.596891Z","iopub.status.idle":"2021-06-20T19:26:43.200468Z","shell.execute_reply.started":"2021-06-20T19:26:42.596857Z","shell.execute_reply":"2021-06-20T19:26:43.199365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<font color=\"red\">\nLets try Convolutional Neural Networks:","metadata":{}},{"cell_type":"code","source":"X = X/255\ntest= test.values/255","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:43.203669Z","iopub.execute_input":"2021-06-20T19:26:43.204001Z","iopub.status.idle":"2021-06-20T19:26:43.381971Z","shell.execute_reply.started":"2021-06-20T19:26:43.203968Z","shell.execute_reply":"2021-06-20T19:26:43.380874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reshape image in 3 dimensions (height = 28px, width = 28px , canal = 1)\n# canal = 1 => For gray scale\nX = X.reshape(-1,28,28,1)\ntest = test.reshape(-1,28,28,1)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:43.383584Z","iopub.execute_input":"2021-06-20T19:26:43.384023Z","iopub.status.idle":"2021-06-20T19:26:43.390482Z","shell.execute_reply.started":"2021-06-20T19:26:43.383975Z","shell.execute_reply":"2021-06-20T19:26:43.389288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we encode labels to one hot vectors (like [0,0,1,0,0,0,0,0,0,0])\nfrom keras.utils.np_utils import to_categorical\ny = to_categorical(y)\n\nprint(f\"Label size {y.shape}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:43.391697Z","iopub.execute_input":"2021-06-20T19:26:43.391997Z","iopub.status.idle":"2021-06-20T19:26:43.407495Z","shell.execute_reply.started":"2021-06-20T19:26:43.391969Z","shell.execute_reply":"2021-06-20T19:26:43.406505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y, test_size=0.05, random_state=42)\nprint(X_train.shape)\nprint(X_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:43.408750Z","iopub.execute_input":"2021-06-20T19:26:43.409040Z","iopub.status.idle":"2021-06-20T19:26:43.830509Z","shell.execute_reply.started":"2021-06-20T19:26:43.409011Z","shell.execute_reply":"2021-06-20T19:26:43.829505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_visualization = X_train.reshape(X_train.shape[0], 28, 28)\n\nfig, axis = plt.subplots(1, 4, figsize=(20, 10))\nfor i, ax in enumerate(axis.flat):\n    ax.imshow(X_visualization[i], cmap='binary')\n    digit = y_train[i].argmax()\n    ax.set(title = f\"Real Number is {digit}\");\n# we see how our data look like.","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:43.831721Z","iopub.execute_input":"2021-06-20T19:26:43.832029Z","iopub.status.idle":"2021-06-20T19:26:44.874218Z","shell.execute_reply.started":"2021-06-20T19:26:43.831999Z","shell.execute_reply":"2021-06-20T19:26:44.873211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten,Dense, Dropout, BatchNormalization\nfrom tensorflow.keras.callbacks import EarlyStopping,  ReduceLROnPlateau\ncnn=Sequential()\n\n#model.add(Lambda(standardize,input_shape=(28,28,1)))    \ncnn.add(Conv2D(filters=64, kernel_size = (3,3), activation=\"relu\", input_shape=(28,28,1)))\ncnn.add(Conv2D(filters=64, kernel_size = (3,3), activation=\"relu\"))\n\ncnn.add(MaxPooling2D(pool_size=(2,2)))\ncnn.add(BatchNormalization())\ncnn.add(Conv2D(filters=128, kernel_size = (3,3), activation=\"relu\"))\ncnn.add(Conv2D(filters=128, kernel_size = (3,3), activation=\"relu\"))\n\ncnn.add(MaxPooling2D(pool_size=(2,2)))\ncnn.add(BatchNormalization())    \ncnn.add(Conv2D(filters=256, kernel_size = (3,3), activation=\"relu\"))\n    \ncnn.add(MaxPooling2D(pool_size=(2,2)))\n    \ncnn.add(Flatten())\ncnn.add(BatchNormalization())\ncnn.add(Dense(512,activation=\"relu\"))\n    \ncnn.add(Dense(10,activation=\"softmax\"))\n    \ncnn.compile(loss=\"categorical_crossentropy\", optimizer=\"adam\", metrics=[\"accuracy\"])","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:44.878773Z","iopub.execute_input":"2021-06-20T19:26:44.879145Z","iopub.status.idle":"2021-06-20T19:26:45.024212Z","shell.execute_reply.started":"2021-06-20T19:26:44.879114Z","shell.execute_reply":"2021-06-20T19:26:45.023120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# With data augmentation to prevent overfitting\nfrom keras.preprocessing.image import ImageDataGenerator\ndatagen = ImageDataGenerator(\n        featurewise_center=False,  # set input mean to 0 over the dataset\n        samplewise_center=False,  # set each sample mean to 0\n        featurewise_std_normalization=False,  # divide inputs by std of the dataset\n        samplewise_std_normalization=False,  # divide each input by its std\n        zca_whitening=False,  # apply ZCA whitening\n        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)\n        zoom_range = 0.1, # Randomly zoom image \n        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)\n        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)\n        horizontal_flip=False,  # randomly flip images\n        vertical_flip=False)  # randomly flip images\n\n\n#datagen.fit(X_train)\ntrain_generator = datagen.flow(X_train, y_train, batch_size=32)\ntest_generator = datagen.flow(X_test, y_test, batch_size=32)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:45.025723Z","iopub.execute_input":"2021-06-20T19:26:45.026022Z","iopub.status.idle":"2021-06-20T19:26:45.085815Z","shell.execute_reply.started":"2021-06-20T19:26:45.025992Z","shell.execute_reply":"2021-06-20T19:26:45.084705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":",\n,\n                              ","metadata":{}},{"cell_type":"code","source":"# Fit the model\nhistory = cnn.fit(train_generator, \n                  epochs = 10,\n                  steps_per_epoch = X_train.shape[0] // 32,\n                  validation_steps = X_test.shape[0] // 32,\n                  validation_data = test_generator)","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:26:45.087099Z","iopub.execute_input":"2021-06-20T19:26:45.087419Z","iopub.status.idle":"2021-06-20T19:53:24.063680Z","shell.execute_reply.started":"2021-06-20T19:26:45.087385Z","shell.execute_reply":"2021-06-20T19:53:24.062701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.set_style(\"darkgrid\")\npd.DataFrame(cnn.history.history).plot(figsize=(15,10))","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:53:24.065359Z","iopub.execute_input":"2021-06-20T19:53:24.065695Z","iopub.status.idle":"2021-06-20T19:53:24.317766Z","shell.execute_reply.started":"2021-06-20T19:53:24.065662Z","shell.execute_reply":"2021-06-20T19:53:24.316872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = cnn.predict(X_test)\nX_new = X_test.reshape(X_test.shape[0], 28, 28)\n\nfig, axis = plt.subplots(4, 4, figsize=(12, 14))\nfor i, ax in enumerate(axis.flat):\n    ax.imshow(X_new[i], cmap='binary')\n    ax.set(title = f\"Real Number is {y_test[i].argmax()}\\nPredict Number is {y_pred[i].argmax()}\");","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:57:46.339987Z","iopub.execute_input":"2021-06-20T19:57:46.340409Z","iopub.status.idle":"2021-06-20T19:57:50.467318Z","shell.execute_reply.started":"2021-06-20T19:57:46.340373Z","shell.execute_reply":"2021-06-20T19:57:50.466303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = cnn.predict(test, verbose=2)\npredictions","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:53:28.361951Z","iopub.execute_input":"2021-06-20T19:53:28.362396Z","iopub.status.idle":"2021-06-20T19:53:52.799398Z","shell.execute_reply.started":"2021-06-20T19:53:28.362349Z","shell.execute_reply":"2021-06-20T19:53:52.798497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_predictions =np.argmax(predictions, axis=1) # we get the original values instead of one hot coded version\nnew_predictions","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:53:52.800627Z","iopub.execute_input":"2021-06-20T19:53:52.800916Z","iopub.status.idle":"2021-06-20T19:53:52.807581Z","shell.execute_reply.started":"2021-06-20T19:53:52.800888Z","shell.execute_reply":"2021-06-20T19:53:52.806345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('../input/digit-recognizer/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:53:52.808920Z","iopub.execute_input":"2021-06-20T19:53:52.809486Z","iopub.status.idle":"2021-06-20T19:53:52.831815Z","shell.execute_reply.started":"2021-06-20T19:53:52.809450Z","shell.execute_reply":"2021-06-20T19:53:52.831057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:53:52.833116Z","iopub.execute_input":"2021-06-20T19:53:52.833599Z","iopub.status.idle":"2021-06-20T19:53:52.844731Z","shell.execute_reply.started":"2021-06-20T19:53:52.833565Z","shell.execute_reply":"2021-06-20T19:53:52.843578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['Label'] = new_predictions\nsubmission.to_csv(\"my_submission3.csv\", index=False)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-20T19:53:52.846229Z","iopub.execute_input":"2021-06-20T19:53:52.846854Z","iopub.status.idle":"2021-06-20T19:53:52.915367Z","shell.execute_reply.started":"2021-06-20T19:53:52.846807Z","shell.execute_reply":"2021-06-20T19:53:52.914292Z"},"trusted":true},"execution_count":null,"outputs":[]}]}