{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import classification_report\n%matplotlib inline","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-15T09:26:17.695531Z","iopub.execute_input":"2022-09-15T09:26:17.695948Z","iopub.status.idle":"2022-09-15T09:26:18.395057Z","shell.execute_reply.started":"2022-09-15T09:26:17.695865Z","shell.execute_reply":"2022-09-15T09:26:18.394087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('../input/digit-recognizer/train.csv')\ntest_data = pd.read_csv('../input/digit-recognizer/test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:18.397033Z","iopub.execute_input":"2022-09-15T09:26:18.397394Z","iopub.status.idle":"2022-09-15T09:26:23.139487Z","shell.execute_reply.started":"2022-09-15T09:26:18.397359Z","shell.execute_reply":"2022-09-15T09:26:23.134141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:23.140844Z","iopub.execute_input":"2022-09-15T09:26:23.141245Z","iopub.status.idle":"2022-09-15T09:26:23.229769Z","shell.execute_reply.started":"2022-09-15T09:26:23.141205Z","shell.execute_reply":"2022-09-15T09:26:23.220282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:23.239898Z","iopub.execute_input":"2022-09-15T09:26:23.241581Z","iopub.status.idle":"2022-09-15T09:26:23.326007Z","shell.execute_reply.started":"2022-09-15T09:26:23.241542Z","shell.execute_reply":"2022-09-15T09:26:23.325107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plotting element at 3rd row of the train_data","metadata":{}},{"cell_type":"code","source":"sample_1 = train_data.iloc[3,1:].values\nsample_1 = sample_1.reshape(28,28).astype('uint8')\nplt.imshow(sample_1)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:23.329885Z","iopub.execute_input":"2022-09-15T09:26:23.338599Z","iopub.status.idle":"2022-09-15T09:26:23.672393Z","shell.execute_reply.started":"2022-09-15T09:26:23.338560Z","shell.execute_reply":"2022-09-15T09:26:23.671271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plotting element at 18th row of the train_data","metadata":{}},{"cell_type":"code","source":"sample_1 = train_data.iloc[18,1:].values\nsample_1 = sample_1.reshape(28,28).astype('uint8')\nplt.imshow(sample_1)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:23.676820Z","iopub.execute_input":"2022-09-15T09:26:23.679025Z","iopub.status.idle":"2022-09-15T09:26:23.957621Z","shell.execute_reply.started":"2022-09-15T09:26:23.678985Z","shell.execute_reply":"2022-09-15T09:26:23.956715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### separating labels and values from the train dataset","metadata":{}},{"cell_type":"code","source":"values_train_data = train_data.iloc[:,1:]\nlabels_train_data = train_data.iloc[:,0]","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:23.961822Z","iopub.execute_input":"2022-09-15T09:26:23.964015Z","iopub.status.idle":"2022-09-15T09:26:23.971660Z","shell.execute_reply.started":"2022-09-15T09:26:23.963977Z","shell.execute_reply":"2022-09-15T09:26:23.970768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating train and test size of the data","metadata":{}},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(values_train_data, labels_train_data, test_size= 0.6666666666666666, random_state = 1)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:23.973634Z","iopub.execute_input":"2022-09-15T09:26:23.974206Z","iopub.status.idle":"2022-09-15T09:26:24.360951Z","shell.execute_reply.started":"2022-09-15T09:26:23.974171Z","shell.execute_reply":"2022-09-15T09:26:24.359976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training with RandomForestClassifier","metadata":{}},{"cell_type":"code","source":"model_1 = RandomForestClassifier()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:24.362256Z","iopub.execute_input":"2022-09-15T09:26:24.363077Z","iopub.status.idle":"2022-09-15T09:26:24.367900Z","shell.execute_reply.started":"2022-09-15T09:26:24.363021Z","shell.execute_reply":"2022-09-15T09:26:24.366984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fitting the model with our data\nmodel_1.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:24.372599Z","iopub.execute_input":"2022-09-15T09:26:24.373435Z","iopub.status.idle":"2022-09-15T09:26:32.028035Z","shell.execute_reply.started":"2022-09-15T09:26:24.373401Z","shell.execute_reply":"2022-09-15T09:26:32.027093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predicting values with our model\npred_1 = model_1.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:32.029507Z","iopub.execute_input":"2022-09-15T09:26:32.029856Z","iopub.status.idle":"2022-09-15T09:26:32.888406Z","shell.execute_reply.started":"2022-09-15T09:26:32.029821Z","shell.execute_reply":"2022-09-15T09:26:32.887411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking Accuracy\nprint(classification_report(y_test,pred_1))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:32.889857Z","iopub.execute_input":"2022-09-15T09:26:32.890271Z","iopub.status.idle":"2022-09-15T09:26:32.958687Z","shell.execute_reply.started":"2022-09-15T09:26:32.890230Z","shell.execute_reply":"2022-09-15T09:26:32.957420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training with SVM ","metadata":{}},{"cell_type":"markdown","source":"## Importing the dependencies","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport seaborn as sns\nsns.set()\nfrom sklearn.preprocessing import scale\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import KFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import GridSearchCV","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:32.960473Z","iopub.execute_input":"2022-09-15T09:26:32.961134Z","iopub.status.idle":"2022-09-15T09:26:33.119075Z","shell.execute_reply.started":"2022-09-15T09:26:32.961094Z","shell.execute_reply":"2022-09-15T09:26:33.118076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading the training and test data","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:33.120409Z","iopub.execute_input":"2022-09-15T09:26:33.120782Z","iopub.status.idle":"2022-09-15T09:26:35.439011Z","shell.execute_reply.started":"2022-09-15T09:26:33.120745Z","shell.execute_reply":"2022-09-15T09:26:35.437956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:35.440780Z","iopub.execute_input":"2022-09-15T09:26:35.441195Z","iopub.status.idle":"2022-09-15T09:26:35.465984Z","shell.execute_reply.started":"2022-09-15T09:26:35.441157Z","shell.execute_reply":"2022-09-15T09:26:35.465055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Getting the data analytics","metadata":{}},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:35.467533Z","iopub.execute_input":"2022-09-15T09:26:35.467896Z","iopub.status.idle":"2022-09-15T09:26:35.476892Z","shell.execute_reply.started":"2022-09-15T09:26:35.467861Z","shell.execute_reply":"2022-09-15T09:26:35.475736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking datatype\ntrain.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:35.478501Z","iopub.execute_input":"2022-09-15T09:26:35.479000Z","iopub.status.idle":"2022-09-15T09:26:35.525943Z","shell.execute_reply.started":"2022-09-15T09:26:35.478966Z","shell.execute_reply":"2022-09-15T09:26:35.524925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe(percentiles = [0.05,0.10,0.25,0.50,0.75,0.90,0.99])","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:35.527360Z","iopub.execute_input":"2022-09-15T09:26:35.530168Z","iopub.status.idle":"2022-09-15T09:26:37.719747Z","shell.execute_reply.started":"2022-09-15T09:26:35.530138Z","shell.execute_reply":"2022-09-15T09:26:37.718496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Checking the unique enteries","metadata":{}},{"cell_type":"code","source":"np.unique(train['label'])","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:37.721142Z","iopub.execute_input":"2022-09-15T09:26:37.721578Z","iopub.status.idle":"2022-09-15T09:26:37.731556Z","shell.execute_reply.started":"2022-09-15T09:26:37.721539Z","shell.execute_reply":"2022-09-15T09:26:37.730446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['label'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:37.732754Z","iopub.execute_input":"2022-09-15T09:26:37.733096Z","iopub.status.idle":"2022-09-15T09:26:37.742871Z","shell.execute_reply.started":"2022-09-15T09:26:37.733030Z","shell.execute_reply":"2022-09-15T09:26:37.741758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing the label columns","metadata":{}},{"cell_type":"code","source":"sns.countplot(train['label'],palette = 'icefire')","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:37.744501Z","iopub.execute_input":"2022-09-15T09:26:37.745325Z","iopub.status.idle":"2022-09-15T09:26:38.001949Z","shell.execute_reply.started":"2022-09-15T09:26:37.745279Z","shell.execute_reply":"2022-09-15T09:26:38.000954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Getting more insights\ny = pd.value_counts(train.values.ravel()).sort_index()\nN = len(y)\nx = range(N)\nwidth =0.9\nplt.figure(figsize=[20,8])\nplt.bar(x, y, width, color=\"red\")\nplt.title('Pixel Value Frequency (Log Scale)')\nplt.yscale('log')\nplt.xlabel('Pixel Value (0-255)')\nplt.ylabel('Frequency')","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:38.003417Z","iopub.execute_input":"2022-09-15T09:26:38.004491Z","iopub.status.idle":"2022-09-15T09:26:39.454586Z","shell.execute_reply.started":"2022-09-15T09:26:38.004454Z","shell.execute_reply":"2022-09-15T09:26:39.453532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5,5))\nsns.distplot(train['pixel56'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:39.456205Z","iopub.execute_input":"2022-09-15T09:26:39.456818Z","iopub.status.idle":"2022-09-15T09:26:39.763029Z","shell.execute_reply.started":"2022-09-15T09:26:39.456778Z","shell.execute_reply":"2022-09-15T09:26:39.761987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5,5))\nsns.distplot(train['pixel656'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:39.764612Z","iopub.execute_input":"2022-09-15T09:26:39.764979Z","iopub.status.idle":"2022-09-15T09:26:40.255669Z","shell.execute_reply.started":"2022-09-15T09:26:39.764943Z","shell.execute_reply":"2022-09-15T09:26:40.254630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5,5))\nsns.distplot(train['pixel756'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:40.257311Z","iopub.execute_input":"2022-09-15T09:26:40.257955Z","iopub.status.idle":"2022-09-15T09:26:40.555905Z","shell.execute_reply.started":"2022-09-15T09:26:40.257915Z","shell.execute_reply":"2022-09-15T09:26:40.554990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5,5))\nsns.distplot(train['pixel685'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:40.557358Z","iopub.execute_input":"2022-09-15T09:26:40.557685Z","iopub.status.idle":"2022-09-15T09:26:41.055161Z","shell.execute_reply.started":"2022-09-15T09:26:40.557658Z","shell.execute_reply":"2022-09-15T09:26:41.054238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(5,5))\nsns.distplot(train['pixel684'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:41.056713Z","iopub.execute_input":"2022-09-15T09:26:41.057089Z","iopub.status.idle":"2022-09-15T09:26:41.550208Z","shell.execute_reply.started":"2022-09-15T09:26:41.057036Z","shell.execute_reply":"2022-09-15T09:26:41.549235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label v/s Pixel","metadata":{}},{"cell_type":"code","source":"sns.barplot(x='label', y='pixel693', data=train)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:41.557190Z","iopub.execute_input":"2022-09-15T09:26:41.558555Z","iopub.status.idle":"2022-09-15T09:26:42.434059Z","shell.execute_reply.started":"2022-09-15T09:26:41.558513Z","shell.execute_reply":"2022-09-15T09:26:42.433007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x='label', y='pixel753', data=train)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:42.436099Z","iopub.execute_input":"2022-09-15T09:26:42.436801Z","iopub.status.idle":"2022-09-15T09:26:43.325926Z","shell.execute_reply.started":"2022-09-15T09:26:42.436761Z","shell.execute_reply":"2022-09-15T09:26:43.324953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x='label', y='pixel683', data=train)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:43.327338Z","iopub.execute_input":"2022-09-15T09:26:43.328388Z","iopub.status.idle":"2022-09-15T09:26:44.217836Z","shell.execute_reply.started":"2022-09-15T09:26:43.328341Z","shell.execute_reply":"2022-09-15T09:26:44.216854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.barplot(x='label', y='pixel576', data=train)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:44.219418Z","iopub.execute_input":"2022-09-15T09:26:44.220024Z","iopub.status.idle":"2022-09-15T09:26:45.466068Z","shell.execute_reply.started":"2022-09-15T09:26:44.219984Z","shell.execute_reply":"2022-09-15T09:26:45.465149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizng some samples","metadata":{}},{"cell_type":"code","source":"one = train.iloc[2, 1:]\none = one.values.reshape(28,28)\nplt.imshow(one)\nplt.title(\"Digit 1\")","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:45.467843Z","iopub.execute_input":"2022-09-15T09:26:45.468632Z","iopub.status.idle":"2022-09-15T09:26:45.713023Z","shell.execute_reply.started":"2022-09-15T09:26:45.468594Z","shell.execute_reply":"2022-09-15T09:26:45.711968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nine = train.iloc[11, 1:]\nnine = nine.values.reshape(28,28)\nplt.imshow(nine)\nplt.title(\"Digit 9\")","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:45.714402Z","iopub.execute_input":"2022-09-15T09:26:45.714958Z","iopub.status.idle":"2022-09-15T09:26:45.962497Z","shell.execute_reply.started":"2022-09-15T09:26:45.714920Z","shell.execute_reply":"2022-09-15T09:26:45.961599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_no = train.iloc[41198, 1:]\nsample_no = sample_no.values.reshape(28,28)\nplt.imshow(sample_no)\nplt.title(\"Digit 3\")","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:45.963785Z","iopub.execute_input":"2022-09-15T09:26:45.964849Z","iopub.status.idle":"2022-09-15T09:26:46.216913Z","shell.execute_reply.started":"2022-09-15T09:26:45.964811Z","shell.execute_reply":"2022-09-15T09:26:46.215923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualizing the heatmap","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,15))\nsns.heatmap(data=train.corr(),annot=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:26:46.219139Z","iopub.execute_input":"2022-09-15T09:26:46.219956Z","iopub.status.idle":"2022-09-15T09:27:51.455415Z","shell.execute_reply.started":"2022-09-15T09:26:46.219916Z","shell.execute_reply":"2022-09-15T09:27:51.454132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation and preprocessing","metadata":{}},{"cell_type":"code","source":"pd.set_option('display.max_rows', 999)\nround(train.drop('label', axis=1).mean(), 2).sort_values(ascending = False)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:27:51.456547Z","iopub.execute_input":"2022-09-15T09:27:51.456921Z","iopub.status.idle":"2022-09-15T09:27:51.614122Z","shell.execute_reply.started":"2022-09-15T09:27:51.456892Z","shell.execute_reply":"2022-09-15T09:27:51.612997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Average ie mean varies between 0 to 140. So, we Scale them","metadata":{}},{"cell_type":"code","source":"# splitting into X and y\nX = train.drop(\"label\", axis = 1)\ny = train['label']","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:27:51.615670Z","iopub.execute_input":"2022-09-15T09:27:51.616150Z","iopub.status.idle":"2022-09-15T09:27:51.700254Z","shell.execute_reply.started":"2022-09-15T09:27:51.616103Z","shell.execute_reply":"2022-09-15T09:27:51.699027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaling the features\nX_scaled = scale(X)\n\n# train test split\nX_train, X_test, y_train, y_test = train_test_split(X_scaled, y, train_size=0.2,test_size = 0.8, random_state = 101)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:27:51.702023Z","iopub.execute_input":"2022-09-15T09:27:51.702514Z","iopub.status.idle":"2022-09-15T09:27:53.070947Z","shell.execute_reply.started":"2022-09-15T09:27:51.702474Z","shell.execute_reply":"2022-09-15T09:27:53.069950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('X_train shape:',X_train.shape)\nprint('y_train shape:',y_train.shape)\nprint('X_test shape:',X_test.shape)\nprint('y_test shape:',y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:27:53.072226Z","iopub.execute_input":"2022-09-15T09:27:53.072594Z","iopub.status.idle":"2022-09-15T09:27:53.079122Z","shell.execute_reply.started":"2022-09-15T09:27:53.072551Z","shell.execute_reply":"2022-09-15T09:27:53.078018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training with SVC - Linear Model","metadata":{}},{"cell_type":"code","source":"model_linear = SVC(kernel='linear')\nmodel_linear.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:27:53.080978Z","iopub.execute_input":"2022-09-15T09:27:53.081830Z","iopub.status.idle":"2022-09-15T09:27:57.065011Z","shell.execute_reply.started":"2022-09-15T09:27:53.081793Z","shell.execute_reply":"2022-09-15T09:27:57.064003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = model_linear.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:27:57.066642Z","iopub.execute_input":"2022-09-15T09:27:57.067066Z","iopub.status.idle":"2022-09-15T09:28:19.003537Z","shell.execute_reply.started":"2022-09-15T09:27:57.067012Z","shell.execute_reply":"2022-09-15T09:28:19.002310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test,y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:28:19.005096Z","iopub.execute_input":"2022-09-15T09:28:19.005469Z","iopub.status.idle":"2022-09-15T09:28:19.082973Z","shell.execute_reply.started":"2022-09-15T09:28:19.005430Z","shell.execute_reply":"2022-09-15T09:28:19.081897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics\nprint(\"accuracy:\", metrics.accuracy_score(y_true=y_test, y_pred=y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:28:19.084745Z","iopub.execute_input":"2022-09-15T09:28:19.085258Z","iopub.status.idle":"2022-09-15T09:28:19.095011Z","shell.execute_reply.started":"2022-09-15T09:28:19.085211Z","shell.execute_reply":"2022-09-15T09:28:19.093735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## So, we got an accuracy of 91.3125% using SVC - Linear Model","metadata":{}},{"cell_type":"markdown","source":"## Viewing the Confusion Matrix","metadata":{}},{"cell_type":"code","source":"print(metrics.confusion_matrix(y_true=y_test, y_pred=y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:28:19.097497Z","iopub.execute_input":"2022-09-15T09:28:19.098720Z","iopub.status.idle":"2022-09-15T09:28:19.116075Z","shell.execute_reply.started":"2022-09-15T09:28:19.098684Z","shell.execute_reply":"2022-09-15T09:28:19.114977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training with SVC Non-Linear Models","metadata":{}},{"cell_type":"markdown","source":"## (A) - Training with **Poly Kernel**","metadata":{}},{"cell_type":"code","source":"poly_kern = SVC(kernel='poly')\npoly_kern.fit(X_train, y_train)\ny_pred_poly_kern = poly_kern.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:28:19.117518Z","iopub.execute_input":"2022-09-15T09:28:19.119592Z","iopub.status.idle":"2022-09-15T09:29:59.453084Z","shell.execute_reply.started":"2022-09-15T09:28:19.119555Z","shell.execute_reply":"2022-09-15T09:29:59.451873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Viewing the **Accuracy, Confusion metrics and the Classification Report**","metadata":{}},{"cell_type":"code","source":"print(classification_report(y_test,y_pred_poly_kern))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:29:59.454625Z","iopub.execute_input":"2022-09-15T09:29:59.455261Z","iopub.status.idle":"2022-09-15T09:29:59.532446Z","shell.execute_reply.started":"2022-09-15T09:29:59.455221Z","shell.execute_reply":"2022-09-15T09:29:59.531256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"accuracy:\", metrics.accuracy_score(y_true=y_test, y_pred=y_pred_poly_kern))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:29:59.534138Z","iopub.execute_input":"2022-09-15T09:29:59.534531Z","iopub.status.idle":"2022-09-15T09:29:59.543163Z","shell.execute_reply.started":"2022-09-15T09:29:59.534494Z","shell.execute_reply":"2022-09-15T09:29:59.541856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.confusion_matrix(y_true=y_test, y_pred=y_pred_poly_kern))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:29:59.544759Z","iopub.execute_input":"2022-09-15T09:29:59.546008Z","iopub.status.idle":"2022-09-15T09:29:59.561152Z","shell.execute_reply.started":"2022-09-15T09:29:59.545966Z","shell.execute_reply":"2022-09-15T09:29:59.560104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## So,we got a Accuracy of 88.226% using Poly Kernel","metadata":{}},{"cell_type":"markdown","source":"## The accuracy dropped from 91.3125% to 88.226%. So, there is no use in going on with the Poly Kernel","metadata":{}},{"cell_type":"markdown","source":"## (B) - Training with **rbf kernel**","metadata":{}},{"cell_type":"code","source":"rbf_ker_mod = SVC(kernel='rbf')\nrbf_ker_mod.fit(X_train, y_train)\ny_pred_rbf_ker_mod = rbf_ker_mod.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:29:59.562636Z","iopub.execute_input":"2022-09-15T09:29:59.563294Z","iopub.status.idle":"2022-09-15T09:31:46.591909Z","shell.execute_reply.started":"2022-09-15T09:29:59.563257Z","shell.execute_reply":"2022-09-15T09:31:46.590894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Viewing the Accuracy, Confusion Matrix and Classification Report","metadata":{}},{"cell_type":"code","source":"print(classification_report(y_test,y_pred_rbf_ker_mod))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.593533Z","iopub.execute_input":"2022-09-15T09:31:46.593914Z","iopub.status.idle":"2022-09-15T09:31:46.671945Z","shell.execute_reply.started":"2022-09-15T09:31:46.593877Z","shell.execute_reply":"2022-09-15T09:31:46.670769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"accuracy:\", metrics.accuracy_score(y_true=y_test, y_pred=y_pred_rbf_ker_mod))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.673679Z","iopub.execute_input":"2022-09-15T09:31:46.674026Z","iopub.status.idle":"2022-09-15T09:31:46.682354Z","shell.execute_reply.started":"2022-09-15T09:31:46.673998Z","shell.execute_reply":"2022-09-15T09:31:46.681298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.confusion_matrix(y_true=y_test, y_pred=y_pred_rbf_ker_mod))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.684358Z","iopub.execute_input":"2022-09-15T09:31:46.684938Z","iopub.status.idle":"2022-09-15T09:31:46.698973Z","shell.execute_reply.started":"2022-09-15T09:31:46.684905Z","shell.execute_reply":"2022-09-15T09:31:46.698107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## So, we got a Accuracy of 93.961% using SVC Non Linear - rbf kernel","metadata":{}},{"cell_type":"markdown","source":"### The Accuracy jumped from 91.312% to 93.961% \n### So, we will be using SVC Non Linear - rbf kernel for training, also the precision values are high \n### i.e. above 90%(0.90) in almost all cases in rbf kernel","metadata":{}},{"cell_type":"markdown","source":"## **GRID SEARCH : HyperParameter Tuning**","metadata":{}},{"cell_type":"markdown","source":"#### Creating a Kfold object with 5 splits","metadata":{}},{"cell_type":"code","source":"folds = KFold(n_splits = 5, shuffle = True, random_state = 101)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.700554Z","iopub.execute_input":"2022-09-15T09:31:46.701284Z","iopub.status.idle":"2022-09-15T09:31:46.706076Z","shell.execute_reply.started":"2022-09-15T09:31:46.701249Z","shell.execute_reply":"2022-09-15T09:31:46.705082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# specifying range of hyperparameters\n# Setting the parameters by cross-validation\nhyper_params = [ {'gamma': [0.01, 0.001,0.0001],\n                     'C': [1, 10, 100]}]\n","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.707537Z","iopub.execute_input":"2022-09-15T09:31:46.708156Z","iopub.status.idle":"2022-09-15T09:31:46.715015Z","shell.execute_reply.started":"2022-09-15T09:31:46.708117Z","shell.execute_reply":"2022-09-15T09:31:46.714008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_model = SVC(kernel=\"rbf\")","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.717120Z","iopub.execute_input":"2022-09-15T09:31:46.717429Z","iopub.status.idle":"2022-09-15T09:31:46.726410Z","shell.execute_reply.started":"2022-09-15T09:31:46.717403Z","shell.execute_reply":"2022-09-15T09:31:46.725506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Setting up GridSearchCV()","metadata":{}},{"cell_type":"code","source":"model_cv = GridSearchCV(estimator = final_model, \n                        param_grid = hyper_params, \n                        scoring= 'accuracy', \n                        cv = folds, \n                        verbose = 1,\n                        return_train_score=True,n_jobs = -1)  ","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.727987Z","iopub.execute_input":"2022-09-15T09:31:46.728665Z","iopub.status.idle":"2022-09-15T09:31:46.737503Z","shell.execute_reply.started":"2022-09-15T09:31:46.728623Z","shell.execute_reply":"2022-09-15T09:31:46.736487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_cv.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:31:46.739180Z","iopub.execute_input":"2022-09-15T09:31:46.739905Z","iopub.status.idle":"2022-09-15T09:56:00.143352Z","shell.execute_reply.started":"2022-09-15T09:31:46.739870Z","shell.execute_reply":"2022-09-15T09:56:00.142051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cv_results = pd.DataFrame(model_cv.cv_results_)\ncv_results","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:58:33.629281Z","iopub.execute_input":"2022-09-15T09:58:33.629545Z","iopub.status.idle":"2022-09-15T09:58:33.661995Z","shell.execute_reply.started":"2022-09-15T09:58:33.629521Z","shell.execute_reply":"2022-09-15T09:58:33.661103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# converting C to numeric type for plotting on x-axis\ncv_results['param_C'] = cv_results['param_C'].astype('int')\n\n# # plotting\nplt.figure(figsize=(20,7))\n\n# subplot 1/3\nplt.subplot(131)\ngamma_01 = cv_results[cv_results['param_gamma']==0.01]\n\nplt.plot(gamma_01[\"param_C\"], gamma_01[\"mean_test_score\"])\nplt.plot(gamma_01[\"param_C\"], gamma_01[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.01\")\nplt.ylim([0.50, 1.1])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')\n\nplt.subplot(132)\ngamma_001 = cv_results[cv_results['param_gamma']==0.001]\n\nplt.plot(gamma_001[\"param_C\"], gamma_001[\"mean_test_score\"])\nplt.plot(gamma_001[\"param_C\"], gamma_001[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.001\")\nplt.ylim([0.50, 1.1])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')\n\nplt.subplot(133)\ngamma_0001 = cv_results[cv_results['param_gamma']==0.0001]\n\nplt.plot(gamma_0001[\"param_C\"], gamma_0001[\"mean_test_score\"])\nplt.plot(gamma_0001[\"param_C\"], gamma_0001[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.0001\")\nplt.ylim([0.50, 1.1])\nplt.legend(['test accuracy', 'train accuracy'], loc='upper left')\nplt.xscale('log')","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:56:00.181563Z","iopub.execute_input":"2022-09-15T09:56:00.181937Z","iopub.status.idle":"2022-09-15T09:56:01.712914Z","shell.execute_reply.started":"2022-09-15T09:56:00.181900Z","shell.execute_reply":"2022-09-15T09:56:01.711963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# printing the optimal accuracy score and hyperparameters\nbest_score = model_cv.best_score_\nbest_hyperparams = model_cv.best_params_\n\nprint(\"The best test score is {0} corresponding to hyperparameters {1}\".format(best_score, best_hyperparams))","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:56:01.714715Z","iopub.execute_input":"2022-09-15T09:56:01.715413Z","iopub.status.idle":"2022-09-15T09:56:01.721747Z","shell.execute_reply.started":"2022-09-15T09:56:01.715373Z","shell.execute_reply":"2022-09-15T09:56:01.720737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Building and Evaluating Final model","metadata":{}},{"cell_type":"code","source":"rbf_final_model = SVC(C=10, gamma=0.001, kernel=\"rbf\")\n\nrbf_final_model.fit(X_train, y_train)\ny_pred = rbf_final_model.predict(X_test)\n\n# metrics\nprint(\"accuracy\", metrics.accuracy_score(y_test, y_pred), \"\\n\")\nprint(metrics.confusion_matrix(y_test, y_pred), \"\\n\")\nprint(classification_report(y_test,y_pred)) ","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:56:01.724085Z","iopub.execute_input":"2022-09-15T09:56:01.724748Z","iopub.status.idle":"2022-09-15T09:57:23.873476Z","shell.execute_reply.started":"2022-09-15T09:56:01.724711Z","shell.execute_reply":"2022-09-15T09:57:23.872328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualzing data on unseen data","metadata":{}},{"cell_type":"code","source":"df = np.random.randint(1,y_pred.shape[0]+1,5)\n\nplt.figure(figsize=(16,4))\nfor i,j in enumerate(df):\n    plt.subplot(150+i+1)\n    d = X_test[j].reshape(28,28)\n    plt.title(f'Predicted Label: {y_pred[j]}')\n    plt.imshow(d)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:57:23.874895Z","iopub.execute_input":"2022-09-15T09:57:23.875531Z","iopub.status.idle":"2022-09-15T09:57:24.491027Z","shell.execute_reply.started":"2022-09-15T09:57:23.875491Z","shell.execute_reply":"2022-09-15T09:57:24.489966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Using Final Model on Test.csv","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('../input/digit-recognizer/test.csv')\ntest_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:57:24.492642Z","iopub.execute_input":"2022-09-15T09:57:24.493180Z","iopub.status.idle":"2022-09-15T09:57:25.684695Z","shell.execute_reply.started":"2022-09-15T09:57:24.493143Z","shell.execute_reply":"2022-09-15T09:57:25.683804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:57:25.685973Z","iopub.execute_input":"2022-09-15T09:57:25.687097Z","iopub.status.idle":"2022-09-15T09:57:25.693909Z","shell.execute_reply.started":"2022-09-15T09:57:25.687058Z","shell.execute_reply":"2022-09-15T09:57:25.692777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:57:25.695412Z","iopub.execute_input":"2022-09-15T09:57:25.696600Z","iopub.status.idle":"2022-09-15T09:57:25.737746Z","shell.execute_reply.started":"2022-09-15T09:57:25.696564Z","shell.execute_reply":"2022-09-15T09:57:25.736592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaling the features\ntest_scaled = scale(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:57:25.739340Z","iopub.execute_input":"2022-09-15T09:57:25.740003Z","iopub.status.idle":"2022-09-15T09:57:26.390337Z","shell.execute_reply.started":"2022-09-15T09:57:25.739968Z","shell.execute_reply":"2022-09-15T09:57:26.389341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predict = rbf_final_model.predict(test_scaled)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:57:26.391891Z","iopub.execute_input":"2022-09-15T09:57:26.392279Z","iopub.status.idle":"2022-09-15T09:58:32.447135Z","shell.execute_reply.started":"2022-09-15T09:57:26.392240Z","shell.execute_reply":"2022-09-15T09:58:32.446082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing the Distribution of the Prediction","metadata":{}},{"cell_type":"code","source":"visual = {'ImageId': np.arange(1,test_predict.shape[0]+1), 'Label': test_predict}\ndata_to_export = pd.DataFrame(visual)\nsns.countplot(data_to_export['Label'], palette = 'icefire')","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:58:32.448828Z","iopub.execute_input":"2022-09-15T09:58:32.449238Z","iopub.status.idle":"2022-09-15T09:58:32.710977Z","shell.execute_reply.started":"2022-09-15T09:58:32.449199Z","shell.execute_reply":"2022-09-15T09:58:32.710086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Visualizing some predicted Numbers","metadata":{}},{"cell_type":"code","source":"df = np.random.randint(1,test_predict.shape[0]+1,5)\n\nplt.figure(figsize=(16,4))\nfor i,j in enumerate(df):\n    plt.subplot(150+i+1)\n    d = test_scaled[j].reshape(28,28)\n    plt.title(f'Predicted Label: {test_predict[j]}')\n    plt.imshow(d)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:58:32.713074Z","iopub.execute_input":"2022-09-15T09:58:32.713763Z","iopub.status.idle":"2022-09-15T09:58:33.538918Z","shell.execute_reply.started":"2022-09-15T09:58:32.713725Z","shell.execute_reply":"2022-09-15T09:58:33.537970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exporting the predicted values into a .csv files","metadata":{}},{"cell_type":"code","source":"data_to_export.to_csv(path_or_buf='submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-15T09:58:33.540479Z","iopub.execute_input":"2022-09-15T09:58:33.540858Z","iopub.status.idle":"2022-09-15T09:58:33.572750Z","shell.execute_reply.started":"2022-09-15T09:58:33.540822Z","shell.execute_reply":"2022-09-15T09:58:33.571811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}