{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"colab":{"provenance":[],"gpuType":"T4"},"accelerator":"GPU","kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30804,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn import linear_model\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import KFold\nfrom sklearn import svm\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn import metrics\nimport gc\nimport cv2","metadata":{"id":"9MpAEepEqwdj","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:35.031534Z","iopub.execute_input":"2024-12-12T07:12:35.031841Z","iopub.status.idle":"2024-12-12T07:12:38.407512Z","shell.execute_reply.started":"2024-12-12T07:12:35.031809Z","shell.execute_reply":"2024-12-12T07:12:38.406370Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:38.409296Z","iopub.execute_input":"2024-12-12T07:12:38.409741Z","iopub.status.idle":"2024-12-12T07:12:38.417086Z","shell.execute_reply.started":"2024-12-12T07:12:38.409710Z","shell.execute_reply":"2024-12-12T07:12:38.416003Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Objective\n\nTo classify handwritten digits in 10 classes from 0 to 9 using the MNIST dataset.","metadata":{"id":"-frmb3TEtCMr"}},{"cell_type":"markdown","source":"# Loading Training Data","metadata":{"id":"m13p5zobtR4v"}},{"cell_type":"code","source":"digits = pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")\ndigits.head()","metadata":{"id":"vPngm5S4tNtS","outputId":"93f151b5-e827-4603-8975-c8349048a279","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:38.418394Z","iopub.execute_input":"2024-12-12T07:12:38.418705Z","iopub.status.idle":"2024-12-12T07:12:41.748495Z","shell.execute_reply.started":"2024-12-12T07:12:38.418675Z","shell.execute_reply":"2024-12-12T07:12:41.747412Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Data Understanding And Cleaning\n\nLet us understand the data and check if it requires cleaning and preprocessing.","metadata":{"id":"_uuufAj9tu10"}},{"cell_type":"code","source":"digits.info()","metadata":{"id":"epQH5oIWuCLP","outputId":"0c4b4d08-c180-469c-a88f-154839293224","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:41.750734Z","iopub.execute_input":"2024-12-12T07:12:41.751036Z","iopub.status.idle":"2024-12-12T07:12:41.794470Z","shell.execute_reply.started":"2024-12-12T07:12:41.751006Z","shell.execute_reply":"2024-12-12T07:12:41.793300Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"digits.shape","metadata":{"id":"06iQUTiouKXH","outputId":"53e4b335-b981-4fba-c36c-6d568eca934b","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:41.795579Z","iopub.execute_input":"2024-12-12T07:12:41.795905Z","iopub.status.idle":"2024-12-12T07:12:41.805202Z","shell.execute_reply.started":"2024-12-12T07:12:41.795841Z","shell.execute_reply":"2024-12-12T07:12:41.804177Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"There are 784 pixel values for each label. Let us visualize this in 28x28 form.","metadata":{"id":"Pr88DEgVudKD"}},{"cell_type":"code","source":"sample_digit4=digits.iloc[3,1:].values.reshape(28,28)\nplt.imshow(sample_digit4,cmap='gray')","metadata":{"id":"yh1CAmXiuNGj","outputId":"ed6cd1e9-eb4f-4d6d-d72f-17c0817b517d","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:41.806626Z","iopub.execute_input":"2024-12-12T07:12:41.807016Z","iopub.status.idle":"2024-12-12T07:12:42.061926Z","shell.execute_reply.started":"2024-12-12T07:12:41.806974Z","shell.execute_reply":"2024-12-12T07:12:42.060787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(sample_digit4)","metadata":{"id":"gwalCpOjvSSx","outputId":"a2d075a9-40a9-4666-f895-851e396caefe","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:42.063237Z","iopub.execute_input":"2024-12-12T07:12:42.063557Z","iopub.status.idle":"2024-12-12T07:12:42.070852Z","shell.execute_reply.started":"2024-12-12T07:12:42.063527Z","shell.execute_reply":"2024-12-12T07:12:42.069697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# checking proportions of each label in dataset\n\n100*(round(digits.label.astype('category').value_counts()/len(digits.index),4))","metadata":{"id":"UcLxYarRLNoF","outputId":"0e45be89-0f28-4a4d-fd89-4cb592f722f7","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:42.071939Z","iopub.execute_input":"2024-12-12T07:12:42.072251Z","iopub.status.idle":"2024-12-12T07:12:42.088820Z","shell.execute_reply.started":"2024-12-12T07:12:42.072222Z","shell.execute_reply":"2024-12-12T07:12:42.087826Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Each label has approximately 9-11% proportion and the daatset is nearly balanced.","metadata":{"id":"vbbuSpHhL5tq"}},{"cell_type":"markdown","source":"Checking nulls","metadata":{"id":"GhZVu4PdMNS7"}},{"cell_type":"code","source":"digits.isnull().sum()","metadata":{"id":"8IRL-m0PLwFg","outputId":"56d8d545-c603-4c27-8f31-d177680b1ba8","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:42.090049Z","iopub.execute_input":"2024-12-12T07:12:42.090395Z","iopub.status.idle":"2024-12-12T07:12:42.125881Z","shell.execute_reply.started":"2024-12-12T07:12:42.090365Z","shell.execute_reply":"2024-12-12T07:12:42.124819Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"digits.isnull().sum()[digits.isnull().sum()>0]","metadata":{"id":"LBmWxD7hMRE4","outputId":"68ecfcac-7268-4e08-c0f7-6e79e01f08ab","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:42.128906Z","iopub.execute_input":"2024-12-12T07:12:42.129250Z","iopub.status.idle":"2024-12-12T07:12:42.189808Z","shell.execute_reply.started":"2024-12-12T07:12:42.129218Z","shell.execute_reply":"2024-12-12T07:12:42.188764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(digits.isnull().sum()[digits.isnull().sum()>0])","metadata":{"id":"iUafujjENFg8","outputId":"c73a7df6-f876-493b-e59a-54d53dada8aa","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:42.190999Z","iopub.execute_input":"2024-12-12T07:12:42.191341Z","iopub.status.idle":"2024-12-12T07:12:42.252001Z","shell.execute_reply.started":"2024-12-12T07:12:42.191309Z","shell.execute_reply":"2024-12-12T07:12:42.250997Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# dropping rows having nulls in columns\ndigits2=digits.dropna()\ndigits2.shape","metadata":{"id":"Bqo-UX_lOtn_","outputId":"bad88e7e-6048-4e3e-b2fc-84e2747407b7","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:42.253189Z","iopub.execute_input":"2024-12-12T07:12:42.253668Z","iopub.status.idle":"2024-12-12T07:12:42.398002Z","shell.execute_reply.started":"2024-12-12T07:12:42.253625Z","shell.execute_reply":"2024-12-12T07:12:42.396927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"digits2.describe()","metadata":{"id":"rV-QemAIQdgu","outputId":"7ae77059-a15e-44f6-b812-445c7009fbd0","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:42.399060Z","iopub.execute_input":"2024-12-12T07:12:42.399363Z","iopub.status.idle":"2024-12-12T07:12:44.379241Z","shell.execute_reply.started":"2024-12-12T07:12:42.399335Z","shell.execute_reply":"2024-12-12T07:12:44.378179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"digits2.iloc[:,:392].describe()","metadata":{"id":"FTmlpE7sXXWV","outputId":"cbb004df-5632-4304-c28b-8726e68f58b1","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:44.380574Z","iopub.execute_input":"2024-12-12T07:12:44.380962Z","iopub.status.idle":"2024-12-12T07:12:45.295718Z","shell.execute_reply.started":"2024-12-12T07:12:44.380918Z","shell.execute_reply":"2024-12-12T07:12:45.294734Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"digits2.iloc[:,392:].describe()","metadata":{"id":"M224Ocy7YJgd","outputId":"af5e5f65-0aec-47dc-f23e-72cb049a9918","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:45.297032Z","iopub.execute_input":"2024-12-12T07:12:45.297476Z","iopub.status.idle":"2024-12-12T07:12:46.196540Z","shell.execute_reply.started":"2024-12-12T07:12:45.297433Z","shell.execute_reply":"2024-12-12T07:12:46.195329Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The pixels have values from 0 to 255. Most pixels appear in the lower ranges. The maximum value of some pixels is about 255. Also, the means of the pixels vary a lot from 0 to higher values like 121.\nDue to the wide variation of the pixels there is a need to scale the values.","metadata":{"id":"D0PVzFbuWwsl"}},{"cell_type":"markdown","source":"# Data Preparation","metadata":{"id":"zNaeCw0_vrtH"}},{"cell_type":"code","source":"digits=digits2","metadata":{"id":"R0HKVcS6v8W5","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:46.198479Z","iopub.execute_input":"2024-12-12T07:12:46.198901Z","iopub.status.idle":"2024-12-12T07:12:46.203479Z","shell.execute_reply.started":"2024-12-12T07:12:46.198856Z","shell.execute_reply":"2024-12-12T07:12:46.202355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Creating training and test sets\n# Splitting the data into train and test\nX = digits.iloc[:, 1:]\nY = digits.iloc[:, 0]","metadata":{"id":"oggkAU3pQ38g","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:46.204850Z","iopub.execute_input":"2024-12-12T07:12:46.205387Z","iopub.status.idle":"2024-12-12T07:12:46.216210Z","shell.execute_reply.started":"2024-12-12T07:12:46.205340Z","shell.execute_reply":"2024-12-12T07:12:46.215176Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Rescaling the features\n\n\n# Initializing the scaler\nscaler = StandardScaler()\n\n# Fiting the scaler on the training data and transform it\nX = scaler.fit_transform(X)\n\n# train test split with train_size=10% and test size=90%\nx_train, x_test, y_train, y_test = train_test_split(X, Y, train_size=0.10, random_state=101)\nprint(x_train.shape)\nprint(x_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)","metadata":{"id":"VCBcjEiwwBCr","outputId":"7d63f24e-fa61-4728-8902-7b941587ced8","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:46.217787Z","iopub.execute_input":"2024-12-12T07:12:46.218206Z","iopub.status.idle":"2024-12-12T07:12:47.205654Z","shell.execute_reply.started":"2024-12-12T07:12:46.218161Z","shell.execute_reply":"2024-12-12T07:12:47.204434Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Model Building","metadata":{"id":"_Ry4ZCov5nkr"}},{"cell_type":"markdown","source":"Model 1 - Fitting Linear Model","metadata":{"id":"P_WUqU6T5vpR"}},{"cell_type":"code","source":"\n\n# an initial SVM model with linear kernel\nsvm_linear = svm.SVC(kernel='linear')\n\n# fit\nsvm_linear.fit(x_train, y_train)","metadata":{"id":"yIX1b5qF5p5g","outputId":"a0546869-6a5b-41c1-a1ef-824ffed6775c","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:47.206832Z","iopub.execute_input":"2024-12-12T07:12:47.207228Z","iopub.status.idle":"2024-12-12T07:12:48.564990Z","shell.execute_reply.started":"2024-12-12T07:12:47.207161Z","shell.execute_reply":"2024-12-12T07:12:48.563941Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predict\npredictions = svm_linear.predict(x_test)","metadata":{"id":"Gv6QLy3n52Xe","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:12:48.566567Z","iopub.execute_input":"2024-12-12T07:12:48.566896Z","iopub.status.idle":"2024-12-12T07:13:03.396540Z","shell.execute_reply.started":"2024-12-12T07:12:48.566863Z","shell.execute_reply":"2024-12-12T07:13:03.395466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# measure accuracy\nmetrics.accuracy_score(y_true=y_test, y_pred=predictions)","metadata":{"id":"Adp_21hJ6Jr7","outputId":"8ddd5e87-037d-4704-db5a-2735ce29825e","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:13:03.397758Z","iopub.execute_input":"2024-12-12T07:13:03.398059Z","iopub.status.idle":"2024-12-12T07:13:03.408406Z","shell.execute_reply.started":"2024-12-12T07:13:03.398024Z","shell.execute_reply":"2024-12-12T07:13:03.407263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"confusion = metrics.confusion_matrix(y_true = y_test, y_pred = predictions)\nconfusion","metadata":{"id":"Fo3Bm5e7-2XN","outputId":"04baa112-fb8b-4972-e64e-870e2e9027ca","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:13:03.409684Z","iopub.execute_input":"2024-12-12T07:13:03.410139Z","iopub.status.idle":"2024-12-12T07:13:03.438208Z","shell.execute_reply.started":"2024-12-12T07:13:03.410074Z","shell.execute_reply":"2024-12-12T07:13:03.437145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# class-wise accuracy\nclass_wise = metrics.classification_report(y_true=y_test, y_pred=predictions)\nprint(class_wise)","metadata":{"id":"Si4PO7xE6QCg","outputId":"6f31b2e4-5a1b-4694-b7d0-c85287e31e1a","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:13:03.439482Z","iopub.execute_input":"2024-12-12T07:13:03.439809Z","iopub.status.idle":"2024-12-12T07:13:03.537481Z","shell.execute_reply.started":"2024-12-12T07:13:03.439777Z","shell.execute_reply":"2024-12-12T07:13:03.536182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gc.collect()","metadata":{"id":"tlor3-fh6T7k","outputId":"828879d0-55d5-4093-9511-45a612d9fc5e","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:13:03.538793Z","iopub.execute_input":"2024-12-12T07:13:03.539205Z","iopub.status.idle":"2024-12-12T07:13:03.633356Z","shell.execute_reply.started":"2024-12-12T07:13:03.539159Z","shell.execute_reply":"2024-12-12T07:13:03.632265Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Model 2 - Non linear Model","metadata":{"id":"xwGOJVJy6hS_"}},{"cell_type":"code","source":"# rbf kernel with other hyperparameters kept to default\nsvm_rbf = svm.SVC(kernel='rbf')\nsvm_rbf.fit(x_train, y_train)","metadata":{"id":"ESRQqmkN6fQO","outputId":"29e63de2-4ed8-4c03-ad42-8ea0aa8d54d3","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:13:03.634675Z","iopub.execute_input":"2024-12-12T07:13:03.635012Z","iopub.status.idle":"2024-12-12T07:13:06.753882Z","shell.execute_reply.started":"2024-12-12T07:13:03.634981Z","shell.execute_reply":"2024-12-12T07:13:06.752754Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predict\npredictions = svm_rbf.predict(x_test)\n\n# accuracy\nprint(metrics.accuracy_score(y_true=y_test, y_pred=predictions))","metadata":{"id":"u5YtIeS-7AuG","outputId":"5434082d-86ba-463c-fc80-a91308e52ef5","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:13:06.755441Z","iopub.execute_input":"2024-12-12T07:13:06.755920Z","iopub.status.idle":"2024-12-12T07:14:18.873611Z","shell.execute_reply.started":"2024-12-12T07:13:06.755825Z","shell.execute_reply":"2024-12-12T07:14:18.872612Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"The non linear model gave better accuracy. Now we will try to tune the non linear model using Grid Search CV.","metadata":{"id":"EN4Tq7Fk7hId"}},{"cell_type":"markdown","source":"Model 3 - Non Linear Model tuned with Grid Search CV","metadata":{"id":"jz82idP87-Dr"}},{"cell_type":"code","source":"# conduct (grid search) cross-validation to find the optimal values\n# of cost C and the choice of kernel\n\n\nfolds = KFold(n_splits = 5, shuffle = True, random_state = 3)\nparameters = {'C':[1, 10, 100],\n             'gamma': [1e-2, 1e-3, 1e-4]}\n\n# instantiate a model\nsvc_grid_search = svm.SVC(kernel=\"rbf\")\n\n# create a classifier to perform grid search\nclf = GridSearchCV(estimator=svc_grid_search, param_grid=parameters, scoring=\"accuracy\", cv=folds, return_train_score=True)\n\n# fit\nclf.fit(x_train, y_train)","metadata":{"id":"jAkmuPx67CH2","outputId":"6c9ecaa9-2529-4325-8e3d-381d4df381a5","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:14:18.874778Z","iopub.execute_input":"2024-12-12T07:14:18.875086Z","iopub.status.idle":"2024-12-12T07:21:52.398430Z","shell.execute_reply.started":"2024-12-12T07:14:18.875056Z","shell.execute_reply":"2024-12-12T07:21:52.397069Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# results\ncv_results = pd.DataFrame(clf.cv_results_)\ncv_results","metadata":{"id":"5eXbaUkz-LtQ","outputId":"43f5ccc1-277a-4d62-c65a-2b43a702f95f","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:21:52.404552Z","iopub.execute_input":"2024-12-12T07:21:52.404920Z","iopub.status.idle":"2024-12-12T07:21:52.438645Z","shell.execute_reply.started":"2024-12-12T07:21:52.404885Z","shell.execute_reply":"2024-12-12T07:21:52.437508Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# converting C to numeric type for plotting on x-axis\ncv_results['param_C'] = cv_results['param_C'].astype('int')\n\n# # plotting\nplt.figure(figsize=(16,6))\n\n# subplot 1/3\nplt.subplot(131)\ngamma_01 = cv_results[cv_results['param_gamma']==0.01]\n\nplt.plot(gamma_01[\"param_C\"], gamma_01[\"mean_test_score\"])\nplt.plot(gamma_01[\"param_C\"], gamma_01[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.01\")\nplt.ylim([0.60, 1])\nplt.legend(['test accuracy', 'train accuracy'], loc='lower right')\nplt.xscale('log')\n\n# subplot 2/3\nplt.subplot(132)\ngamma_001 = cv_results[cv_results['param_gamma']==0.001]\n\nplt.plot(gamma_001[\"param_C\"], gamma_001[\"mean_test_score\"])\nplt.plot(gamma_001[\"param_C\"], gamma_001[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.001\")\nplt.ylim([0.60, 1])\nplt.legend(['test accuracy', 'train accuracy'], loc='lower right')\nplt.xscale('log')\n\n\n# subplot 3/3\nplt.subplot(133)\ngamma_0001 = cv_results[cv_results['param_gamma']==0.0001]\n\nplt.plot(gamma_0001[\"param_C\"], gamma_0001[\"mean_test_score\"])\nplt.plot(gamma_0001[\"param_C\"], gamma_0001[\"mean_train_score\"])\nplt.xlabel('C')\nplt.ylabel('Accuracy')\nplt.title(\"Gamma=0.0001\")\nplt.ylim([0.60, 1])\nplt.legend(['test accuracy', 'train accuracy'], loc='lower right')\nplt.xscale('log')\n\nplt.show()","metadata":{"id":"B8xGsdHL-MgW","outputId":"1a9d193e-4b03-4d2f-c627-b55814e5bbc5","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:21:52.439880Z","iopub.execute_input":"2024-12-12T07:21:52.440298Z","iopub.status.idle":"2024-12-12T07:21:53.567344Z","shell.execute_reply.started":"2024-12-12T07:21:52.440250Z","shell.execute_reply":"2024-12-12T07:21:53.566186Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"From the plot above, we can observe that (from higher to lower gamma / left to right):\n\nAt very high gamma (0.01), the model is achieving 100% accuracy on the training data, though the test score is quite low (<75%).\n\nThus, the model is overfitting.\n\nAt gamma=0.001, the training and test scores are comparable at around C=1, though the model starts to overfit at higher values of C\n\nAt gamma=0.0001, the model does not overfit till C=10 but starts showing signs at C=100. Also, the training and test scores are slightly lower than at gamma=0.001.\n\nThus, it seems that the best combination is gamma=0.001 and C=1 (the plot in the middle), which gives the highest test accuracy (~92%) while avoiding overfitting.","metadata":{"id":"XMhK9oNI_9wi"}},{"cell_type":"code","source":"# optimal hyperparameters\nbest_C = 1\nbest_gamma = 0.001\n\n# model\nsvm_final = svm.SVC(kernel='rbf', C=best_C, gamma=best_gamma)\n\n# fit\nsvm_final.fit(x_train, y_train)","metadata":{"id":"tuZkME43FXIS","outputId":"3be3582f-10d9-4cf3-e5b7-6979716a03a6","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:21:53.568879Z","iopub.execute_input":"2024-12-12T07:21:53.569382Z","iopub.status.idle":"2024-12-12T07:21:56.200709Z","shell.execute_reply.started":"2024-12-12T07:21:53.569332Z","shell.execute_reply":"2024-12-12T07:21:56.199684Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# predict\npredictions = svm_final.predict(x_test)","metadata":{"id":"Ec7W_n1sALtY","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:21:56.202081Z","iopub.execute_input":"2024-12-12T07:21:56.202865Z","iopub.status.idle":"2024-12-12T07:22:55.990176Z","shell.execute_reply.started":"2024-12-12T07:21:56.202814Z","shell.execute_reply":"2024-12-12T07:22:55.988828Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# evaluation: CM\nconfusion = metrics.confusion_matrix(y_true = y_test, y_pred = predictions)\n\n# measure accuracy\ntest_accuracy = metrics.accuracy_score(y_true=y_test, y_pred=predictions)\n\nprint(test_accuracy, \"\\n\")\nprint(confusion)","metadata":{"id":"eh9pgJelAOaA","outputId":"0c691a99-2100-48bc-ecc7-b25b9c9ab0b8","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:22:55.991684Z","iopub.execute_input":"2024-12-12T07:22:55.992132Z","iopub.status.idle":"2024-12-12T07:22:56.012562Z","shell.execute_reply.started":"2024-12-12T07:22:55.992063Z","shell.execute_reply":"2024-12-12T07:22:56.011414Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Loading Test Data","metadata":{"id":"JHmcDjThAhma"}},{"cell_type":"code","source":"test=pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\ntest.head()","metadata":{"id":"uvCSCGK9Akh1","outputId":"ac707aac-0a9a-49e0-fbbe-6e9d8d0e5662","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:22:56.013775Z","iopub.execute_input":"2024-12-12T07:22:56.014090Z","iopub.status.idle":"2024-12-12T07:22:57.710241Z","shell.execute_reply.started":"2024-12-12T07:22:56.014059Z","shell.execute_reply":"2024-12-12T07:22:57.709235Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Applying transformation on test data","metadata":{"id":"_aPkMt7RFPXX"}},{"cell_type":"code","source":"test=scaler.transform(test)","metadata":{"id":"GMe3DZ_FFSmv","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:22:57.711474Z","iopub.execute_input":"2024-12-12T07:22:57.711781Z","iopub.status.idle":"2024-12-12T07:22:57.852792Z","shell.execute_reply.started":"2024-12-12T07:22:57.711751Z","shell.execute_reply":"2024-12-12T07:22:57.851648Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Predicting on Test Data","metadata":{"id":"qddWKgW1AxkJ"}},{"cell_type":"code","source":"predictiontest=svm_final.predict(test)","metadata":{"id":"FO2XIFBGA2jV","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:22:57.854068Z","iopub.execute_input":"2024-12-12T07:22:57.854427Z","iopub.status.idle":"2024-12-12T07:23:48.283489Z","shell.execute_reply.started":"2024-12-12T07:22:57.854393Z","shell.execute_reply":"2024-12-12T07:23:48.282442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result=pd.DataFrame({\n    'ImageId':range(1,len(test)+1),\n    'Label':predictiontest\n})","metadata":{"id":"BA0QhccxB6Cp","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:23:48.284877Z","iopub.execute_input":"2024-12-12T07:23:48.285338Z","iopub.status.idle":"2024-12-12T07:23:48.292442Z","shell.execute_reply.started":"2024-12-12T07:23:48.285294Z","shell.execute_reply":"2024-12-12T07:23:48.291205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(result)","metadata":{"id":"Lt-8iKbTCwnZ","outputId":"d7611e92-eb05-4f4b-9cb8-93b90451368f","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:23:48.293958Z","iopub.execute_input":"2024-12-12T07:23:48.294374Z","iopub.status.idle":"2024-12-12T07:23:48.308535Z","shell.execute_reply.started":"2024-12-12T07:23:48.294327Z","shell.execute_reply":"2024-12-12T07:23:48.307432Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"result.to_csv('digit_classifier_predictions_svm.csv', index=False)","metadata":{"id":"yKLXBP93Dc-i","trusted":true,"execution":{"iopub.status.busy":"2024-12-12T07:23:48.309990Z","iopub.execute_input":"2024-12-12T07:23:48.310451Z","iopub.status.idle":"2024-12-12T07:23:48.345669Z","shell.execute_reply.started":"2024-12-12T07:23:48.310393Z","shell.execute_reply":"2024-12-12T07:23:48.344591Z"}},"outputs":[],"execution_count":null}]}