{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-10T07:39:42.260969Z","iopub.execute_input":"2023-01-10T07:39:42.261604Z","iopub.status.idle":"2023-01-10T07:39:42.291932Z","shell.execute_reply.started":"2023-01-10T07:39:42.261522Z","shell.execute_reply":"2023-01-10T07:39:42.290953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn import svm\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import classification_report, confusion_matrix","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:39:42.293833Z","iopub.execute_input":"2023-01-10T07:39:42.294217Z","iopub.status.idle":"2023-01-10T07:39:43.329911Z","shell.execute_reply.started":"2023-01-10T07:39:42.294179Z","shell.execute_reply":"2023-01-10T07:39:43.328761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"# use to train model\ndf_train = pd.read_csv('/kaggle/input/Kannada-MNIST/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:39:43.335783Z","iopub.execute_input":"2023-01-10T07:39:43.338295Z","iopub.status.idle":"2023-01-10T07:39:50.239599Z","shell.execute_reply.started":"2023-01-10T07:39:43.338257Z","shell.execute_reply":"2023-01-10T07:39:50.238533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use to predict to make submission\ndf_test = pd.read_csv('/kaggle/input/Kannada-MNIST/test.csv')\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:39:50.242325Z","iopub.execute_input":"2023-01-10T07:39:50.242992Z","iopub.status.idle":"2023-01-10T07:39:50.639464Z","shell.execute_reply.started":"2023-01-10T07:39:50.242953Z","shell.execute_reply":"2023-01-10T07:39:50.638494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prepare data to train","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(df_train.drop('label', axis=1), df_train['label'], test_size = 0.30)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:39:50.641069Z","iopub.execute_input":"2023-01-10T07:39:50.641720Z","iopub.status.idle":"2023-01-10T07:39:51.144597Z","shell.execute_reply.started":"2023-01-10T07:39:50.641683Z","shell.execute_reply":"2023-01-10T07:39:51.143549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Because the number just take a few pixels, i will use pca to reduce feature for faster training","metadata":{}},{"cell_type":"code","source":"# save 70% features\npca = PCA(n_components=0.7,whiten=True)\nX_train_PCA = pca.fit_transform(X_train)\nX_test_PCA = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T08:02:03.605582Z","iopub.execute_input":"2023-01-10T08:02:03.606277Z","iopub.status.idle":"2023-01-10T08:02:12.334341Z","shell.execute_reply.started":"2023-01-10T08:02:03.606241Z","shell.execute_reply":"2023-01-10T08:02:12.332929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train\n### I will train by SVM algortihm with 3 different kernels: linear, poly and Gaussian","metadata":{}},{"cell_type":"markdown","source":"### Linear kernel","metadata":{}},{"cell_type":"code","source":"svc_linear = svm.SVC(kernel='linear')\nsvc_linear.fit(X_train_PCA, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:40:01.182765Z","iopub.execute_input":"2023-01-10T07:40:01.183154Z","iopub.status.idle":"2023-01-10T07:40:17.408478Z","shell.execute_reply.started":"2023-01-10T07:40:01.183115Z","shell.execute_reply":"2023-01-10T07:40:17.407525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = svc_linear.predict(X_test_PCA)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:40:17.409908Z","iopub.execute_input":"2023-01-10T07:40:17.410268Z","iopub.status.idle":"2023-01-10T07:40:21.118883Z","shell.execute_reply.started":"2023-01-10T07:40:17.410240Z","shell.execute_reply":"2023-01-10T07:40:21.117806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(confusion_matrix(y_test,y_pred))\nprint(classification_report(y_test,y_pred))\n\nlin_score = accuracy_score(y_pred, y_test)\nprint(\"Linear Kernel score:\", lin_score)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:40:21.122935Z","iopub.execute_input":"2023-01-10T07:40:21.123880Z","iopub.status.idle":"2023-01-10T07:40:21.196824Z","shell.execute_reply.started":"2023-01-10T07:40:21.123833Z","shell.execute_reply":"2023-01-10T07:40:21.195688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Poly kernel","metadata":{}},{"cell_type":"code","source":"svc_poly = svm.SVC(kernel='poly')\nsvc_poly.fit(X_train_PCA, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:40:21.203524Z","iopub.execute_input":"2023-01-10T07:40:21.203908Z","iopub.status.idle":"2023-01-10T07:41:07.833538Z","shell.execute_reply.started":"2023-01-10T07:40:21.203871Z","shell.execute_reply":"2023-01-10T07:41:07.832335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = svc_poly.predict(X_test_PCA)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:41:07.835160Z","iopub.execute_input":"2023-01-10T07:41:07.835547Z","iopub.status.idle":"2023-01-10T07:41:19.851942Z","shell.execute_reply.started":"2023-01-10T07:41:07.835511Z","shell.execute_reply":"2023-01-10T07:41:19.850794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(confusion_matrix(y_test, y_pred))\nprint(classification_report(y_test, y_pred))\n\npoly_score = accuracy_score(y_pred, y_test)\nprint(\"Poly Kernel score:\", poly_score)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:41:19.853775Z","iopub.execute_input":"2023-01-10T07:41:19.854209Z","iopub.status.idle":"2023-01-10T07:41:19.903581Z","shell.execute_reply.started":"2023-01-10T07:41:19.854171Z","shell.execute_reply":"2023-01-10T07:41:19.902386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Gaussian Kernel","metadata":{}},{"cell_type":"code","source":"svc_gaus = svm.SVC(kernel='rbf')\nsvc_gaus.fit(X_train_PCA, y_train)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:41:19.905421Z","iopub.execute_input":"2023-01-10T07:41:19.906031Z","iopub.status.idle":"2023-01-10T07:41:47.782262Z","shell.execute_reply.started":"2023-01-10T07:41:19.905989Z","shell.execute_reply":"2023-01-10T07:41:47.781147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = svc_gaus.predict(X_test_PCA)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:41:47.783878Z","iopub.execute_input":"2023-01-10T07:41:47.784267Z","iopub.status.idle":"2023-01-10T07:42:10.356221Z","shell.execute_reply.started":"2023-01-10T07:41:47.784231Z","shell.execute_reply":"2023-01-10T07:42:10.355195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(confusion_matrix(y_test, y_pred))\nprint(classification_report(y_test, y_pred))\n\ngaus_score = accuracy_score(y_pred, y_test)\nprint(\"Gaussian Kernel score:\", gaus_score)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:42:10.357882Z","iopub.execute_input":"2023-01-10T07:42:10.358291Z","iopub.status.idle":"2023-01-10T07:42:10.406758Z","shell.execute_reply.started":"2023-01-10T07:42:10.358248Z","shell.execute_reply":"2023-01-10T07:42:10.405683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Summary","metadata":{}},{"cell_type":"code","source":"scores = pd.DataFrame({\n    'Kernel': ['Linear','Poly', 'Gaus'],\n    'Score': [lin_score, poly_score, gaus_score]\n})\nscores.sort_values(by='Score', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T07:42:10.408355Z","iopub.execute_input":"2023-01-10T07:42:10.408777Z","iopub.status.idle":"2023-01-10T07:42:10.423089Z","shell.execute_reply.started":"2023-01-10T07:42:10.408741Z","shell.execute_reply":"2023-01-10T07:42:10.422042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Gauss and Poly have almost the same results and are only slightly better than Linear. \n#### I think the reason is because the data has many features so the latter two did better","metadata":{}},{"cell_type":"markdown","source":"# Submit","metadata":{}},{"cell_type":"code","source":"X_submit = df_test.drop(columns='id')\nX_submit = pca.transform(X_submit)\nX_submit","metadata":{"execution":{"iopub.status.busy":"2023-01-10T08:02:30.194476Z","iopub.execute_input":"2023-01-10T08:02:30.195170Z","iopub.status.idle":"2023-01-10T08:02:30.259075Z","shell.execute_reply.started":"2023-01-10T08:02:30.195132Z","shell.execute_reply":"2023-01-10T08:02:30.257690Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_submit = svc_gaus.predict(X_submit)","metadata":{"execution":{"iopub.status.busy":"2023-01-10T08:02:46.098209Z","iopub.execute_input":"2023-01-10T08:02:46.098573Z","iopub.status.idle":"2023-01-10T08:02:52.099633Z","shell.execute_reply.started":"2023-01-10T08:02:46.098543Z","shell.execute_reply":"2023-01-10T08:02:52.098633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(y_submit, columns=['label'])\nsubmission = submission.reset_index().rename(columns={'index': 'id'})\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-10T08:08:20.941954Z","iopub.execute_input":"2023-01-10T08:08:20.942312Z","iopub.status.idle":"2023-01-10T08:08:20.963553Z","shell.execute_reply.started":"2023-01-10T08:08:20.942282Z","shell.execute_reply":"2023-01-10T08:08:20.962599Z"},"trusted":true},"execution_count":null,"outputs":[]}]}