{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn import svm\nfrom sklearn import metrics\nfrom sklearn import preprocessing\n\ntrain_file = '/kaggle/input/digit-recognizer/train.csv'\ntest_file = '/kaggle/input/digit-recognizer/test.csv'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-15T04:30:17.614096Z","iopub.execute_input":"2022-12-15T04:30:17.614623Z","iopub.status.idle":"2022-12-15T04:30:17.621433Z","shell.execute_reply.started":"2022-12-15T04:30:17.614583Z","shell.execute_reply":"2022-12-15T04:30:17.620062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read input files","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(train_file)\ndf_test = pd.read_csv(test_file)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:30:17.623837Z","iopub.execute_input":"2022-12-15T04:30:17.624279Z","iopub.status.idle":"2022-12-15T04:30:21.062160Z","shell.execute_reply.started":"2022-12-15T04:30:17.624242Z","shell.execute_reply":"2022-12-15T04:30:21.060937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:30:21.064204Z","iopub.execute_input":"2022-12-15T04:30:21.064567Z","iopub.status.idle":"2022-12-15T04:30:23.523982Z","shell.execute_reply.started":"2022-12-15T04:30:21.064533Z","shell.execute_reply":"2022-12-15T04:30:23.522722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.describe()","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:30:23.525590Z","iopub.execute_input":"2022-12-15T04:30:23.526705Z","iopub.status.idle":"2022-12-15T04:30:25.611578Z","shell.execute_reply.started":"2022-12-15T04:30:23.526664Z","shell.execute_reply":"2022-12-15T04:30:25.610707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:30:25.613926Z","iopub.execute_input":"2022-12-15T04:30:25.614845Z","iopub.status.idle":"2022-12-15T04:30:25.621466Z","shell.execute_reply.started":"2022-12-15T04:30:25.614786Z","shell.execute_reply":"2022-12-15T04:30:25.620255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract X and y, and split the dataset","metadata":{}},{"cell_type":"code","source":"# train dataset\nX_train = df_train.drop(\"label\", axis=1).values\ny_train = df_train[\"label\"].values\n\n# test dataset\nX_test = df_test.values\n\n# preprocess X values\nX_train = preprocessing.scale(X_train)\nX_test = preprocessing.scale(X_test)\n\n# split train into partial train and holdout (to run test on train dataset)\nX_train_partial, X_holdout, y_train_partial, y_holdout = train_test_split(X_train, y_train, train_size=.75, random_state=99)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:30:25.623013Z","iopub.execute_input":"2022-12-15T04:30:25.623858Z","iopub.status.idle":"2022-12-15T04:30:27.935635Z","shell.execute_reply.started":"2022-12-15T04:30:25.623821Z","shell.execute_reply":"2022-12-15T04:30:27.934445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dimensionality reduction usind PCA","metadata":{}},{"cell_type":"code","source":"# fit initial PCA model\npca = PCA(n_components=X_train.shape[1])\npca.fit(X_train)\n\n# find optimal K that keeps 80% of the components\nevs = pca.explained_variance_ratio_\nevs_sum = 0\nkeep = 0.9\nfor k, ev in enumerate(evs):\n    evs_sum += ev\n    if evs_sum >= keep:\n        break\nprint('K =', k)\n\n# fit PCA model\nn_components = k\npca = PCA(n_components=n_components)\npca.fit(X_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:59:46.467048Z","iopub.execute_input":"2022-12-15T04:59:46.467470Z","iopub.status.idle":"2022-12-15T05:00:02.139131Z","shell.execute_reply.started":"2022-12-15T04:59:46.467438Z","shell.execute_reply":"2022-12-15T05:00:02.137728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transform the dataset\nTheta_train_partial = pca.transform(X_train_partial)\nTheta_holdout = pca.transform(X_holdout)\nTheta_train = pca.transform(X_train)\nTheta_test = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T05:00:41.447870Z","iopub.execute_input":"2022-12-15T05:00:41.448533Z","iopub.status.idle":"2022-12-15T05:00:43.451277Z","shell.execute_reply.started":"2022-12-15T05:00:41.448485Z","shell.execute_reply":"2022-12-15T05:00:43.449582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Find the optimal hyper parameters","metadata":{}},{"cell_type":"code","source":"Cs = [0.5,1,5,10,15,20,30]\nkernels = ['rbf','poly']\ngammas = ['scale', 'auto']\nresults = {'error': 1.}\n\nfor kernel in kernels:\n    for gamma in gammas:\n        for C in Cs:\n            \n            # train with poly kernel\n            model = svm.SVC(kernel=kernel, gamma=gamma, C=C)\n            model.fit(Theta_train_partial, y_train_partial)\n            error = 1 - model.score(Theta_holdout, y_holdout)\n            print(f'Validating: kernel = {kernel}, gamma = {gamma}, C = {C}, error = {error}')\n            \n            # store the best C\n            if error < results['error']:\n                results = {'kernel': kernel, 'gamma': gamma, 'C': C, 'error': error}\n","metadata":{"execution":{"iopub.status.busy":"2022-12-15T05:00:49.351134Z","iopub.execute_input":"2022-12-15T05:00:49.351589Z","iopub.status.idle":"2022-12-15T05:06:43.512421Z","shell.execute_reply.started":"2022-12-15T05:00:49.351550Z","shell.execute_reply":"2022-12-15T05:06:43.511058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# retrain the model using the whole training dataset and the best parameters\nmodel = svm.SVC(kernel=results['kernel'], gamma=results['gamma'], C=results['C'])\nmodel.fit(Theta_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:53:21.573219Z","iopub.execute_input":"2022-12-15T04:53:21.573730Z","iopub.status.idle":"2022-12-15T04:55:22.587226Z","shell.execute_reply.started":"2022-12-15T04:53:21.573686Z","shell.execute_reply":"2022-12-15T04:55:22.586050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# calculate accuracy on tain dataset\ny_predict = model.predict(Theta_train)\naccuracy = metrics.accuracy_score(y_train, y_predict)\nprint(f'Accuracy = {accuracy:.3f}%')","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:55:22.589809Z","iopub.execute_input":"2022-12-15T04:55:22.590312Z","iopub.status.idle":"2022-12-15T04:59:16.819439Z","shell.execute_reply.started":"2022-12-15T04:55:22.590262Z","shell.execute_reply":"2022-12-15T04:59:16.818416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Calculate Results for Submission","metadata":{}},{"cell_type":"code","source":"# predict for the transformed test dataset\ny_test = model.predict(Theta_test)\ndf_result = pd.DataFrame(y_test, columns=['Label'], index=np.arange(1,28001))\ndf_result.to_csv('subimission.csv', index_label='ImageId')","metadata":{"execution":{"iopub.status.busy":"2022-12-15T04:38:00.193585Z","iopub.status.idle":"2022-12-15T04:38:00.193981Z","shell.execute_reply.started":"2022-12-15T04:38:00.193799Z","shell.execute_reply":"2022-12-15T04:38:00.193817Z"},"trusted":true},"execution_count":null,"outputs":[]}]}