{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-05-03T05:40:13.257872Z","iopub.execute_input":"2023-05-03T05:40:13.258277Z","iopub.status.idle":"2023-05-03T05:40:13.266663Z","shell.execute_reply.started":"2023-05-03T05:40:13.258246Z","shell.execute_reply":"2023-05-03T05:40:13.265691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:40:22.695990Z","iopub.execute_input":"2023-05-03T05:40:22.697446Z","iopub.status.idle":"2023-05-03T05:40:24.972507Z","shell.execute_reply.started":"2023-05-03T05:40:22.697395Z","shell.execute_reply":"2023-05-03T05:40:24.971191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:40:26.031741Z","iopub.execute_input":"2023-05-03T05:40:26.032199Z","iopub.status.idle":"2023-05-03T05:40:26.039816Z","shell.execute_reply.started":"2023-05-03T05:40:26.032165Z","shell.execute_reply":"2023-05-03T05:40:26.038419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-05-03T04:56:07.941729Z","iopub.execute_input":"2023-05-03T04:56:07.942148Z","iopub.status.idle":"2023-05-03T04:56:07.976153Z","shell.execute_reply.started":"2023-05-03T04:56:07.942106Z","shell.execute_reply":"2023-05-03T04:56:07.975450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T04:57:26.309964Z","iopub.execute_input":"2023-05-03T04:57:26.310358Z","iopub.status.idle":"2023-05-03T04:57:26.331206Z","shell.execute_reply.started":"2023-05-03T04:57:26.310327Z","shell.execute_reply":"2023-05-03T04:57:26.330508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-05-03T04:58:14.195688Z","iopub.execute_input":"2023-05-03T04:58:14.196118Z","iopub.status.idle":"2023-05-03T04:58:14.201867Z","shell.execute_reply.started":"2023-05-03T04:58:14.196065Z","shell.execute_reply":"2023-05-03T04:58:14.200752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(df.iloc[18306,1:].values.reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2023-05-03T04:59:37.284155Z","iopub.execute_input":"2023-05-03T04:59:37.284523Z","iopub.status.idle":"2023-05-03T04:59:37.538230Z","shell.execute_reply.started":"2023-05-03T04:59:37.284497Z","shell.execute_reply":"2023-05-03T04:59:37.536982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(df.iloc[5869,1:].values.reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:00:04.627561Z","iopub.execute_input":"2023-05-03T05:00:04.628365Z","iopub.status.idle":"2023-05-03T05:00:04.860260Z","shell.execute_reply.started":"2023-05-03T05:00:04.628327Z","shell.execute_reply":"2023-05-03T05:00:04.859161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:40:33.338261Z","iopub.execute_input":"2023-05-03T05:40:33.338753Z","iopub.status.idle":"2023-05-03T05:40:33.345406Z","shell.execute_reply.started":"2023-05-03T05:40:33.338717Z","shell.execute_reply":"2023-05-03T05:40:33.343894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:40:40.388741Z","iopub.execute_input":"2023-05-03T05:40:40.389826Z","iopub.status.idle":"2023-05-03T05:40:40.639291Z","shell.execute_reply.started":"2023-05-03T05:40:40.389788Z","shell.execute_reply":"2023-05-03T05:40:40.638330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:04:34.123537Z","iopub.execute_input":"2023-05-03T05:04:34.123920Z","iopub.status.idle":"2023-05-03T05:04:34.134191Z","shell.execute_reply.started":"2023-05-03T05:04:34.123889Z","shell.execute_reply":"2023-05-03T05:04:34.131712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:40:47.122441Z","iopub.execute_input":"2023-05-03T05:40:47.122857Z","iopub.status.idle":"2023-05-03T05:40:47.127684Z","shell.execute_reply.started":"2023-05-03T05:40:47.122825Z","shell.execute_reply":"2023-05-03T05:40:47.126564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:05:58.400309Z","iopub.execute_input":"2023-05-03T05:05:58.400687Z","iopub.status.idle":"2023-05-03T05:05:58.406491Z","shell.execute_reply.started":"2023-05-03T05:05:58.400655Z","shell.execute_reply":"2023-05-03T05:05:58.404991Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:07:19.254211Z","iopub.execute_input":"2023-05-03T05:07:19.255022Z","iopub.status.idle":"2023-05-03T05:07:19.415439Z","shell.execute_reply.started":"2023-05-03T05:07:19.254986Z","shell.execute_reply":"2023-05-03T05:07:19.414417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = knn.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:07:55.366079Z","iopub.execute_input":"2023-05-03T05:07:55.366541Z","iopub.status.idle":"2023-05-03T05:08:08.342831Z","shell.execute_reply.started":"2023-05-03T05:07:55.366459Z","shell.execute_reply":"2023-05-03T05:08:08.341733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:08:30.670360Z","iopub.execute_input":"2023-05-03T05:08:30.670754Z","iopub.status.idle":"2023-05-03T05:08:30.677596Z","shell.execute_reply.started":"2023-05-03T05:08:30.670723Z","shell.execute_reply":"2023-05-03T05:08:30.676443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_test,y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:09:29.768399Z","iopub.execute_input":"2023-05-03T05:09:29.768819Z","iopub.status.idle":"2023-05-03T05:09:29.778232Z","shell.execute_reply.started":"2023-05-03T05:09:29.768785Z","shell.execute_reply":"2023-05-03T05:09:29.777171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PCA ","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:40:58.023158Z","iopub.execute_input":"2023-05-03T05:40:58.023555Z","iopub.status.idle":"2023-05-03T05:40:58.029561Z","shell.execute_reply.started":"2023-05-03T05:40:58.023522Z","shell.execute_reply":"2023-05-03T05:40:58.028175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:00.526227Z","iopub.execute_input":"2023-05-03T05:41:00.527018Z","iopub.status.idle":"2023-05-03T05:41:01.281627Z","shell.execute_reply.started":"2023-05-03T05:41:00.526976Z","shell.execute_reply":"2023-05-03T05:41:01.280474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components=200)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:03.215981Z","iopub.execute_input":"2023-05-03T05:41:03.216392Z","iopub.status.idle":"2023-05-03T05:41:03.221819Z","shell.execute_reply.started":"2023-05-03T05:41:03.216359Z","shell.execute_reply":"2023-05-03T05:41:03.220568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:09.021544Z","iopub.execute_input":"2023-05-03T05:41:09.021924Z","iopub.status.idle":"2023-05-03T05:41:15.547451Z","shell.execute_reply.started":"2023-05-03T05:41:09.021894Z","shell.execute_reply":"2023-05-03T05:41:15.546381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:19:48.912494Z","iopub.execute_input":"2023-05-03T05:19:48.912879Z","iopub.status.idle":"2023-05-03T05:19:48.920424Z","shell.execute_reply.started":"2023-05-03T05:19:48.912851Z","shell.execute_reply":"2023-05-03T05:19:48.919305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:18.383127Z","iopub.execute_input":"2023-05-03T05:41:18.383512Z","iopub.status.idle":"2023-05-03T05:41:18.390287Z","shell.execute_reply.started":"2023-05-03T05:41:18.383481Z","shell.execute_reply":"2023-05-03T05:41:18.389166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:22.731684Z","iopub.execute_input":"2023-05-03T05:41:22.732069Z","iopub.status.idle":"2023-05-03T05:41:22.737912Z","shell.execute_reply.started":"2023-05-03T05:41:22.732038Z","shell.execute_reply":"2023-05-03T05:41:22.736614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn.fit(X_train_trf,y_train)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:25.593919Z","iopub.execute_input":"2023-05-03T05:41:25.594361Z","iopub.status.idle":"2023-05-03T05:41:25.632509Z","shell.execute_reply.started":"2023-05-03T05:41:25.594325Z","shell.execute_reply":"2023-05-03T05:41:25.631379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = knn.predict(X_test_trf,)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:23:47.122812Z","iopub.execute_input":"2023-05-03T05:23:47.123197Z","iopub.status.idle":"2023-05-03T05:23:49.871426Z","shell.execute_reply.started":"2023-05-03T05:23:47.123169Z","shell.execute_reply":"2023-05-03T05:23:49.870159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_test,y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:23:52.238556Z","iopub.execute_input":"2023-05-03T05:23:52.238963Z","iopub.status.idle":"2023-05-03T05:23:52.248893Z","shell.execute_reply.started":"2023-05-03T05:23:52.238931Z","shell.execute_reply":"2023-05-03T05:23:52.247537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,785):\n    pca = PCA(n_components=i)\n    X_train_trf = pca.fit_transform(X_train)\n    X_test_trf = pca.transform(X_test)\n    \n    knn = KNeighborsClassifier()\n    knn.fit(X_train_trf,y_train)\n    \n    y_pred = knn.predict(X_test_trf)\n    \n    print('for component ', i ,'is : ',accuracy_score(y_test,y_pred))\n    \n    ","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:28:47.848084Z","iopub.execute_input":"2023-05-03T05:28:47.848486Z","iopub.status.idle":"2023-05-03T05:38:28.495904Z","shell.execute_reply.started":"2023-05-03T05:28:47.848458Z","shell.execute_reply":"2023-05-03T05:38:28.492314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transforming to a 2D coordinate system\npca = PCA(n_components=2)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:36.860007Z","iopub.execute_input":"2023-05-03T05:41:36.860428Z","iopub.status.idle":"2023-05-03T05:41:39.511889Z","shell.execute_reply.started":"2023-05-03T05:41:36.860396Z","shell.execute_reply":"2023-05-03T05:41:39.510306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_trf","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:41:40.974682Z","iopub.execute_input":"2023-05-03T05:41:40.975065Z","iopub.status.idle":"2023-05-03T05:41:40.982555Z","shell.execute_reply.started":"2023-05-03T05:41:40.975036Z","shell.execute_reply":"2023-05-03T05:41:40.981373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter(x=X_train_trf[:,0],\n                 y=X_train_trf[:,1],\n                 color=y_train_trf,\n                 color_discrete_sequence=px.colors.qualitative.G10\n                )\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:42:01.813110Z","iopub.execute_input":"2023-05-03T05:42:01.813483Z","iopub.status.idle":"2023-05-03T05:42:04.304350Z","shell.execute_reply.started":"2023-05-03T05:42:01.813454Z","shell.execute_reply":"2023-05-03T05:42:04.303242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transforming in 3D\npca = PCA(n_components=3)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:43:42.317566Z","iopub.execute_input":"2023-05-03T05:43:42.317946Z","iopub.status.idle":"2023-05-03T05:43:45.106693Z","shell.execute_reply.started":"2023-05-03T05:43:42.317919Z","shell.execute_reply":"2023-05-03T05:43:45.104554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x=X_train_trf[:,0], y=X_train_trf[:,1], z=X_train_trf[:,2],\n              color=y_train_trf)\nfig.update_layout(\n    margin=dict(l=20, r=20, t=20, b=20),\n    paper_bgcolor=\"LightSteelBlue\",\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:43:56.794492Z","iopub.execute_input":"2023-05-03T05:43:56.794882Z","iopub.status.idle":"2023-05-03T05:43:57.004215Z","shell.execute_reply.started":"2023-05-03T05:43:56.794849Z","shell.execute_reply":"2023-05-03T05:43:57.002749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Eigen values\npca.explained_variance_","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:46:56.017720Z","iopub.execute_input":"2023-05-03T05:46:56.018126Z","iopub.status.idle":"2023-05-03T05:46:56.026025Z","shell.execute_reply.started":"2023-05-03T05:46:56.018076Z","shell.execute_reply":"2023-05-03T05:46:56.024671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npca.explained_variance_ratio_","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:47:54.712721Z","iopub.execute_input":"2023-05-03T05:47:54.713124Z","iopub.status.idle":"2023-05-03T05:47:54.719714Z","shell.execute_reply.started":"2023-05-03T05:47:54.713076Z","shell.execute_reply":"2023-05-03T05:47:54.718855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# eigen vector\npca.components_.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:48:28.346636Z","iopub.execute_input":"2023-05-03T05:48:28.347356Z","iopub.status.idle":"2023-05-03T05:48:28.354573Z","shell.execute_reply.started":"2023-05-03T05:48:28.347315Z","shell.execute_reply":"2023-05-03T05:48:28.353496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=None)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:55:33.815232Z","iopub.execute_input":"2023-05-03T05:55:33.815607Z","iopub.status.idle":"2023-05-03T05:55:38.435291Z","shell.execute_reply.started":"2023-05-03T05:55:33.815578Z","shell.execute_reply":"2023-05-03T05:55:38.433926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca.explained_variance_.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:55:54.312147Z","iopub.execute_input":"2023-05-03T05:55:54.312544Z","iopub.status.idle":"2023-05-03T05:55:54.321317Z","shell.execute_reply.started":"2023-05-03T05:55:54.312515Z","shell.execute_reply":"2023-05-03T05:55:54.320135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca.components_","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:56:06.319384Z","iopub.execute_input":"2023-05-03T05:56:06.319756Z","iopub.status.idle":"2023-05-03T05:56:06.327737Z","shell.execute_reply.started":"2023-05-03T05:56:06.319728Z","shell.execute_reply":"2023-05-03T05:56:06.326528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca.components_.shape","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:56:21.709972Z","iopub.execute_input":"2023-05-03T05:56:21.710399Z","iopub.status.idle":"2023-05-03T05:56:21.718577Z","shell.execute_reply.started":"2023-05-03T05:56:21.710365Z","shell.execute_reply":"2023-05-03T05:56:21.717354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca.explained_variance_ratio_","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:56:40.126160Z","iopub.execute_input":"2023-05-03T05:56:40.126515Z","iopub.status.idle":"2023-05-03T05:56:40.145314Z","shell.execute_reply.started":"2023-05-03T05:56:40.126489Z","shell.execute_reply":"2023-05-03T05:56:40.144248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.cumsum(pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:57:23.913413Z","iopub.execute_input":"2023-05-03T05:57:23.913868Z","iopub.status.idle":"2023-05-03T05:57:23.926846Z","shell.execute_reply.started":"2023-05-03T05:57:23.913837Z","shell.execute_reply":"2023-05-03T05:57:23.925699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(np.cumsum(pca.explained_variance_ratio_))","metadata":{"execution":{"iopub.status.busy":"2023-05-03T05:58:20.885574Z","iopub.execute_input":"2023-05-03T05:58:20.886050Z","iopub.status.idle":"2023-05-03T05:58:21.106991Z","shell.execute_reply.started":"2023-05-03T05:58:20.886004Z","shell.execute_reply":"2023-05-03T05:58:21.105975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}