{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:31:38.103590Z","iopub.execute_input":"2025-05-02T04:31:38.103830Z","iopub.status.idle":"2025-05-02T04:31:38.109566Z","shell.execute_reply.started":"2025-05-02T04:31:38.103812Z","shell.execute_reply":"2025-05-02T04:31:38.108739Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MNIST Data classification using KNN without PCA","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:31:53.543063Z","iopub.execute_input":"2025-05-02T04:31:53.543379Z","iopub.status.idle":"2025-05-02T04:31:55.773081Z","shell.execute_reply.started":"2025-05-02T04:31:53.543356Z","shell.execute_reply":"2025-05-02T04:31:55.772170Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:03.816461Z","iopub.execute_input":"2025-05-02T04:32:03.816737Z","iopub.status.idle":"2025-05-02T04:32:03.821963Z","shell.execute_reply.started":"2025-05-02T04:32:03.816713Z","shell.execute_reply":"2025-05-02T04:32:03.821184Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:06.111852Z","iopub.execute_input":"2025-05-02T04:32:06.112202Z","iopub.status.idle":"2025-05-02T04:32:06.116245Z","shell.execute_reply.started":"2025-05-02T04:32:06.112180Z","shell.execute_reply":"2025-05-02T04:32:06.115374Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(df.iloc[6060,1:].values.reshape(28,28)) #it is zero","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:09.268270Z","iopub.execute_input":"2025-05-02T04:32:09.269068Z","iopub.status.idle":"2025-05-02T04:32:09.429635Z","shell.execute_reply.started":"2025-05-02T04:32:09.269041Z","shell.execute_reply":"2025-05-02T04:32:09.428766Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(df.iloc[14458,1:].values.reshape(28,28)) #it is five ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:12.457952Z","iopub.execute_input":"2025-05-02T04:32:12.458270Z","iopub.status.idle":"2025-05-02T04:32:12.619816Z","shell.execute_reply.started":"2025-05-02T04:32:12.458245Z","shell.execute_reply":"2025-05-02T04:32:12.619055Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"*we can see that each row in a datset represent a image of a digit*","metadata":{}},{"cell_type":"code","source":"X = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:16.684829Z","iopub.execute_input":"2025-05-02T04:32:16.685564Z","iopub.status.idle":"2025-05-02T04:32:16.689600Z","shell.execute_reply.started":"2025-05-02T04:32:16.685538Z","shell.execute_reply":"2025-05-02T04:32:16.688629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:19.516521Z","iopub.execute_input":"2025-05-02T04:32:19.516840Z","iopub.status.idle":"2025-05-02T04:32:19.530358Z","shell.execute_reply.started":"2025-05-02T04:32:19.516815Z","shell.execute_reply":"2025-05-02T04:32:19.529444Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:22.169457Z","iopub.execute_input":"2025-05-02T04:32:22.170212Z","iopub.status.idle":"2025-05-02T04:32:22.175493Z","shell.execute_reply.started":"2025-05-02T04:32:22.170186Z","shell.execute_reply":"2025-05-02T04:32:22.174641Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:25.227056Z","iopub.execute_input":"2025-05-02T04:32:25.227340Z","iopub.status.idle":"2025-05-02T04:32:25.231332Z","shell.execute_reply.started":"2025-05-02T04:32:25.227321Z","shell.execute_reply":"2025-05-02T04:32:25.230442Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:28.355366Z","iopub.execute_input":"2025-05-02T04:32:28.355676Z","iopub.status.idle":"2025-05-02T04:32:28.585091Z","shell.execute_reply.started":"2025-05-02T04:32:28.355651Z","shell.execute_reply":"2025-05-02T04:32:28.584183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:30.460454Z","iopub.execute_input":"2025-05-02T04:32:30.460785Z","iopub.status.idle":"2025-05-02T04:32:30.465848Z","shell.execute_reply.started":"2025-05-02T04:32:30.460761Z","shell.execute_reply":"2025-05-02T04:32:30.465049Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:32.856453Z","iopub.execute_input":"2025-05-02T04:32:32.856879Z","iopub.status.idle":"2025-05-02T04:32:32.863105Z","shell.execute_reply.started":"2025-05-02T04:32:32.856844Z","shell.execute_reply":"2025-05-02T04:32:32.862071Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train.shape,y_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:35.475718Z","iopub.execute_input":"2025-05-02T04:32:35.476431Z","iopub.status.idle":"2025-05-02T04:32:35.481510Z","shell.execute_reply.started":"2025-05-02T04:32:35.476397Z","shell.execute_reply":"2025-05-02T04:32:35.480723Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"*y-train and y_test is a series not a dataframe as we can see its shape*","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:38.290051Z","iopub.execute_input":"2025-05-02T04:32:38.290341Z","iopub.status.idle":"2025-05-02T04:32:38.294246Z","shell.execute_reply.started":"2025-05-02T04:32:38.290322Z","shell.execute_reply":"2025-05-02T04:32:38.293402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:40.617638Z","iopub.execute_input":"2025-05-02T04:32:40.617997Z","iopub.status.idle":"2025-05-02T04:32:40.621701Z","shell.execute_reply.started":"2025-05-02T04:32:40.617972Z","shell.execute_reply":"2025-05-02T04:32:40.620812Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn.fit(X_train,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:43.361308Z","iopub.execute_input":"2025-05-02T04:32:43.361685Z","iopub.status.idle":"2025-05-02T04:32:43.488952Z","shell.execute_reply.started":"2025-05-02T04:32:43.361653Z","shell.execute_reply":"2025-05-02T04:32:43.488025Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = knn.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:32:45.910250Z","iopub.execute_input":"2025-05-02T04:32:45.910802Z","iopub.status.idle":"2025-05-02T04:32:59.690644Z","shell.execute_reply.started":"2025-05-02T04:32:45.910774Z","shell.execute_reply":"2025-05-02T04:32:59.689744Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:33:08.234754Z","iopub.execute_input":"2025-05-02T04:33:08.235631Z","iopub.status.idle":"2025-05-02T04:33:08.240152Z","shell.execute_reply.started":"2025-05-02T04:33:08.235603Z","shell.execute_reply":"2025-05-02T04:33:08.239067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(y_pred,y_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:33:11.688726Z","iopub.execute_input":"2025-05-02T04:33:11.689408Z","iopub.status.idle":"2025-05-02T04:33:11.695765Z","shell.execute_reply.started":"2025-05-02T04:33:11.689381Z","shell.execute_reply":"2025-05-02T04:33:11.694967Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# MNIST Data classification using KNN with PCA","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:34:53.443637Z","iopub.execute_input":"2025-05-02T04:34:53.443979Z","iopub.status.idle":"2025-05-02T04:34:53.447635Z","shell.execute_reply.started":"2025-05-02T04:34:53.443952Z","shell.execute_reply":"2025-05-02T04:34:53.446850Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"scaler = StandardScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:35:23.564055Z","iopub.execute_input":"2025-05-02T04:35:23.564335Z","iopub.status.idle":"2025-05-02T04:35:23.568574Z","shell.execute_reply.started":"2025-05-02T04:35:23.564316Z","shell.execute_reply":"2025-05-02T04:35:23.567688Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:36:37.275878Z","iopub.execute_input":"2025-05-02T04:36:37.276650Z","iopub.status.idle":"2025-05-02T04:36:37.835040Z","shell.execute_reply.started":"2025-05-02T04:36:37.276624Z","shell.execute_reply":"2025-05-02T04:36:37.834145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:37:10.377333Z","iopub.execute_input":"2025-05-02T04:37:10.378281Z","iopub.status.idle":"2025-05-02T04:37:10.383725Z","shell.execute_reply.started":"2025-05-02T04:37:10.378251Z","shell.execute_reply":"2025-05-02T04:37:10.382674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#PCA\nfrom sklearn.decomposition import PCA\npca = PCA(n_components = None)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:38:29.718456Z","iopub.execute_input":"2025-05-02T04:38:29.718821Z","iopub.status.idle":"2025-05-02T04:38:29.723177Z","shell.execute_reply.started":"2025-05-02T04:38:29.718796Z","shell.execute_reply":"2025-05-02T04:38:29.721866Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = pca.fit_transform(X_train)\nX_test = pca.fit_transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:42:55.601613Z","iopub.execute_input":"2025-05-02T04:42:55.602019Z","iopub.status.idle":"2025-05-02T04:43:01.031601Z","shell.execute_reply.started":"2025-05-02T04:42:55.601992Z","shell.execute_reply":"2025-05-02T04:43:01.030946Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape,X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:43:06.096396Z","iopub.execute_input":"2025-05-02T04:43:06.096676Z","iopub.status.idle":"2025-05-02T04:43:06.102090Z","shell.execute_reply.started":"2025-05-02T04:43:06.096658Z","shell.execute_reply":"2025-05-02T04:43:06.101245Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Now lets take the first 100 principal components and calculate accuracy score**","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:43:52.862551Z","iopub.execute_input":"2025-05-02T04:43:52.863317Z","iopub.status.idle":"2025-05-02T04:43:55.116278Z","shell.execute_reply.started":"2025-05-02T04:43:52.863291Z","shell.execute_reply":"2025-05-02T04:43:55.115365Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:44:15.107025Z","iopub.execute_input":"2025-05-02T04:44:15.107306Z","iopub.status.idle":"2025-05-02T04:44:15.111908Z","shell.execute_reply.started":"2025-05-02T04:44:15.107288Z","shell.execute_reply":"2025-05-02T04:44:15.111118Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:44:39.723722Z","iopub.execute_input":"2025-05-02T04:44:39.724567Z","iopub.status.idle":"2025-05-02T04:44:39.970283Z","shell.execute_reply.started":"2025-05-02T04:44:39.724540Z","shell.execute_reply":"2025-05-02T04:44:39.969008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape,X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:45:01.107369Z","iopub.execute_input":"2025-05-02T04:45:01.107932Z","iopub.status.idle":"2025-05-02T04:45:01.112711Z","shell.execute_reply.started":"2025-05-02T04:45:01.107899Z","shell.execute_reply":"2025-05-02T04:45:01.111939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca = PCA(n_components = 100)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.fit_transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:45:41.326731Z","iopub.execute_input":"2025-05-02T04:45:41.327728Z","iopub.status.idle":"2025-05-02T04:45:45.205565Z","shell.execute_reply.started":"2025-05-02T04:45:41.327701Z","shell.execute_reply":"2025-05-02T04:45:45.204927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape,X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:46:01.804331Z","iopub.execute_input":"2025-05-02T04:46:01.804607Z","iopub.status.idle":"2025-05-02T04:46:01.809872Z","shell.execute_reply.started":"2025-05-02T04:46:01.804587Z","shell.execute_reply":"2025-05-02T04:46:01.809192Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf.shape,X_test_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:46:23.473833Z","iopub.execute_input":"2025-05-02T04:46:23.474169Z","iopub.status.idle":"2025-05-02T04:46:23.478797Z","shell.execute_reply.started":"2025-05-02T04:46:23.474146Z","shell.execute_reply":"2025-05-02T04:46:23.478197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#now fitting this to knn\nknn = KNeighborsClassifier()\nknn.fit(X_train_trf,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:47:36.769560Z","iopub.execute_input":"2025-05-02T04:47:36.769964Z","iopub.status.idle":"2025-05-02T04:47:36.787496Z","shell.execute_reply.started":"2025-05-02T04:47:36.769940Z","shell.execute_reply":"2025-05-02T04:47:36.786699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_ped = knn.predict(X_test_trf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:48:33.892091Z","iopub.execute_input":"2025-05-02T04:48:33.892461Z","iopub.status.idle":"2025-05-02T04:48:35.651389Z","shell.execute_reply.started":"2025-05-02T04:48:33.892434Z","shell.execute_reply":"2025-05-02T04:48:35.650364Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(y_pred,y_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:48:48.682126Z","iopub.execute_input":"2025-05-02T04:48:48.682449Z","iopub.status.idle":"2025-05-02T04:48:48.689496Z","shell.execute_reply.started":"2025-05-02T04:48:48.682427Z","shell.execute_reply":"2025-05-02T04:48:48.688771Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Our score is very low, i dont know where i messed up!! let me restrat from scratch**","metadata":{}},{"cell_type":"code","source":"#importing dataset\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\ndf.sample(5)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:49:56.694572Z","iopub.execute_input":"2025-05-02T04:49:56.695249Z","iopub.status.idle":"2025-05-02T04:49:58.786496Z","shell.execute_reply.started":"2025-05-02T04:49:56.695223Z","shell.execute_reply":"2025-05-02T04:49:58.785567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:50:47.866661Z","iopub.execute_input":"2025-05-02T04:50:47.867400Z","iopub.status.idle":"2025-05-02T04:50:47.872851Z","shell.execute_reply.started":"2025-05-02T04:50:47.867370Z","shell.execute_reply":"2025-05-02T04:50:47.872009Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:50:55.038583Z","iopub.execute_input":"2025-05-02T04:50:55.038976Z","iopub.status.idle":"2025-05-02T04:50:55.052873Z","shell.execute_reply.started":"2025-05-02T04:50:55.038953Z","shell.execute_reply":"2025-05-02T04:50:55.052260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:51:00.947936Z","iopub.execute_input":"2025-05-02T04:51:00.948231Z","iopub.status.idle":"2025-05-02T04:51:00.953965Z","shell.execute_reply.started":"2025-05-02T04:51:00.948212Z","shell.execute_reply":"2025-05-02T04:51:00.953154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:52:08.646262Z","iopub.execute_input":"2025-05-02T04:52:08.646578Z","iopub.status.idle":"2025-05-02T04:52:08.873277Z","shell.execute_reply.started":"2025-05-02T04:52:08.646554Z","shell.execute_reply":"2025-05-02T04:52:08.872516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape,X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:52:28.051268Z","iopub.execute_input":"2025-05-02T04:52:28.051593Z","iopub.status.idle":"2025-05-02T04:52:28.057117Z","shell.execute_reply.started":"2025-05-02T04:52:28.051567Z","shell.execute_reply":"2025-05-02T04:52:28.056284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train.shape,y_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:52:45.931359Z","iopub.execute_input":"2025-05-02T04:52:45.931678Z","iopub.status.idle":"2025-05-02T04:52:45.937411Z","shell.execute_reply.started":"2025-05-02T04:52:45.931651Z","shell.execute_reply":"2025-05-02T04:52:45.936324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#now mean centering \nfrom sklearn.preprocessing import StandardScaler\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)\nX_train.shape,X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:54:41.499022Z","iopub.execute_input":"2025-05-02T04:54:41.499320Z","iopub.status.idle":"2025-05-02T04:54:42.057006Z","shell.execute_reply.started":"2025-05-02T04:54:41.499300Z","shell.execute_reply":"2025-05-02T04:54:42.056195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#now applying pca\nfrom sklearn.decomposition import PCA\npca = PCA(n_components = 100)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:55:33.593243Z","iopub.execute_input":"2025-05-02T04:55:33.594129Z","iopub.status.idle":"2025-05-02T04:55:33.598233Z","shell.execute_reply.started":"2025-05-02T04:55:33.594095Z","shell.execute_reply":"2025-05-02T04:55:33.597380Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:56:11.750525Z","iopub.execute_input":"2025-05-02T04:56:11.751276Z","iopub.status.idle":"2025-05-02T04:56:14.641814Z","shell.execute_reply.started":"2025-05-02T04:56:11.751244Z","shell.execute_reply":"2025-05-02T04:56:14.640846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf.shape,X_test_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:56:43.672257Z","iopub.execute_input":"2025-05-02T04:56:43.672576Z","iopub.status.idle":"2025-05-02T04:56:43.677808Z","shell.execute_reply.started":"2025-05-02T04:56:43.672553Z","shell.execute_reply":"2025-05-02T04:56:43.676949Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:57:17.224094Z","iopub.execute_input":"2025-05-02T04:57:17.224408Z","iopub.status.idle":"2025-05-02T04:57:17.228445Z","shell.execute_reply.started":"2025-05-02T04:57:17.224382Z","shell.execute_reply":"2025-05-02T04:57:17.227455Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:58:04.835082Z","iopub.execute_input":"2025-05-02T04:58:04.835379Z","iopub.status.idle":"2025-05-02T04:58:04.840813Z","shell.execute_reply.started":"2025-05-02T04:58:04.835361Z","shell.execute_reply":"2025-05-02T04:58:04.839970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn.fit(X_train_trf,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:58:53.827994Z","iopub.execute_input":"2025-05-02T04:58:53.828276Z","iopub.status.idle":"2025-05-02T04:58:53.845809Z","shell.execute_reply.started":"2025-05-02T04:58:53.828258Z","shell.execute_reply":"2025-05-02T04:58:53.844982Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = knn.predict(X_test_trf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:59:11.640324Z","iopub.execute_input":"2025-05-02T04:59:11.641051Z","iopub.status.idle":"2025-05-02T04:59:13.337101Z","shell.execute_reply.started":"2025-05-02T04:59:11.641028Z","shell.execute_reply":"2025-05-02T04:59:13.336260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_pred,y_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T04:59:46.220322Z","iopub.execute_input":"2025-05-02T04:59:46.221371Z","iopub.status.idle":"2025-05-02T04:59:46.228438Z","shell.execute_reply.started":"2025-05-02T04:59:46.221344Z","shell.execute_reply":"2025-05-02T04:59:46.227569Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**we have reduced columns (dimensions) from 784 to 100 and still gets 95% score quite similar to score without pca,we can select and adjust n_components to get better score**","metadata":{}},{"cell_type":"markdown","source":"# PCA with 200pc","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:02:31.429092Z","iopub.execute_input":"2025-05-02T05:02:31.429842Z","iopub.status.idle":"2025-05-02T05:02:31.657024Z","shell.execute_reply.started":"2025-05-02T05:02:31.429815Z","shell.execute_reply":"2025-05-02T05:02:31.656220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape,X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:02:41.292849Z","iopub.execute_input":"2025-05-02T05:02:41.293156Z","iopub.status.idle":"2025-05-02T05:02:41.298315Z","shell.execute_reply.started":"2025-05-02T05:02:41.293137Z","shell.execute_reply":"2025-05-02T05:02:41.297599Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#now mean centering \nfrom sklearn.preprocessing import StandardScaler\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)\nX_train.shape,X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:02:51.093690Z","iopub.execute_input":"2025-05-02T05:02:51.094008Z","iopub.status.idle":"2025-05-02T05:02:51.655039Z","shell.execute_reply.started":"2025-05-02T05:02:51.093987Z","shell.execute_reply":"2025-05-02T05:02:51.654108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#now applying pca\nfrom sklearn.decomposition import PCA\npca = PCA(n_components = 200)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:03:04.349789Z","iopub.execute_input":"2025-05-02T05:03:04.350102Z","iopub.status.idle":"2025-05-02T05:03:04.354160Z","shell.execute_reply.started":"2025-05-02T05:03:04.350082Z","shell.execute_reply":"2025-05-02T05:03:04.353280Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:03:12.778417Z","iopub.execute_input":"2025-05-02T05:03:12.778762Z","iopub.status.idle":"2025-05-02T05:03:17.255796Z","shell.execute_reply.started":"2025-05-02T05:03:12.778737Z","shell.execute_reply":"2025-05-02T05:03:17.255098Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:03:25.384688Z","iopub.execute_input":"2025-05-02T05:03:25.385045Z","iopub.status.idle":"2025-05-02T05:03:25.389462Z","shell.execute_reply.started":"2025-05-02T05:03:25.385022Z","shell.execute_reply":"2025-05-02T05:03:25.388132Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn.fit(X_train_trf,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:03:37.386974Z","iopub.execute_input":"2025-05-02T05:03:37.387292Z","iopub.status.idle":"2025-05-02T05:03:37.414069Z","shell.execute_reply.started":"2025-05-02T05:03:37.387270Z","shell.execute_reply":"2025-05-02T05:03:37.412910Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = knn.predict(X_test_trf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:03:45.908404Z","iopub.execute_input":"2025-05-02T05:03:45.909201Z","iopub.status.idle":"2025-05-02T05:03:48.460638Z","shell.execute_reply.started":"2025-05-02T05:03:45.909172Z","shell.execute_reply":"2025-05-02T05:03:48.459913Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_pred,y_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:03:57.320275Z","iopub.execute_input":"2025-05-02T05:03:57.321068Z","iopub.status.idle":"2025-05-02T05:03:57.328437Z","shell.execute_reply.started":"2025-05-02T05:03:57.321031Z","shell.execute_reply":"2025-05-02T05:03:57.327525Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**our score is decreased on taking 200 principal components**","metadata":{}},{"cell_type":"code","source":"#doing all this iteratively as \nfor i in range(1,785):\n    pca = PCA(n_components = i)\n    X_train_trf = pca.fit_transform(X_train)\n    X_test_trf = pca.transform(X_test)\n    knn = KNeighborsClassifier()\n    knn.fit(X_train_trf,y_train)\n    y_pred = knn.predict(X_test_trf)\n    print(accuracy_score(y_pred,y_test))\n    ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:10:30.542416Z","iopub.execute_input":"2025-05-02T05:10:30.543248Z","iopub.status.idle":"2025-05-02T05:22:55.045336Z","shell.execute_reply.started":"2025-05-02T05:10:30.543220Z","shell.execute_reply":"2025-05-02T05:22:55.043950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transforming to a 2D coordinate system\npca = PCA(n_components=2)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:23:04.295250Z","iopub.execute_input":"2025-05-02T05:23:04.295526Z","iopub.status.idle":"2025-05-02T05:23:05.074926Z","shell.execute_reply.started":"2025-05-02T05:23:04.295507Z","shell.execute_reply":"2025-05-02T05:23:05.074076Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:23:13.724465Z","iopub.execute_input":"2025-05-02T05:23:13.724758Z","iopub.status.idle":"2025-05-02T05:23:13.730940Z","shell.execute_reply.started":"2025-05-02T05:23:13.724736Z","shell.execute_reply":"2025-05-02T05:23:13.729969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter(x=X_train_trf[:,0],\n                 y=X_train_trf[:,1],\n                 color=y_train_trf,\n                 color_discrete_sequence=px.colors.qualitative.G10\n                )\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:23:50.409438Z","iopub.execute_input":"2025-05-02T05:23:50.409715Z","iopub.status.idle":"2025-05-02T05:23:50.506736Z","shell.execute_reply.started":"2025-05-02T05:23:50.409696Z","shell.execute_reply":"2025-05-02T05:23:50.505612Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"see how we transform 784 th dimensional data to 2d using plotly taking 2 pricncipal components","metadata":{}},{"cell_type":"code","source":"# transforming in 3D\npca = PCA(n_components=3)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:24:58.793209Z","iopub.execute_input":"2025-05-02T05:24:58.793843Z","iopub.status.idle":"2025-05-02T05:24:59.892334Z","shell.execute_reply.started":"2025-05-02T05:24:58.793814Z","shell.execute_reply":"2025-05-02T05:24:59.891358Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x=X_train_trf[:,0], y=X_train_trf[:,1], z=X_train_trf[:,2],\n              color=y_train_trf)\nfig.update_layout(\n    margin=dict(l=20, r=20, t=20, b=20),\n    paper_bgcolor=\"LightSteelBlue\",\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:25:26.127762Z","iopub.execute_input":"2025-05-02T05:25:26.128566Z","iopub.status.idle":"2025-05-02T05:25:26.298318Z","shell.execute_reply.started":"2025-05-02T05:25:26.128538Z","shell.execute_reply":"2025-05-02T05:25:26.297212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_\n# Eigen values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:25:58.532777Z","iopub.execute_input":"2025-05-02T05:25:58.533589Z","iopub.status.idle":"2025-05-02T05:25:58.538742Z","shell.execute_reply.started":"2025-05-02T05:25:58.533564Z","shell.execute_reply":"2025-05-02T05:25:58.537750Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape\n# Eigen vectors","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:26:26.451340Z","iopub.execute_input":"2025-05-02T05:26:26.451645Z","iopub.status.idle":"2025-05-02T05:26:26.456482Z","shell.execute_reply.started":"2025-05-02T05:26:26.451619Z","shell.execute_reply":"2025-05-02T05:26:26.455637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_ratio_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:26:51.819198Z","iopub.execute_input":"2025-05-02T05:26:51.819486Z","iopub.status.idle":"2025-05-02T05:26:51.824631Z","shell.execute_reply.started":"2025-05-02T05:26:51.819464Z","shell.execute_reply":"2025-05-02T05:26:51.824005Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# how to find optimal number of principal compnents?","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=None)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:34:18.707294Z","iopub.execute_input":"2025-05-02T05:34:18.707594Z","iopub.status.idle":"2025-05-02T05:34:23.623389Z","shell.execute_reply.started":"2025-05-02T05:34:18.707571Z","shell.execute_reply":"2025-05-02T05:34:23.622361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:34:35.263292Z","iopub.execute_input":"2025-05-02T05:34:35.263629Z","iopub.status.idle":"2025-05-02T05:34:35.269068Z","shell.execute_reply.started":"2025-05-02T05:34:35.263598Z","shell.execute_reply":"2025-05-02T05:34:35.268206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:34:49.084047Z","iopub.execute_input":"2025-05-02T05:34:49.084354Z","iopub.status.idle":"2025-05-02T05:34:49.090098Z","shell.execute_reply.started":"2025-05-02T05:34:49.084333Z","shell.execute_reply":"2025-05-02T05:34:49.088919Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.cumsum(pca.explained_variance_ratio_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:35:05.196551Z","iopub.execute_input":"2025-05-02T05:35:05.196835Z","iopub.status.idle":"2025-05-02T05:35:05.206760Z","shell.execute_reply.started":"2025-05-02T05:35:05.196815Z","shell.execute_reply":"2025-05-02T05:35:05.205958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(np.cumsum(pca.explained_variance_ratio_))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-05-02T05:35:21.745142Z","iopub.execute_input":"2025-05-02T05:35:21.746034Z","iopub.status.idle":"2025-05-02T05:35:21.900838Z","shell.execute_reply.started":"2025-05-02T05:35:21.746008Z","shell.execute_reply":"2025-05-02T05:35:21.899915Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"we can see that we can get 90% variance if we take n_components = 350-400 , in this way we can select optimal n_components","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}