{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30839,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:32:36.946176Z","iopub.execute_input":"2025-01-25T02:32:36.946564Z","iopub.status.idle":"2025-01-25T02:32:37.338747Z","shell.execute_reply.started":"2025-01-25T02:32:36.946532Z","shell.execute_reply":"2025-01-25T02:32:37.337521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df= pd.read_csv(\"/kaggle/input/digit-recognizer/train.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:32:40.151212Z","iopub.execute_input":"2025-01-25T02:32:40.151750Z","iopub.status.idle":"2025-01-25T02:32:43.341626Z","shell.execute_reply.started":"2025-01-25T02:32:40.151715Z","shell.execute_reply":"2025-01-25T02:32:43.340588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:32:45.245040Z","iopub.execute_input":"2025-01-25T02:32:45.245423Z","iopub.status.idle":"2025-01-25T02:32:45.252112Z","shell.execute_reply.started":"2025-01-25T02:32:45.245391Z","shell.execute_reply":"2025-01-25T02:32:45.251067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.sample(4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:32:46.352727Z","iopub.execute_input":"2025-01-25T02:32:46.353059Z","iopub.status.idle":"2025-01-25T02:32:46.388980Z","shell.execute_reply.started":"2025-01-25T02:32:46.353031Z","shell.execute_reply":"2025-01-25T02:32:46.387814Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:32:48.378679Z","iopub.execute_input":"2025-01-25T02:32:48.379012Z","iopub.status.idle":"2025-01-25T02:32:48.383773Z","shell.execute_reply.started":"2025-01-25T02:32:48.378977Z","shell.execute_reply":"2025-01-25T02:32:48.382623Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(df.iloc[1232,1:].values.reshape(28,28))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:32:51.317656Z","iopub.execute_input":"2025-01-25T02:32:51.317999Z","iopub.status.idle":"2025-01-25T02:32:51.616908Z","shell.execute_reply.started":"2025-01-25T02:32:51.317972Z","shell.execute_reply":"2025-01-25T02:32:51.615839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:32:55.037307Z","iopub.execute_input":"2025-01-25T02:32:55.037797Z","iopub.status.idle":"2025-01-25T02:32:55.043727Z","shell.execute_reply.started":"2025-01-25T02:32:55.037753Z","shell.execute_reply":"2025-01-25T02:32:55.042619Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:35:05.803703Z","iopub.execute_input":"2025-01-25T02:35:05.804114Z","iopub.status.idle":"2025-01-25T02:35:06.053194Z","shell.execute_reply.started":"2025-01-25T02:35:05.804080Z","shell.execute_reply":"2025-01-25T02:35:06.052033Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:35:20.517380Z","iopub.execute_input":"2025-01-25T02:35:20.517771Z","iopub.status.idle":"2025-01-25T02:35:20.523522Z","shell.execute_reply.started":"2025-01-25T02:35:20.517741Z","shell.execute_reply":"2025-01-25T02:35:20.522589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:36:12.728719Z","iopub.execute_input":"2025-01-25T02:36:12.729104Z","iopub.status.idle":"2025-01-25T02:36:12.953209Z","shell.execute_reply.started":"2025-01-25T02:36:12.729075Z","shell.execute_reply":"2025-01-25T02:36:12.952127Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn = KNeighborsClassifier()\n\nknn.fit(X_train, y_train)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:37:15.017885Z","iopub.execute_input":"2025-01-25T02:37:15.018241Z","iopub.status.idle":"2025-01-25T02:37:15.170962Z","shell.execute_reply.started":"2025-01-25T02:37:15.018212Z","shell.execute_reply":"2025-01-25T02:37:15.170054Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Checking the time it takes to predict without any dimension reduction using PCA","metadata":{}},{"cell_type":"code","source":"import time \nstart = time.time()\n\ny_pred = knn.predict(X_test)\n\nprint(time.time()-start)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:38:36.742209Z","iopub.execute_input":"2025-01-25T02:38:36.742625Z","iopub.status.idle":"2025-01-25T02:38:49.663955Z","shell.execute_reply.started":"2025-01-25T02:38:36.742595Z","shell.execute_reply":"2025-01-25T02:38:49.662998Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Checking for accuracy ","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\naccuracy_score(y_test,y_pred)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:41:01.858840Z","iopub.execute_input":"2025-01-25T02:41:01.859222Z","iopub.status.idle":"2025-01-25T02:41:01.867663Z","shell.execute_reply.started":"2025-01-25T02:41:01.859193Z","shell.execute_reply":"2025-01-25T02:41:01.866554Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"\n### Now we are doing dimension reduction using PCA Steps to follow:\n\n### Step1: Make the mean of the data to 0. so use standard scaling.\n ","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nscaler = StandardScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:43:34.344808Z","iopub.execute_input":"2025-01-25T02:43:34.345193Z","iopub.status.idle":"2025-01-25T02:43:34.349492Z","shell.execute_reply.started":"2025-01-25T02:43:34.345165Z","shell.execute_reply":"2025-01-25T02:43:34.348391Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.decomposition import PCA\n\npca = PCA(n_components=200)\n# here n_components means we want to reduce the dimention from 785 to 200\n# reduce to features to 200\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:45:02.983774Z","iopub.execute_input":"2025-01-25T02:45:02.984184Z","iopub.status.idle":"2025-01-25T02:45:02.988742Z","shell.execute_reply.started":"2025-01-25T02:45:02.984153Z","shell.execute_reply":"2025-01-25T02:45:02.987605Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf = pca.fit_transform(X_train)\n\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:45:56.915899Z","iopub.execute_input":"2025-01-25T02:45:56.916335Z","iopub.status.idle":"2025-01-25T02:46:00.172135Z","shell.execute_reply.started":"2025-01-25T02:45:56.916298Z","shell.execute_reply":"2025-01-25T02:46:00.171212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:46:50.181616Z","iopub.execute_input":"2025-01-25T02:46:50.182114Z","iopub.status.idle":"2025-01-25T02:46:50.189598Z","shell.execute_reply.started":"2025-01-25T02:46:50.182070Z","shell.execute_reply":"2025-01-25T02:46:50.188376Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### here is the transformed data with only 200 features instead of 785 features","metadata":{}},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:50:55.516966Z","iopub.execute_input":"2025-01-25T02:50:55.517351Z","iopub.status.idle":"2025-01-25T02:50:55.524023Z","shell.execute_reply.started":"2025-01-25T02:50:55.517319Z","shell.execute_reply":"2025-01-25T02:50:55.522918Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn1 = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:50:57.089636Z","iopub.execute_input":"2025-01-25T02:50:57.089983Z","iopub.status.idle":"2025-01-25T02:50:57.094399Z","shell.execute_reply.started":"2025-01-25T02:50:57.089955Z","shell.execute_reply":"2025-01-25T02:50:57.093209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn1.fit(X_train_trf, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:51:17.115217Z","iopub.execute_input":"2025-01-25T02:51:17.115619Z","iopub.status.idle":"2025-01-25T02:51:17.152729Z","shell.execute_reply.started":"2025-01-25T02:51:17.115589Z","shell.execute_reply":"2025-01-25T02:51:17.151589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = knn1.predict(X_test_trf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:51:19.263743Z","iopub.execute_input":"2025-01-25T02:51:19.264151Z","iopub.status.idle":"2025-01-25T02:51:21.683646Z","shell.execute_reply.started":"2025-01-25T02:51:19.264111Z","shell.execute_reply":"2025-01-25T02:51:21.682376Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(y_test,y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:51:23.695363Z","iopub.execute_input":"2025-01-25T02:51:23.695776Z","iopub.status.idle":"2025-01-25T02:51:23.703766Z","shell.execute_reply.started":"2025-01-25T02:51:23.695745Z","shell.execute_reply":"2025-01-25T02:51:23.702789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(1,20):\n    pca = PCA(n_components=i)\n    X_train_trf = pca.fit_transform(X_train)\n    X_test_trf = pca.transform(X_test)\n\n    knn = KNeighborsClassifier()\n\n    knn.fit(X_train_trf,y_train)\n\n    y_pred = knn.predict(X_test_trf)\n\n    print(accuracy_score(y_test,y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T02:55:45.668608Z","iopub.execute_input":"2025-01-25T02:55:45.668945Z","iopub.status.idle":"2025-01-25T02:56:25.577960Z","shell.execute_reply.started":"2025-01-25T02:55:45.668917Z","shell.execute_reply":"2025-01-25T02:56:25.577014Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## To find the best n_components value(no of principle components) to take we basicaly plot a graph till where the PC covers a cummilative of 90% of the total variance. ","metadata":{}},{"cell_type":"markdown","source":"PC basically are the new features extracted from the existing feature such that it contains the max info(spread ie variance) of the data.","metadata":{}},{"cell_type":"markdown","source":"### How are 785D data looks in 3D","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=3)\n\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:09:14.234183Z","iopub.execute_input":"2025-01-25T03:09:14.234583Z","iopub.status.idle":"2025-01-25T03:09:14.981739Z","shell.execute_reply.started":"2025-01-25T03:09:14.234550Z","shell.execute_reply":"2025-01-25T03:09:14.980678Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x=X_train_trf[:,0], y=X_train_trf[:,1], z=X_train_trf[:,2],\n              color=y_train_trf)\nfig.update_layout(\n    margin=dict(l=20, r=20, t=20, b=20),\n    paper_bgcolor=\"LightSteelBlue\",\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:11:09.399799Z","iopub.execute_input":"2025-01-25T03:11:09.400159Z","iopub.status.idle":"2025-01-25T03:11:09.525462Z","shell.execute_reply.started":"2025-01-25T03:11:09.400122Z","shell.execute_reply":"2025-01-25T03:11:09.524246Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_\n# Eigen values(spread over that eigen vector)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:15:32.733262Z","iopub.execute_input":"2025-01-25T03:15:32.733677Z","iopub.status.idle":"2025-01-25T03:15:32.739416Z","shell.execute_reply.started":"2025-01-25T03:15:32.733643Z","shell.execute_reply":"2025-01-25T03:15:32.738571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_\n# eigen vectors ie pc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:18:07.713500Z","iopub.execute_input":"2025-01-25T03:18:07.713906Z","iopub.status.idle":"2025-01-25T03:18:07.720801Z","shell.execute_reply.started":"2025-01-25T03:18:07.713875Z","shell.execute_reply":"2025-01-25T03:18:07.719545Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape\n# Eigen vectors shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:18:11.352087Z","iopub.execute_input":"2025-01-25T03:18:11.352403Z","iopub.status.idle":"2025-01-25T03:18:11.358249Z","shell.execute_reply.started":"2025-01-25T03:18:11.352377Z","shell.execute_reply":"2025-01-25T03:18:11.357328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_ratio_\n# how much of variance or spread this particular PC(eigen vector)\n# covers\n# main goal is to take that many pc such that Cummilative valicance is 90%\n# of the total","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:15:08.885588Z","iopub.execute_input":"2025-01-25T03:15:08.885951Z","iopub.status.idle":"2025-01-25T03:15:08.892141Z","shell.execute_reply.started":"2025-01-25T03:15:08.885918Z","shell.execute_reply":"2025-01-25T03:15:08.891262Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Now we plot the graph for finding the value of n_components ie PC required to cover 90% of total variance","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=None)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:19:48.941117Z","iopub.execute_input":"2025-01-25T03:19:48.941473Z","iopub.status.idle":"2025-01-25T03:19:52.299877Z","shell.execute_reply.started":"2025-01-25T03:19:48.941448Z","shell.execute_reply":"2025-01-25T03:19:52.298970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:19:56.712729Z","iopub.execute_input":"2025-01-25T03:19:56.713090Z","iopub.status.idle":"2025-01-25T03:19:56.718882Z","shell.execute_reply.started":"2025-01-25T03:19:56.713056Z","shell.execute_reply":"2025-01-25T03:19:56.717787Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:20:02.532751Z","iopub.execute_input":"2025-01-25T03:20:02.533083Z","iopub.status.idle":"2025-01-25T03:20:02.539088Z","shell.execute_reply.started":"2025-01-25T03:20:02.533049Z","shell.execute_reply":"2025-01-25T03:20:02.537950Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.cumsum(pca.explained_variance_ratio_)\n# finding the cummilative variance to find to what poins or n_components \n# 90% variance is covered","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:21:00.132594Z","iopub.execute_input":"2025-01-25T03:21:00.132928Z","iopub.status.idle":"2025-01-25T03:21:00.143938Z","shell.execute_reply.started":"2025-01-25T03:21:00.132901Z","shell.execute_reply":"2025-01-25T03:21:00.142935Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(np.cumsum(pca.explained_variance_ratio_))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-01-25T03:21:09.361134Z","iopub.execute_input":"2025-01-25T03:21:09.361464Z","iopub.status.idle":"2025-01-25T03:21:09.545408Z","shell.execute_reply.started":"2025-01-25T03:21:09.361438Z","shell.execute_reply":"2025-01-25T03:21:09.544352Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## So from graph we understood to cover 90% of variance or spread we require n_components as 250","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}