{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:23.522053Z","iopub.execute_input":"2025-04-07T00:13:23.522510Z","iopub.status.idle":"2025-04-07T00:13:23.531402Z","shell.execute_reply.started":"2025-04-07T00:13:23.522476Z","shell.execute_reply":"2025-04-07T00:13:23.530062Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:25.830794Z","iopub.execute_input":"2025-04-07T00:13:25.831231Z","iopub.status.idle":"2025-04-07T00:13:29.444776Z","shell.execute_reply.started":"2025-04-07T00:13:25.831201Z","shell.execute_reply":"2025-04-07T00:13:29.443467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:29.446422Z","iopub.execute_input":"2025-04-07T00:13:29.446772Z","iopub.status.idle":"2025-04-07T00:13:29.484076Z","shell.execute_reply.started":"2025-04-07T00:13:29.446742Z","shell.execute_reply":"2025-04-07T00:13:29.482895Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.sample()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:29.486165Z","iopub.execute_input":"2025-04-07T00:13:29.486613Z","iopub.status.idle":"2025-04-07T00:13:29.512330Z","shell.execute_reply.started":"2025-04-07T00:13:29.486570Z","shell.execute_reply":"2025-04-07T00:13:29.511145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:32.158148Z","iopub.execute_input":"2025-04-07T00:13:32.158583Z","iopub.status.idle":"2025-04-07T00:13:32.163759Z","shell.execute_reply.started":"2025-04-07T00:13:32.158553Z","shell.execute_reply":"2025-04-07T00:13:32.162478Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(df.iloc[12089,1:].values.reshape(28,28)) # Here, it shows that 12089 row has label # 5 ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:34.816737Z","iopub.execute_input":"2025-04-07T00:13:34.817214Z","iopub.status.idle":"2025-04-07T00:13:35.205501Z","shell.execute_reply.started":"2025-04-07T00:13:34.817174Z","shell.execute_reply":"2025-04-07T00:13:35.204346Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:38.529052Z","iopub.execute_input":"2025-04-07T00:13:38.529456Z","iopub.status.idle":"2025-04-07T00:13:38.534531Z","shell.execute_reply.started":"2025-04-07T00:13:38.529428Z","shell.execute_reply":"2025-04-07T00:13:38.533189Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(X,y,random_state=42,test_size =0.2)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:40.256649Z","iopub.execute_input":"2025-04-07T00:13:40.256971Z","iopub.status.idle":"2025-04-07T00:13:41.123440Z","shell.execute_reply.started":"2025-04-07T00:13:40.256947Z","shell.execute_reply":"2025-04-07T00:13:41.122409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:43.522621Z","iopub.execute_input":"2025-04-07T00:13:43.523293Z","iopub.status.idle":"2025-04-07T00:13:43.530625Z","shell.execute_reply.started":"2025-04-07T00:13:43.523242Z","shell.execute_reply":"2025-04-07T00:13:43.529145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:13:45.324716Z","iopub.execute_input":"2025-04-07T00:13:45.325076Z","iopub.status.idle":"2025-04-07T00:13:45.331380Z","shell.execute_reply.started":"2025-04-07T00:13:45.325051Z","shell.execute_reply":"2025-04-07T00:13:45.330331Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Here we will try to train and test on KNN Model without implemting PCA","metadata":{}},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:27.459564Z","iopub.execute_input":"2025-04-07T00:14:27.460042Z","iopub.status.idle":"2025-04-07T00:14:27.738476Z","shell.execute_reply.started":"2025-04-07T00:14:27.460004Z","shell.execute_reply":"2025-04-07T00:14:27.737355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:31.448939Z","iopub.execute_input":"2025-04-07T00:14:31.449425Z","iopub.status.idle":"2025-04-07T00:14:31.454656Z","shell.execute_reply.started":"2025-04-07T00:14:31.449389Z","shell.execute_reply":"2025-04-07T00:14:31.452977Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn.fit(X_train,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:33.163333Z","iopub.execute_input":"2025-04-07T00:14:33.163696Z","iopub.status.idle":"2025-04-07T00:14:33.320165Z","shell.execute_reply.started":"2025-04-07T00:14:33.163670Z","shell.execute_reply":"2025-04-07T00:14:33.318891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_predict = knn.predict(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:34.910222Z","iopub.execute_input":"2025-04-07T00:14:34.910627Z","iopub.status.idle":"2025-04-07T00:14:48.751595Z","shell.execute_reply.started":"2025-04-07T00:14:34.910598Z","shell.execute_reply":"2025-04-07T00:14:48.750534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_test,y_predict)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:51.674471Z","iopub.execute_input":"2025-04-07T00:14:51.674837Z","iopub.status.idle":"2025-04-07T00:14:51.684202Z","shell.execute_reply.started":"2025-04-07T00:14:51.674801Z","shell.execute_reply":"2025-04-07T00:14:51.682988Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now applying PCA with 3 steps","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler #Step 1\nscaler = StandardScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:54.889086Z","iopub.execute_input":"2025-04-07T00:14:54.889450Z","iopub.status.idle":"2025-04-07T00:14:54.893813Z","shell.execute_reply.started":"2025-04-07T00:14:54.889425Z","shell.execute_reply":"2025-04-07T00:14:54.892804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:56.874986Z","iopub.execute_input":"2025-04-07T00:14:56.875371Z","iopub.status.idle":"2025-04-07T00:14:57.541652Z","shell.execute_reply.started":"2025-04-07T00:14:56.875343Z","shell.execute_reply":"2025-04-07T00:14:57.540657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#PCA\nfrom sklearn.decomposition import PCA\npca = PCA(n_components = None) #here none means all of the variables will be selected, STEP 2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:14:59.327447Z","iopub.execute_input":"2025-04-07T00:14:59.327869Z","iopub.status.idle":"2025-04-07T00:14:59.334052Z","shell.execute_reply.started":"2025-04-07T00:14:59.327839Z","shell.execute_reply":"2025-04-07T00:14:59.332152Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:01.053156Z","iopub.execute_input":"2025-04-07T00:15:01.053569Z","iopub.status.idle":"2025-04-07T00:15:06.020216Z","shell.execute_reply.started":"2025-04-07T00:15:01.053541Z","shell.execute_reply":"2025-04-07T00:15:06.019089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:06.022877Z","iopub.execute_input":"2025-04-07T00:15:06.023215Z","iopub.status.idle":"2025-04-07T00:15:06.030528Z","shell.execute_reply.started":"2025-04-07T00:15:06.023188Z","shell.execute_reply":"2025-04-07T00:15:06.029520Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:06.032688Z","iopub.execute_input":"2025-04-07T00:15:06.033068Z","iopub.status.idle":"2025-04-07T00:15:06.060250Z","shell.execute_reply.started":"2025-04-07T00:15:06.033036Z","shell.execute_reply":"2025-04-07T00:15:06.059026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Now applying KNN on PCA","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:06.684980Z","iopub.execute_input":"2025-04-07T00:15:06.685555Z","iopub.status.idle":"2025-04-07T00:15:06.690632Z","shell.execute_reply.started":"2025-04-07T00:15:06.685503Z","shell.execute_reply":"2025-04-07T00:15:06.689378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:08.424106Z","iopub.execute_input":"2025-04-07T00:15:08.424513Z","iopub.status.idle":"2025-04-07T00:15:08.429456Z","shell.execute_reply.started":"2025-04-07T00:15:08.424485Z","shell.execute_reply":"2025-04-07T00:15:08.428308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn.fit(X_train_trf,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:10.451859Z","iopub.execute_input":"2025-04-07T00:15:10.452330Z","iopub.status.idle":"2025-04-07T00:15:10.660473Z","shell.execute_reply.started":"2025-04-07T00:15:10.452297Z","shell.execute_reply":"2025-04-07T00:15:10.659272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = knn.predict(X_test_trf)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:12.299873Z","iopub.execute_input":"2025-04-07T00:15:12.300289Z","iopub.status.idle":"2025-04-07T00:15:20.527735Z","shell.execute_reply.started":"2025-04-07T00:15:12.300258Z","shell.execute_reply":"2025-04-07T00:15:20.526926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(y_test,y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:20.529216Z","iopub.execute_input":"2025-04-07T00:15:20.529587Z","iopub.status.idle":"2025-04-07T00:15:20.537001Z","shell.execute_reply.started":"2025-04-07T00:15:20.529551Z","shell.execute_reply":"2025-04-07T00:15:20.536156Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for i in range(1,785):\n    pca = PCA(n_components=i)\n    X_train_trf = pca.fit_transform(X_train)\n    X_test_trf = pca.transform(X_test)\n    \n    knn = KNeighborsClassifier()\n    \n    knn.fit(X_train_trf,y_train)\n    \n    y_pred = knn.predict(X_test_trf)\n    \n    print(accuracy_score(y_test,y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-05T18:10:52.519879Z","iopub.execute_input":"2025-04-05T18:10:52.520250Z","execution_failed":"2025-04-07T00:06:51.111Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transforming to a 2D coordinate system\npca = PCA(n_components=200)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:20.538623Z","iopub.execute_input":"2025-04-07T00:15:20.538949Z","iopub.status.idle":"2025-04-07T00:15:23.951544Z","shell.execute_reply.started":"2025-04-07T00:15:20.538924Z","shell.execute_reply":"2025-04-07T00:15:23.950419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter(x=X_train_trf[:,0],\n                 y=X_train_trf[:,1],\n                 color=y_train_trf,\n                 color_discrete_sequence=px.colors.qualitative.G10\n                )\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:38:12.031502Z","iopub.execute_input":"2025-04-07T00:38:12.031952Z","iopub.status.idle":"2025-04-07T00:38:12.143955Z","shell.execute_reply.started":"2025-04-07T00:38:12.031906Z","shell.execute_reply":"2025-04-07T00:38:12.142651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transforming in 3D\npca = PCA(n_components=3)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:26:30.484387Z","iopub.execute_input":"2025-04-07T00:26:30.484792Z","iopub.status.idle":"2025-04-07T00:26:31.368707Z","shell.execute_reply.started":"2025-04-07T00:26:30.484763Z","shell.execute_reply":"2025-04-07T00:26:31.367593Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x=X_train_trf[:,0], y=X_train_trf[:,1], z=X_train_trf[:,2],\n              color=y_train_trf)\nfig.update_layout(\n    margin=dict(l=20, r=20, t=20, b=20),\n    paper_bgcolor=\"LightSteelBlue\",\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:38:03.650022Z","iopub.execute_input":"2025-04-07T00:38:03.650405Z","iopub.status.idle":"2025-04-07T00:38:03.786195Z","shell.execute_reply.started":"2025-04-07T00:38:03.650377Z","shell.execute_reply":"2025-04-07T00:38:03.784254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_\n# Eigen values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:33.902171Z","iopub.execute_input":"2025-04-07T00:15:33.902534Z","iopub.status.idle":"2025-04-07T00:15:33.910106Z","shell.execute_reply.started":"2025-04-07T00:15:33.902507Z","shell.execute_reply":"2025-04-07T00:15:33.908468Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape\n# Eigen vectors","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:35.609722Z","iopub.execute_input":"2025-04-07T00:15:35.610154Z","iopub.status.idle":"2025-04-07T00:15:35.617210Z","shell.execute_reply.started":"2025-04-07T00:15:35.610103Z","shell.execute_reply":"2025-04-07T00:15:35.615731Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_ratio_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:37.606449Z","iopub.execute_input":"2025-04-07T00:15:37.606864Z","iopub.status.idle":"2025-04-07T00:15:37.614258Z","shell.execute_reply.started":"2025-04-07T00:15:37.606834Z","shell.execute_reply":"2025-04-07T00:15:37.613147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca = PCA(n_components=None)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:39.595079Z","iopub.execute_input":"2025-04-07T00:15:39.595542Z","iopub.status.idle":"2025-04-07T00:15:43.962018Z","shell.execute_reply.started":"2025-04-07T00:15:39.595511Z","shell.execute_reply":"2025-04-07T00:15:43.960891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:43.963302Z","iopub.execute_input":"2025-04-07T00:15:43.963690Z","iopub.status.idle":"2025-04-07T00:15:43.969751Z","shell.execute_reply.started":"2025-04-07T00:15:43.963664Z","shell.execute_reply":"2025-04-07T00:15:43.968923Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:45.930959Z","iopub.execute_input":"2025-04-07T00:15:45.931421Z","iopub.status.idle":"2025-04-07T00:15:45.939390Z","shell.execute_reply.started":"2025-04-07T00:15:45.931388Z","shell.execute_reply":"2025-04-07T00:15:45.937939Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"np.cumsum(pca.explained_variance_ratio_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:47.872064Z","iopub.execute_input":"2025-04-07T00:15:47.872537Z","iopub.status.idle":"2025-04-07T00:15:47.886780Z","shell.execute_reply.started":"2025-04-07T00:15:47.872503Z","shell.execute_reply":"2025-04-07T00:15:47.885571Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(np.cumsum(pca.explained_variance_ratio_))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-07T00:15:51.492840Z","iopub.execute_input":"2025-04-07T00:15:51.493357Z","iopub.status.idle":"2025-04-07T00:15:51.705618Z","shell.execute_reply.started":"2025-04-07T00:15:51.493313Z","shell.execute_reply":"2025-04-07T00:15:51.704421Z"}},"outputs":[],"execution_count":null}]}