{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:02.920212Z","iopub.execute_input":"2025-06-28T10:14:02.920508Z","iopub.status.idle":"2025-06-28T10:14:05.222392Z","shell.execute_reply.started":"2025-06-28T10:14:02.920484Z","shell.execute_reply":"2025-06-28T10:14:05.221541Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Principal Component Analysis (PCA)\n#### The dataset contains a high number of features (columns), which can lead to increased computational cost, risk of overfitting, and difficulty in interpretation.\n#### To address this, we will apply Principal Component Analysis (PCA), a dimensionality reduction technique that projects the original features into a lower-dimensional space while preserving as much variance as possible.","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:05.224125Z","iopub.execute_input":"2025-06-28T10:14:05.224585Z","iopub.status.idle":"2025-06-28T10:14:08.616102Z","shell.execute_reply.started":"2025-06-28T10:14:05.224553Z","shell.execute_reply":"2025-06-28T10:14:08.615093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:08.617385Z","iopub.execute_input":"2025-06-28T10:14:08.617682Z","iopub.status.idle":"2025-06-28T10:14:08.648146Z","shell.execute_reply.started":"2025-06-28T10:14:08.617661Z","shell.execute_reply":"2025-06-28T10:14:08.647162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:08.649100Z","iopub.execute_input":"2025-06-28T10:14:08.649448Z","iopub.status.idle":"2025-06-28T10:14:08.654889Z","shell.execute_reply.started":"2025-06-28T10:14:08.649423Z","shell.execute_reply":"2025-06-28T10:14:08.654210Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(df.iloc[18306,1:].values.reshape(28,28))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:08.657394Z","iopub.execute_input":"2025-06-28T10:14:08.657652Z","iopub.status.idle":"2025-06-28T10:14:08.983608Z","shell.execute_reply.started":"2025-06-28T10:14:08.657630Z","shell.execute_reply":"2025-06-28T10:14:08.982706Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Splitting training data into X feature and Y result\nX = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:08.984480Z","iopub.execute_input":"2025-06-28T10:14:08.984746Z","iopub.status.idle":"2025-06-28T10:14:08.989367Z","shell.execute_reply.started":"2025-06-28T10:14:08.984725Z","shell.execute_reply":"2025-06-28T10:14:08.988466Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Before applying PCA, we will train and evaluate our model using the original feature set.\n#### This will serve as our baseline to assess how much predictive performance is retained after reducing dimensions.\n","metadata":{}},{"cell_type":"code","source":"# Spliting data into training and testing set\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:08.990391Z","iopub.execute_input":"2025-06-28T10:14:08.990993Z","iopub.status.idle":"2025-06-28T10:14:10.830364Z","shell.execute_reply.started":"2025-06-28T10:14:08.990964Z","shell.execute_reply":"2025-06-28T10:14:10.829568Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:10.831832Z","iopub.execute_input":"2025-06-28T10:14:10.832344Z","iopub.status.idle":"2025-06-28T10:14:10.837966Z","shell.execute_reply.started":"2025-06-28T10:14:10.832314Z","shell.execute_reply":"2025-06-28T10:14:10.837138Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Calling KNN model to calculate the accuracy without applying PCA mean, by considering all the 784 columns\nfrom sklearn.neighbors import KNeighborsClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:10.838966Z","iopub.execute_input":"2025-06-28T10:14:10.839253Z","iopub.status.idle":"2025-06-28T10:14:11.165044Z","shell.execute_reply.started":"2025-06-28T10:14:10.839225Z","shell.execute_reply":"2025-06-28T10:14:11.164167Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:11.165960Z","iopub.execute_input":"2025-06-28T10:14:11.166250Z","iopub.status.idle":"2025-06-28T10:14:11.170234Z","shell.execute_reply.started":"2025-06-28T10:14:11.166228Z","shell.execute_reply":"2025-06-28T10:14:11.169323Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:11.171135Z","iopub.execute_input":"2025-06-28T10:14:11.171593Z","iopub.status.idle":"2025-06-28T10:14:11.338737Z","shell.execute_reply.started":"2025-06-28T10:14:11.171567Z","shell.execute_reply":"2025-06-28T10:14:11.337769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Calculating total time taken to predict the value while considering all the 784 columns\nimport time\nstart = time.time()\ny_pred = knn.predict(X_test)\nprint(time.time()-start)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:11.339596Z","iopub.execute_input":"2025-06-28T10:14:11.339863Z","iopub.status.idle":"2025-06-28T10:14:23.668614Z","shell.execute_reply.started":"2025-06-28T10:14:11.339842Z","shell.execute_reply":"2025-06-28T10:14:23.667693Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:23.669427Z","iopub.execute_input":"2025-06-28T10:14:23.669771Z","iopub.status.idle":"2025-06-28T10:14:23.674965Z","shell.execute_reply.started":"2025-06-28T10:14:23.669742Z","shell.execute_reply":"2025-06-28T10:14:23.674186Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Checking the accuracy score, turns out to be good, as the data is clean\n","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_pred, y_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:23.677688Z","iopub.execute_input":"2025-06-28T10:14:23.677930Z","iopub.status.idle":"2025-06-28T10:14:23.698632Z","shell.execute_reply.started":"2025-06-28T10:14:23.677912Z","shell.execute_reply":"2025-06-28T10:14:23.697771Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### We will apply PCA and reduce the number of components to 200.\n#### This number is chosen based on the assumption that it captures a significant portion of the total variance.\n#### PCA will transform the original feature space into 200 principal components that are uncorrelated and ranked by the variance they capture.\n","metadata":{}},{"cell_type":"code","source":"# Importing StandardScaler to make the data centered\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:23.699601Z","iopub.execute_input":"2025-06-28T10:14:23.699888Z","iopub.status.idle":"2025-06-28T10:14:23.714438Z","shell.execute_reply.started":"2025-06-28T10:14:23.699869Z","shell.execute_reply":"2025-06-28T10:14:23.713447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Centering the traing and testing data\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:23.735227Z","iopub.execute_input":"2025-06-28T10:14:23.735433Z","iopub.status.idle":"2025-06-28T10:14:24.311119Z","shell.execute_reply.started":"2025-06-28T10:14:23.735417Z","shell.execute_reply":"2025-06-28T10:14:24.310309Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Importing PCA from decompostion with 200 n_components, mean we are now considering 200 top high variance columns\nfrom sklearn.decomposition import PCA\npca = PCA(n_components = 200)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:24.312050Z","iopub.execute_input":"2025-06-28T10:14:24.312574Z","iopub.status.idle":"2025-06-28T10:14:24.316818Z","shell.execute_reply.started":"2025-06-28T10:14:24.312542Z","shell.execute_reply":"2025-06-28T10:14:24.315937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_pca = pca.fit_transform(X_train)\nX_test_pca = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:24.317579Z","iopub.execute_input":"2025-06-28T10:14:24.317800Z","iopub.status.idle":"2025-06-28T10:14:28.690112Z","shell.execute_reply.started":"2025-06-28T10:14:24.317778Z","shell.execute_reply":"2025-06-28T10:14:28.689154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_pca.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:28.691069Z","iopub.execute_input":"2025-06-28T10:14:28.691327Z","iopub.status.idle":"2025-06-28T10:14:28.696635Z","shell.execute_reply.started":"2025-06-28T10:14:28.691306Z","shell.execute_reply":"2025-06-28T10:14:28.695822Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### We will now train and evaluate the model using the transformed dataset with 200 principal components.","metadata":{}},{"cell_type":"code","source":"knn.fit(X_train_pca,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:28.697480Z","iopub.execute_input":"2025-06-28T10:14:28.697831Z","iopub.status.idle":"2025-06-28T10:14:28.767546Z","shell.execute_reply.started":"2025-06-28T10:14:28.697798Z","shell.execute_reply":"2025-06-28T10:14:28.766578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = knn.predict(X_test_pca)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:28.768482Z","iopub.execute_input":"2025-06-28T10:14:28.769352Z","iopub.status.idle":"2025-06-28T10:14:31.141730Z","shell.execute_reply.started":"2025-06-28T10:14:28.769318Z","shell.execute_reply":"2025-06-28T10:14:31.140856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# With 200 column the accuracy score is 95%\naccuracy_score(y_test, y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:31.142504Z","iopub.execute_input":"2025-06-28T10:14:31.142768Z","iopub.status.idle":"2025-06-28T10:14:31.150089Z","shell.execute_reply.started":"2025-06-28T10:14:31.142749Z","shell.execute_reply":"2025-06-28T10:14:31.149217Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Insight After Applying PCA\n\nAfter reducing the dataset to 200 principal components, we still achieve **95% of the original model's accuracy**.\n\n🎯 **Key Insight:**  \nThis demonstrates the effectiveness of PCA in preserving essential patterns in the data while significantly reducing dimensionality. Despite dropping a large number of original features, the model continues to perform at a high level, indicating that the majority of the predictive information is retained in these 200 components.\n\n💡 **Impact of PCA:**  \n- Reduces computational load and training time  \n- Minimizes risk of overfitting  \n- Helps in visualization and understanding of data structure  \n- Maintains high model accuracy (95% in our case)\n\nThis confirms that PCA is a powerful tool for simplifying complex datasets without a major compromise in performance.\n**-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------**","metadata":{}},{"cell_type":"markdown","source":"### Helps in Visualization\nPrincipal Component Analysis (PCA) allows us to reduce high-dimensional data to lower dimensions for better **visualization** as well.\n* When we reduce the data to 2 principal components, we can visualize it in 2D space.\n* When we reduce it to 3 principal components, we can explore the data in 3D space.\nThis is especially useful for understanding patterns, clusters, or separability in the data.","metadata":{}},{"cell_type":"code","source":"# transforming to a 2D coordinate system\npca = PCA(n_components=2)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:31.150976Z","iopub.execute_input":"2025-06-28T10:14:31.151397Z","iopub.status.idle":"2025-06-28T10:14:31.910745Z","shell.execute_reply.started":"2025-06-28T10:14:31.151364Z","shell.execute_reply":"2025-06-28T10:14:31.910081Z"},"_kg_hide-output":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Visualising in 2D coordinate\nimport plotly.express as px\nimport plotly.io as pio\ny_train_trf = y_train.astype(str)\nfig = px.scatter(x=X_train_trf[:,0],\n                 y=X_train_trf[:,1],\n                 color=y_train_trf,\n                 color_discrete_sequence=px.colors.qualitative.G10\n                )\n#pio.renderers.default = 'notebook_connected'\nfig.show(renderer='iframe')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:14:45.659448Z","iopub.execute_input":"2025-06-28T10:14:45.659821Z","iopub.status.idle":"2025-06-28T10:14:48.802341Z","shell.execute_reply.started":"2025-06-28T10:14:45.659793Z","shell.execute_reply":"2025-06-28T10:14:48.801467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transforming in 3D\npca = PCA(n_components=3)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:15:00.126417Z","iopub.execute_input":"2025-06-28T10:15:00.127273Z","iopub.status.idle":"2025-06-28T10:15:00.918890Z","shell.execute_reply.started":"2025-06-28T10:15:00.127244Z","shell.execute_reply":"2025-06-28T10:15:00.918001Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:15:07.765412Z","iopub.execute_input":"2025-06-28T10:15:07.765919Z","iopub.status.idle":"2025-06-28T10:15:07.774110Z","shell.execute_reply.started":"2025-06-28T10:15:07.765887Z","shell.execute_reply":"2025-06-28T10:15:07.773148Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x=X_train_trf[:,0], y=X_train_trf[:,1], z=X_train_trf[:,2],\n              color=y_train_trf)\nfig.update_layout(\n    margin=dict(l=20, r=20, t=20, b=20),\n    paper_bgcolor=\"LightSteelBlue\",\n)\nfig.show(renderer='iframe')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-06-28T10:15:16.542088Z","iopub.execute_input":"2025-06-28T10:15:16.542365Z","iopub.status.idle":"2025-06-28T10:15:16.721231Z","shell.execute_reply.started":"2025-06-28T10:15:16.542345Z","shell.execute_reply":"2025-06-28T10:15:16.720420Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}