{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":3004,"databundleVersionId":861823}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:32.239873Z","iopub.execute_input":"2026-02-20T10:15:32.240629Z","iopub.status.idle":"2026-02-20T10:15:32.247805Z","shell.execute_reply.started":"2026-02-20T10:15:32.240596Z","shell.execute_reply":"2026-02-20T10:15:32.246730Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### TO UNDERSTAND PCA IN DEPTH -> [PCA STEP BY STEP -> THE MATH BEHIND IT (CODED)](https://github.com/curiouscurrent/MLOPS/blob/main/Feature-Engineering/Feature-Extraction/Principal-Component-Analysis/PCA_step_by_step.ipynb)","metadata":{}},{"cell_type":"code","source":"# let us use the training data\ndf = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:32.250112Z","iopub.execute_input":"2026-02-20T10:15:32.250563Z","iopub.status.idle":"2026-02-20T10:15:35.082080Z","shell.execute_reply.started":"2026-02-20T10:15:32.250533Z","shell.execute_reply":"2026-02-20T10:15:35.081032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.sample(1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:35.083606Z","iopub.execute_input":"2026-02-20T10:15:35.083873Z","iopub.status.idle":"2026-02-20T10:15:35.101527Z","shell.execute_reply.started":"2026-02-20T10:15:35.083849Z","shell.execute_reply":"2026-02-20T10:15:35.100393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:35.103019Z","iopub.execute_input":"2026-02-20T10:15:35.103339Z","iopub.status.idle":"2026-02-20T10:15:35.123843Z","shell.execute_reply.started":"2026-02-20T10:15:35.103313Z","shell.execute_reply":"2026-02-20T10:15:35.122998Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"we can see that we have \n1. 784 input columns (where 1 pixel = 1 column)\n2. the 1st column is the label (output column)\n3. 1 row in data = 1 image (28*28)\n\n","metadata":{}},{"cell_type":"markdown","source":"28 pixel values stacked next to each other one after the other (row wise) untill all 784 values are placed \n\n2D(28*28)-> 1D(784)","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:35.126544Z","iopub.execute_input":"2026-02-20T10:15:35.127399Z","iopub.status.idle":"2026-02-20T10:15:35.142574Z","shell.execute_reply.started":"2026-02-20T10:15:35.127353Z","shell.execute_reply":"2026-02-20T10:15:35.141443Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.sample(1)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:35.145018Z","iopub.execute_input":"2026-02-20T10:15:35.145424Z","iopub.status.idle":"2026-02-20T10:15:35.175019Z","shell.execute_reply.started":"2026-02-20T10:15:35.145383Z","shell.execute_reply":"2026-02-20T10:15:35.174218Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.imshow(df.iloc[6797,1:].values.reshape(28,28))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:35.176675Z","iopub.execute_input":"2026-02-20T10:15:35.177043Z","iopub.status.idle":"2026-02-20T10:15:35.326837Z","shell.execute_reply.started":"2026-02-20T10:15:35.177004Z","shell.execute_reply":"2026-02-20T10:15:35.325863Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:35.328375Z","iopub.execute_input":"2026-02-20T10:15:35.328641Z","iopub.status.idle":"2026-02-20T10:15:35.334782Z","shell.execute_reply.started":"2026-02-20T10:15:35.328618Z","shell.execute_reply":"2026-02-20T10:15:35.333671Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train test split \nfrom sklearn.model_selection import train_test_split\nX_train,X_test,y_train,y_test = train_test_split(\n    X,y,test_size=0.2,random_state=12\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:35.336116Z","iopub.execute_input":"2026-02-20T10:15:35.336755Z","iopub.status.idle":"2026-02-20T10:15:36.300856Z","shell.execute_reply.started":"2026-02-20T10:15:35.336727Z","shell.execute_reply":"2026-02-20T10:15:36.299786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:36.302002Z","iopub.execute_input":"2026-02-20T10:15:36.302344Z","iopub.status.idle":"2026-02-20T10:15:36.308785Z","shell.execute_reply.started":"2026-02-20T10:15:36.302318Z","shell.execute_reply":"2026-02-20T10:15:36.307864Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. we will implement Kneighboursclassifier \n2. calculates the distance from a given image to all the other images\n3. find the images with the closest distance to the given image (they are the closest neighbors)\n4. if my closest neighbours are 8, then i am 8 (by using weights as uniform or distance)\n","metadata":{}},{"cell_type":"code","source":"\nfrom sklearn.neighbors import KNeighborsClassifier\nknn = KNeighborsClassifier()\nknn.fit(X_train,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:36.309840Z","iopub.execute_input":"2026-02-20T10:15:36.310224Z","iopub.status.idle":"2026-02-20T10:15:36.453988Z","shell.execute_reply.started":"2026-02-20T10:15:36.310175Z","shell.execute_reply":"2026-02-20T10:15:36.453015Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_test.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:36.455503Z","iopub.execute_input":"2026-02-20T10:15:36.456257Z","iopub.status.idle":"2026-02-20T10:15:36.462554Z","shell.execute_reply.started":"2026-02-20T10:15:36.456215Z","shell.execute_reply":"2026-02-20T10:15:36.461484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. for predicting one label it needs to calculate the\n2. distance of this image from all the other images in the training data\n3. but here we pass 8400 images (test), distances to be calculated \n4. from every image in the test sit to all 33600 images in the train set ","metadata":{}},{"cell_type":"code","source":"import time \nstart = time.time()\n\ny_pred = knn.predict(X_test)\nprint(time.time() - start)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:36.463481Z","iopub.execute_input":"2026-02-20T10:15:36.463811Z","iopub.status.idle":"2026-02-20T10:15:57.762561Z","shell.execute_reply.started":"2026-02-20T10:15:36.463777Z","shell.execute_reply":"2026-02-20T10:15:57.761417Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_test,y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:57.765926Z","iopub.execute_input":"2026-02-20T10:15:57.766314Z","iopub.status.idle":"2026-02-20T10:15:57.774755Z","shell.execute_reply.started":"2026-02-20T10:15:57.766288Z","shell.execute_reply":"2026-02-20T10:15:57.773925Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Principle Component Analysis","metadata":{}},{"cell_type":"markdown","source":"PCA is a technique which can transform a higher dimensional data to a lower dimensional data while keeping the essence of the data, it is a dimensionality reduction technique ","metadata":{}},{"cell_type":"markdown","source":"## STEP-1 : MEAN CENTERING","metadata":{}},{"cell_type":"markdown","source":"subtract every value from the mean of that particular column, and then once everything is done, take the mean of that column, it will be 0, basically we are shifting the mean of the data cloud to the origin.","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:57.775690Z","iopub.execute_input":"2026-02-20T10:15:57.775913Z","iopub.status.idle":"2026-02-20T10:15:57.792543Z","shell.execute_reply.started":"2026-02-20T10:15:57.775892Z","shell.execute_reply":"2026-02-20T10:15:57.791387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:57.793951Z","iopub.execute_input":"2026-02-20T10:15:57.794425Z","iopub.status.idle":"2026-02-20T10:15:58.320942Z","shell.execute_reply.started":"2026-02-20T10:15:57.794386Z","shell.execute_reply":"2026-02-20T10:15:58.320030Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## STEP-2 : APPLY PCA ","metadata":{}},{"cell_type":"markdown","source":"1. We first find the covariance matrix\n2. Then we find the eigen values and eigen vectors of the covariance matrix\n\n(A matrix is nothing but a linear transformation)","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(\n    n_components=200)\n# n_components -> no of principle components ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:58.322115Z","iopub.execute_input":"2026-02-20T10:15:58.322586Z","iopub.status.idle":"2026-02-20T10:15:58.327391Z","shell.execute_reply.started":"2026-02-20T10:15:58.322558Z","shell.execute_reply":"2026-02-20T10:15:58.326326Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"    Principle components : Principal axes in feature space, representing the directions of maximum variance in the data.","metadata":{}},{"cell_type":"markdown","source":"if you keep n_components=None, we will have 784 principle components (since we have 784 columns)\n\nBut we are gonna select only the top principle components that explains the maximum variance of the data. \n\n","metadata":{}},{"cell_type":"markdown","source":"the largest eigen vector (the one with the largest eigen value) explains the maximum variance of the data. -> Principle Component 1 ","metadata":{}},{"cell_type":"markdown","source":"    Upon fitting PCA we find the eigen values and the eigen vectors ","metadata":{}},{"cell_type":"markdown","source":"The entire data is projected on to these principle components \n\n1. Find the top principle components (n_components) upon fitting PCA\n2. Then transform using this formula : dot product of (entire data,transpose(Principal components))\n\n","metadata":{}},{"cell_type":"code","source":"X_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:58.328623Z","iopub.execute_input":"2026-02-20T10:15:58.329073Z","iopub.status.idle":"2026-02-20T10:15:59.816865Z","shell.execute_reply.started":"2026-02-20T10:15:58.328978Z","shell.execute_reply":"2026-02-20T10:15:59.816044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:59.818083Z","iopub.execute_input":"2026-02-20T10:15:59.819008Z","iopub.status.idle":"2026-02-20T10:15:59.824922Z","shell.execute_reply.started":"2026-02-20T10:15:59.818975Z","shell.execute_reply":"2026-02-20T10:15:59.823917Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"we can see we have 200 principle components, these are the components that explain the maximum variance in the data \n\n(784 components -> 200 components)","metadata":{}},{"cell_type":"code","source":"knn = KNeighborsClassifier()\nknn.fit(X_train_trf,y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:59.826401Z","iopub.execute_input":"2026-02-20T10:15:59.826784Z","iopub.status.idle":"2026-02-20T10:15:59.856813Z","shell.execute_reply.started":"2026-02-20T10:15:59.826743Z","shell.execute_reply":"2026-02-20T10:15:59.855500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = knn.predict(X_test_trf)\naccuracy_score(y_test,y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:15:59.858509Z","iopub.execute_input":"2026-02-20T10:15:59.858880Z","iopub.status.idle":"2026-02-20T10:16:02.536871Z","shell.execute_reply.started":"2026-02-20T10:15:59.858838Z","shell.execute_reply":"2026-02-20T10:16:02.536017Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"If Σ is the covariance matrix of the centered data, PCA solves:","metadata":{}},{"cell_type":"markdown","source":"![image.png](attachment:46360aa0-93b4-42f3-95b2-f7697bdefe87.png)\n\nwhere:\n\n\nλ = eigenvalues\n\n\nv = eigenvectors","metadata":{},"attachments":{"f14fa37a-0f53-48ab-9ca6-542127169dea.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAGsAAAAtCAIAAAAm3AIqAAAF7klEQVR4AeyXb2gTZxjAHySwg37Ih4GnK1qmpKdbXe1Yqg7absxG3BoVaaKb2emMyT6YRFCrdFnwQ4gDLcKW6qYzKLcIM4mrphnFszKjMmc65uKHsmu/LC0dXqHQEwo3KGzvXZrmT5v2krdhQ+54cnfvn+dPfu/zPne37B/1wCOwDNQDj4BKEI8fgEpQJYhLAFdfzUGVIC4BXH01B1WCuARw9dUcVAniEsDVV3NQJaiMQOVmKc9BgbsTjfYsLHFuqnKhlmA5FXE7Ov3RJ7xYglKZU5UT5JiDFst+i8XqDf3EsjlyK+A95pKH9hvP3C8zjqVVI1vpxpEzlhad6Sq/tJbnWlNOsNEXsmuRgWkuRXV0f9U9K4HYwPDo89HbHfUA43zFI0YhLCqElqLPeBoB4t+EUovOxpugnCDAlq7+UxRyl+y0+IfQNU+0WzyXT9b8kXqW1/sfNqp30s0Ag5eYwcoGUQpBAOpI0NeAAkq6d3uT0+gmT6j3zTD2v8hBOSzSfMAAkGLCSblZqVNpBEFDOa/40G6FkbNtLrawTm/YtHPiz+yumRYFnhdyny1TAi8UKlXqnwEQ2/fsAODPf5eY6wNFgkLJSQJxArXnzlu8p0SCyOAaZ+yaiQAQgtbDfQLqyIrG0HXDXpNuC/Hj77RaO/fVrdS5H8ldU2HLylW6VYfZnLjlgYqdqow7jQAiE8p/vgn3jzdts372Ud3y9e6EHIwYsSx/VbfKxZYRSukEAbTGc8G96KEihPcfZXNTLOtfZDs+1/ofRL6gNwLPJuS8fKmWWoNmPOVG0Hke4Xsc+vU63eKid/QorBWEyWKSEN6MZ/1NsUc92u57kdMfS6E9loMh1lLSwj/l5ECzc5XclUMQMTRcCNAa0O7as6lqPi98yP+Xzd4A4oMoiv2NWik80NR7rnSQsEnmOI8Wud0Xu/fw4eLS32Uk59GfrwvVDLRdxMvMbOLzYf/4J/Z6jXi/TwqNWi2rNXiCx0h4U+Yodyg/lUcQuPNeZoOv/4IBpWKOM1GYkMvcy+bgdZoEnvmaBcJOt2amvFKzrrlRKqOZjrwroSUVHVpCk6dXrCH0WutcQLcjhmHmphwYAGkKhi0kjDEX+4A4RBsyplbUrGspHloxF6i/HIJCn3VrV20k6qQy7pEhSUa+beuQS4mG0FahGn67dwAIi7Fldtpv8cnWFqX5I1ks/8dHLHX7hk7cDXTZaIQw2tM7g7BKWgD+Tm8CCHpXy6yD5M+ThuZyQiud4JC/zfX3uV8CBemHQhETj5MEihbdSpLewsZ3Z6MUw8Ehs1He0dJ44U985He4HErE/2iGRqGJTJu7anzrYMp+N+asRa+xNifaqr3Xe7MlO72Fjc1vZxSmwgxn3iGV6UyP4muJBMfClg9C5h+Dpuo5HqY5f1eUoqRX7vQYP4bqMrXx9XQL0MZhwGUrHiXRQHvcHiVCN2TXKWM9e+XOb9W7fjdcj3n06RpDmT9Ey8ZeDM4+f/hnKLTXNtZnNgcfZMBpQ5OyVhTflUJQYK2breLJy2Ytn3eMJNnv/dbNeu8g1FRnNwJZjWhOTqZfeKYF5kTI4JVeg4rGprQOktriALkvm/SdiXpvf2B7Gp/kjTrgRF94idCtDEJyxVqAiclJaRDdMMdCBp9ULtPt0s7KCab8O9rDArBH9bqCo66p3e4Oy995K3L+HLHL090KZ81brS5H+3tt3Kc3nMUTEJbkGPEf8iS1eyOxI2jxciymv/AG/JeepDsJ06luA5xt32Z1uNqbdnO2H5zlJSAyp5xgjTP+fNEDIUNGZ0RD0TeGx3+9dtrtCdx+4GvOJsXMhCW/rLbFhkeHC98QkBuSjowOD/e7UFlELSS1dASFxpz2uAN34r4WjNCUE0Ruy5H01sxJzXKMKNZB3qRH7Tzz0QhJSm8I2TG5a6GSkJ26wF3FCS7g+8UYUgnirqNKUCWISwBXX81BlSAuAVx9NQdVgrgEcPXLyUFcny+WvkoQdz1VgipBXAK4+moOqgRxCeDqqzmIS/BfAAAA///jiCEcAAAABklEQVQDANRhokciyO7vAAAAAElFTkSuQmCC"},"46360aa0-93b4-42f3-95b2-f7697bdefe87.png":{"image/png":"iVBORw0KGgoAAAANSUhEUgAAAGsAAAAtCAIAAAAm3AIqAAAF7klEQVR4AeyXb2gTZxjAHySwg37Ih4GnK1qmpKdbXe1Yqg7absxG3BoVaaKb2emMyT6YRFCrdFnwQ4gDLcKW6qYzKLcIM4mrphnFszKjMmc65uKHsmu/LC0dXqHQEwo3KGzvXZrmT5v2krdhQ+54cnfvn+dPfu/zPne37B/1wCOwDNQDj4BKEI8fgEpQJYhLAFdfzUGVIC4BXH01B1WCuARw9dUcVAniEsDVV3NQJaiMQOVmKc9BgbsTjfYsLHFuqnKhlmA5FXE7Ov3RJ7xYglKZU5UT5JiDFst+i8XqDf3EsjlyK+A95pKH9hvP3C8zjqVVI1vpxpEzlhad6Sq/tJbnWlNOsNEXsmuRgWkuRXV0f9U9K4HYwPDo89HbHfUA43zFI0YhLCqElqLPeBoB4t+EUovOxpugnCDAlq7+UxRyl+y0+IfQNU+0WzyXT9b8kXqW1/sfNqp30s0Ag5eYwcoGUQpBAOpI0NeAAkq6d3uT0+gmT6j3zTD2v8hBOSzSfMAAkGLCSblZqVNpBEFDOa/40G6FkbNtLrawTm/YtHPiz+yumRYFnhdyny1TAi8UKlXqnwEQ2/fsAODPf5eY6wNFgkLJSQJxArXnzlu8p0SCyOAaZ+yaiQAQgtbDfQLqyIrG0HXDXpNuC/Hj77RaO/fVrdS5H8ldU2HLylW6VYfZnLjlgYqdqow7jQAiE8p/vgn3jzdts372Ud3y9e6EHIwYsSx/VbfKxZYRSukEAbTGc8G96KEihPcfZXNTLOtfZDs+1/ofRL6gNwLPJuS8fKmWWoNmPOVG0Hke4Xsc+vU63eKid/QorBWEyWKSEN6MZ/1NsUc92u57kdMfS6E9loMh1lLSwj/l5ECzc5XclUMQMTRcCNAa0O7as6lqPi98yP+Xzd4A4oMoiv2NWik80NR7rnSQsEnmOI8Wud0Xu/fw4eLS32Uk59GfrwvVDLRdxMvMbOLzYf/4J/Z6jXi/TwqNWi2rNXiCx0h4U+Yodyg/lUcQuPNeZoOv/4IBpWKOM1GYkMvcy+bgdZoEnvmaBcJOt2amvFKzrrlRKqOZjrwroSUVHVpCk6dXrCH0WutcQLcjhmHmphwYAGkKhi0kjDEX+4A4RBsyplbUrGspHloxF6i/HIJCn3VrV20k6qQy7pEhSUa+beuQS4mG0FahGn67dwAIi7Fldtpv8cnWFqX5I1ks/8dHLHX7hk7cDXTZaIQw2tM7g7BKWgD+Tm8CCHpXy6yD5M+ThuZyQiud4JC/zfX3uV8CBemHQhETj5MEihbdSpLewsZ3Z6MUw8Ehs1He0dJ44U985He4HErE/2iGRqGJTJu7anzrYMp+N+asRa+xNifaqr3Xe7MlO72Fjc1vZxSmwgxn3iGV6UyP4muJBMfClg9C5h+Dpuo5HqY5f1eUoqRX7vQYP4bqMrXx9XQL0MZhwGUrHiXRQHvcHiVCN2TXKWM9e+XOb9W7fjdcj3n06RpDmT9Ey8ZeDM4+f/hnKLTXNtZnNgcfZMBpQ5OyVhTflUJQYK2breLJy2Ytn3eMJNnv/dbNeu8g1FRnNwJZjWhOTqZfeKYF5kTI4JVeg4rGprQOktriALkvm/SdiXpvf2B7Gp/kjTrgRF94idCtDEJyxVqAiclJaRDdMMdCBp9ULtPt0s7KCab8O9rDArBH9bqCo66p3e4Oy995K3L+HLHL090KZ81brS5H+3tt3Kc3nMUTEJbkGPEf8iS1eyOxI2jxciymv/AG/JeepDsJ06luA5xt32Z1uNqbdnO2H5zlJSAyp5xgjTP+fNEDIUNGZ0RD0TeGx3+9dtrtCdx+4GvOJsXMhCW/rLbFhkeHC98QkBuSjowOD/e7UFlELSS1dASFxpz2uAN34r4WjNCUE0Ruy5H01sxJzXKMKNZB3qRH7Tzz0QhJSm8I2TG5a6GSkJ26wF3FCS7g+8UYUgnirqNKUCWISwBXX81BlSAuAVx9NQdVgrgEcPXLyUFcny+WvkoQdz1VgipBXAK4+moOqgRxCeDqqzmIS/BfAAAA///jiCEcAAAABklEQVQDANRhokciyO7vAAAAAElFTkSuQmCC"}}},{"cell_type":"markdown","source":"\nlinear transformation(Σ) when applied on vector(v) = multiplying the same vector(v) with a scalar λ (eigen value)","metadata":{}},{"cell_type":"markdown","source":"**Eigen values -> Equal to n_components largest eigenvalues of the covariance matrix of X.**","metadata":{}},{"cell_type":"markdown","source":"lambda is the stretching or shrinking factor ","metadata":{}},{"cell_type":"code","source":"pca.explained_variance_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:02.537975Z","iopub.execute_input":"2026-02-20T10:16:02.538265Z","iopub.status.idle":"2026-02-20T10:16:02.547566Z","shell.execute_reply.started":"2026-02-20T10:16:02.538240Z","shell.execute_reply":"2026-02-20T10:16:02.546393Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Eigen vectors : ","metadata":{}},{"cell_type":"code","source":"pca.components_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:02.549016Z","iopub.execute_input":"2026-02-20T10:16:02.549457Z","iopub.status.idle":"2026-02-20T10:16:02.568506Z","shell.execute_reply.started":"2026-02-20T10:16:02.549417Z","shell.execute_reply":"2026-02-20T10:16:02.567557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# in percentage (mul by 100)\npca.explained_variance_ratio_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:02.569756Z","iopub.execute_input":"2026-02-20T10:16:02.570159Z","iopub.status.idle":"2026-02-20T10:16:02.592224Z","shell.execute_reply.started":"2026-02-20T10:16:02.570094Z","shell.execute_reply":"2026-02-20T10:16:02.590737Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"1. it will find the covariance matrix of the mean centered data\n2. then it finds the eigen values and eigen vectors of the covariance matrix\n3. project the entire data onto these principal components (eigen vectors)","metadata":{}},{"cell_type":"markdown","source":"    for i in range(1,785):\n        pca = PCA(n_components=i)\n        X_train_trf = pca.fit_transform(X_train)\n        X_test_trf = pca.transform(X_test)\n    \n        knn = KNeighborsClassifier()\n        knn.fit(X_train_trf,y_train)\n        y_pred = knn.predict(X_test_trf)\n        print(accuracy_score(y_test,y_pred))","metadata":{}},{"cell_type":"markdown","source":"### Transforming to a 2D coordinate system","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=2)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:02.593565Z","iopub.execute_input":"2026-02-20T10:16:02.593956Z","iopub.status.idle":"2026-02-20T10:16:03.662607Z","shell.execute_reply.started":"2026-02-20T10:16:02.593898Z","shell.execute_reply":"2026-02-20T10:16:03.661564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:03.663811Z","iopub.execute_input":"2026-02-20T10:16:03.664308Z","iopub.status.idle":"2026-02-20T10:16:03.671357Z","shell.execute_reply.started":"2026-02-20T10:16:03.664279Z","shell.execute_reply":"2026-02-20T10:16:03.670342Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:03.672665Z","iopub.execute_input":"2026-02-20T10:16:03.673288Z","iopub.status.idle":"2026-02-20T10:16:03.696773Z","shell.execute_reply.started":"2026-02-20T10:16:03.673257Z","shell.execute_reply":"2026-02-20T10:16:03.695596Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"(784 components -> 2 components)","metadata":{}},{"cell_type":"code","source":"y_train","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:03.698074Z","iopub.execute_input":"2026-02-20T10:16:03.698708Z","iopub.status.idle":"2026-02-20T10:16:03.721368Z","shell.execute_reply.started":"2026-02-20T10:16:03.698676Z","shell.execute_reply":"2026-02-20T10:16:03.720261Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype('str')\nfig = px.scatter(\n    x = X_train_trf[:,0],\n    y = X_train_trf[:,1],\n    color = y_train_trf,\n    color_discrete_sequence = px.colors.qualitative.G10\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:16.064764Z","iopub.execute_input":"2026-02-20T10:16:16.065517Z","iopub.status.idle":"2026-02-20T10:16:16.164888Z","shell.execute_reply.started":"2026-02-20T10:16:16.065481Z","shell.execute_reply":"2026-02-20T10:16:16.163475Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"we can see similar numbers overlapping -> for ex : 3 and 8, since 3 is half of 8","metadata":{}},{"cell_type":"markdown","source":"all 33000 datapoints are displayed here","metadata":{}},{"cell_type":"markdown","source":"### Transforming to a 3D coordinate system","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=3)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:03.836278Z","iopub.execute_input":"2026-02-20T10:16:03.836762Z","iopub.status.idle":"2026-02-20T10:16:04.916961Z","shell.execute_reply.started":"2026-02-20T10:16:03.836731Z","shell.execute_reply":"2026-02-20T10:16:04.915314Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:04.918605Z","iopub.execute_input":"2026-02-20T10:16:04.919050Z","iopub.status.idle":"2026-02-20T10:16:04.925734Z","shell.execute_reply.started":"2026-02-20T10:16:04.919021Z","shell.execute_reply":"2026-02-20T10:16:04.924790Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:04.927116Z","iopub.execute_input":"2026-02-20T10:16:04.927555Z","iopub.status.idle":"2026-02-20T10:16:04.947871Z","shell.execute_reply.started":"2026-02-20T10:16:04.927528Z","shell.execute_reply":"2026-02-20T10:16:04.946727Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"784 -> 3","metadata":{}},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype('str')\nfig = px.scatter_3d(\n    df,\n    x = X_train_trf[:,0],\n    y = X_train_trf[:,1],\n    z = X_train_trf[:,2],\n    color = y_train_trf\n)\nfig.update_layout(\n    margin = dict(\n        l=20, r=20, t=20, b=20\n    ),\n    paper_bgcolor = 'LightSteelBlue'\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:04.949071Z","iopub.execute_input":"2026-02-20T10:16:04.949519Z","iopub.status.idle":"2026-02-20T10:16:05.066100Z","shell.execute_reply.started":"2026-02-20T10:16:04.949494Z","shell.execute_reply":"2026-02-20T10:16:05.064764Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"we can see there is no overlap between different digits -> like 0 and 7, but there is overlap between similar digits like (0,9), (1,7), (3,8),...","metadata":{}},{"cell_type":"code","source":"pca = PCA(n_components=None) # None -> considers all the 784 components\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:05.067430Z","iopub.execute_input":"2026-02-20T10:16:05.067873Z","iopub.status.idle":"2026-02-20T10:16:07.829799Z","shell.execute_reply.started":"2026-02-20T10:16:05.067836Z","shell.execute_reply":"2026-02-20T10:16:07.828375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_.shape # eigen values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:07.831209Z","iopub.execute_input":"2026-02-20T10:16:07.831569Z","iopub.status.idle":"2026-02-20T10:16:07.838924Z","shell.execute_reply.started":"2026-02-20T10:16:07.831537Z","shell.execute_reply":"2026-02-20T10:16:07.837714Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape # eigen vectors","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:07.840216Z","iopub.execute_input":"2026-02-20T10:16:07.840479Z","iopub.status.idle":"2026-02-20T10:16:07.861862Z","shell.execute_reply.started":"2026-02-20T10:16:07.840455Z","shell.execute_reply":"2026-02-20T10:16:07.860922Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.explained_variance_ratio_[0] # you can convert it to percentage ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:07.863318Z","iopub.execute_input":"2026-02-20T10:16:07.863760Z","iopub.status.idle":"2026-02-20T10:16:07.883767Z","shell.execute_reply.started":"2026-02-20T10:16:07.863730Z","shell.execute_reply":"2026-02-20T10:16:07.882821Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"the first principle component explains 5% of the variance of the data ","metadata":{}},{"cell_type":"code","source":"pca.explained_variance_ratio_[1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:07.885232Z","iopub.execute_input":"2026-02-20T10:16:07.885673Z","iopub.status.idle":"2026-02-20T10:16:07.908289Z","shell.execute_reply.started":"2026-02-20T10:16:07.885635Z","shell.execute_reply":"2026-02-20T10:16:07.906522Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"the second principle component explains 4% of the variance of the data \n\nvariance is directly proportional to the spread of data","metadata":{}},{"cell_type":"code","source":"# first 2 principle components combined variance \npca.explained_variance_ratio_[0] + pca.explained_variance_ratio_[1]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:07.909473Z","iopub.execute_input":"2026-02-20T10:16:07.909839Z","iopub.status.idle":"2026-02-20T10:16:07.928204Z","shell.execute_reply.started":"2026-02-20T10:16:07.909798Z","shell.execute_reply":"2026-02-20T10:16:07.927060Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### we need to find the top principle components that explain minimum of 90% of variance of the data (spread of data)","metadata":{}},{"cell_type":"code","source":"np.cumsum(pca.explained_variance_ratio_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:07.929053Z","iopub.execute_input":"2026-02-20T10:16:07.929876Z","iopub.status.idle":"2026-02-20T10:16:07.955459Z","shell.execute_reply.started":"2026-02-20T10:16:07.929847Z","shell.execute_reply":"2026-02-20T10:16:07.954272Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## To decide the optimal number of principle components that explain a minimum of 90% variance in the data","metadata":{}},{"cell_type":"code","source":"plt.plot(np.cumsum(pca.explained_variance_ratio_))\nplt.xlabel('no_of_components')\nplt.ylabel('explained variance')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-20T10:16:07.961679Z","iopub.execute_input":"2026-02-20T10:16:07.962079Z","iopub.status.idle":"2026-02-20T10:16:08.136782Z","shell.execute_reply.started":"2026-02-20T10:16:07.962040Z","shell.execute_reply":"2026-02-20T10:16:08.135232Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"we can see that around 200 components, we get the minimum of 90% explained variance ","metadata":{}}]}