{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('<owner>/<dataset-slug>')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:49.083227Z","iopub.execute_input":"2026-07-29T14:05:49.083619Z","iopub.status.idle":"2026-07-29T14:05:50.542225Z","shell.execute_reply.started":"2026-07-29T14:05:49.083588Z","shell.execute_reply":"2026-07-29T14:05:50.541205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/competitions/digit-recognizer/train.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:50.544725Z","iopub.execute_input":"2026-07-29T14:05:50.545179Z","iopub.status.idle":"2026-07-29T14:05:53.343178Z","shell.execute_reply.started":"2026-07-29T14:05:50.545149Z","shell.execute_reply":"2026-07-29T14:05:53.342260Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:53.344278Z","iopub.execute_input":"2026-07-29T14:05:53.344587Z","iopub.status.idle":"2026-07-29T14:05:53.381051Z","shell.execute_reply.started":"2026-07-29T14:05:53.344561Z","shell.execute_reply":"2026-07-29T14:05:53.380191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:53.383641Z","iopub.execute_input":"2026-07-29T14:05:53.383952Z","iopub.status.idle":"2026-07-29T14:05:53.390129Z","shell.execute_reply.started":"2026-07-29T14:05:53.383926Z","shell.execute_reply":"2026-07-29T14:05:53.389087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.imshow(df.iloc[13, 1:].values.reshape(28, 28)) # 13th row","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:53.391238Z","iopub.execute_input":"2026-07-29T14:05:53.391596Z","iopub.status.idle":"2026-07-29T14:05:53.614343Z","shell.execute_reply.started":"2026-07-29T14:05:53.391562Z","shell.execute_reply":"2026-07-29T14:05:53.613605Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Train a ML Model","metadata":{}},{"cell_type":"code","source":"X = df.iloc[:, 1:]\ny = df.iloc[:, 0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:53.615342Z","iopub.execute_input":"2026-07-29T14:05:53.615995Z","iopub.status.idle":"2026-07-29T14:05:53.620720Z","shell.execute_reply.started":"2026-07-29T14:05:53.615965Z","shell.execute_reply":"2026-07-29T14:05:53.619786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:53.621839Z","iopub.execute_input":"2026-07-29T14:05:53.622253Z","iopub.status.idle":"2026-07-29T14:05:54.758170Z","shell.execute_reply.started":"2026-07-29T14:05:53.622219Z","shell.execute_reply":"2026-07-29T14:05:54.757406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape, y_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:54.759296Z","iopub.execute_input":"2026-07-29T14:05:54.759721Z","iopub.status.idle":"2026-07-29T14:05:54.765836Z","shell.execute_reply.started":"2026-07-29T14:05:54.759696Z","shell.execute_reply":"2026-07-29T14:05:54.764958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:54.767015Z","iopub.execute_input":"2026-07-29T14:05:54.767410Z","iopub.status.idle":"2026-07-29T14:05:55.033095Z","shell.execute_reply.started":"2026-07-29T14:05:54.767371Z","shell.execute_reply":"2026-07-29T14:05:55.032298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:55.036518Z","iopub.execute_input":"2026-07-29T14:05:55.036969Z","iopub.status.idle":"2026-07-29T14:05:55.041460Z","shell.execute_reply.started":"2026-07-29T14:05:55.036941Z","shell.execute_reply":"2026-07-29T14:05:55.040598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn.fit(X_train, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:55.042872Z","iopub.execute_input":"2026-07-29T14:05:55.043244Z","iopub.status.idle":"2026-07-29T14:05:55.206759Z","shell.execute_reply.started":"2026-07-29T14:05:55.043207Z","shell.execute_reply":"2026-07-29T14:05:55.205849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Currently datapoints are in 784 dimension.\n# Search time will be more, lets find out\n\nimport time\nstart = time.time()\ny_pred = knn.predict(X_test)\nprint(f\"Time consumed -> {time.time() - start}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:05:55.207862Z","iopub.execute_input":"2026-07-29T14:05:55.208239Z","iopub.status.idle":"2026-07-29T14:06:06.530747Z","shell.execute_reply.started":"2026-07-29T14:05:55.208210Z","shell.execute_reply":"2026-07-29T14:06:06.529856Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_test, y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:06.531833Z","iopub.execute_input":"2026-07-29T14:06:06.532712Z","iopub.status.idle":"2026-07-29T14:06:06.540814Z","shell.execute_reply.started":"2026-07-29T14:06:06.532679Z","shell.execute_reply":"2026-07-29T14:06:06.539921Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Applying PCA","metadata":{}},{"cell_type":"code","source":"# Step 1: Mean Center the data --> Standardization\n\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:06.542030Z","iopub.execute_input":"2026-07-29T14:06:06.542406Z","iopub.status.idle":"2026-07-29T14:06:06.554412Z","shell.execute_reply.started":"2026-07-29T14:06:06.542369Z","shell.execute_reply":"2026-07-29T14:06:06.553530Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = scaler.fit_transform(X_train)\nX_test = scaler.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:06.555719Z","iopub.execute_input":"2026-07-29T14:06:06.556537Z","iopub.status.idle":"2026-07-29T14:06:07.011539Z","shell.execute_reply.started":"2026-07-29T14:06:06.556505Z","shell.execute_reply":"2026-07-29T14:06:07.010430Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Step 2: Apply PCA\n\nfrom sklearn.decomposition import PCA\npca = PCA(n_components = 200) # Selecting 200 Principle Components\n# n_components = 200 -> Number of Principle Components = Number of Feature","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:07.012591Z","iopub.execute_input":"2026-07-29T14:06:07.012845Z","iopub.status.idle":"2026-07-29T14:06:07.017637Z","shell.execute_reply.started":"2026-07-29T14:06:07.012822Z","shell.execute_reply":"2026-07-29T14:06:07.016830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:07.018698Z","iopub.execute_input":"2026-07-29T14:06:07.019050Z","iopub.status.idle":"2026-07-29T14:06:07.980531Z","shell.execute_reply.started":"2026-07-29T14:06:07.019012Z","shell.execute_reply":"2026-07-29T14:06:07.979830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dimensions reduced from 784 ---> 200\nX_train_trf.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:07.981386Z","iopub.execute_input":"2026-07-29T14:06:07.981683Z","iopub.status.idle":"2026-07-29T14:06:07.989940Z","shell.execute_reply.started":"2026-07-29T14:06:07.981653Z","shell.execute_reply":"2026-07-29T14:06:07.988702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn2 = KNeighborsClassifier()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:07.990775Z","iopub.execute_input":"2026-07-29T14:06:07.991131Z","iopub.status.idle":"2026-07-29T14:06:08.008130Z","shell.execute_reply.started":"2026-07-29T14:06:07.991097Z","shell.execute_reply":"2026-07-29T14:06:08.007084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"knn2.fit(X_train_trf, y_train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:08.011340Z","iopub.execute_input":"2026-07-29T14:06:08.012141Z","iopub.status.idle":"2026-07-29T14:06:08.040456Z","shell.execute_reply.started":"2026-07-29T14:06:08.012102Z","shell.execute_reply":"2026-07-29T14:06:08.039701Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# For 200D srarch space, it should take less time than in 784D\nstart = time.time()\ny_pred = knn2.predict(X_test_trf)\nprint(f\"Time consumed -> {time.time() - start}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:08.041340Z","iopub.execute_input":"2026-07-29T14:06:08.041644Z","iopub.status.idle":"2026-07-29T14:06:10.883817Z","shell.execute_reply.started":"2026-07-29T14:06:08.041612Z","shell.execute_reply":"2026-07-29T14:06:10.882899Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"accuracy_score(y_test, y_pred)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:10.885166Z","iopub.execute_input":"2026-07-29T14:06:10.885652Z","iopub.status.idle":"2026-07-29T14:06:10.893593Z","shell.execute_reply.started":"2026-07-29T14:06:10.885623Z","shell.execute_reply":"2026-07-29T14:06:10.892857Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Finetune on Number of Components","metadata":{}},{"cell_type":"code","source":"for i in range(1, 784, 50):\n    pca = PCA(n_components = i)\n    X_train_trf = pca.fit_transform(X_train)\n    X_test_trf = pca.transform(X_test)\n\n    knn = KNeighborsClassifier()\n    knn.fit(X_train_trf, y_train)\n\n    start = time.time()\n    y_pred = knn.predict(X_test_trf)\n    print(f\"Components -> {i} | Accuracy -> {accuracy_score(y_pred, y_test):.3f} | ExTime -> {(time.time()-start):.2f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:06:10.894725Z","iopub.execute_input":"2026-07-29T14:06:10.895089Z","iopub.status.idle":"2026-07-29T14:07:45.256897Z","shell.execute_reply.started":"2026-07-29T14:06:10.895063Z","shell.execute_reply":"2026-07-29T14:07:45.256024Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transforming to 2D Coordinate plane\n\npca = PCA(n_components = 2)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:45.257920Z","iopub.execute_input":"2026-07-29T14:07:45.258186Z","iopub.status.idle":"2026-07-29T14:07:45.939062Z","shell.execute_reply.started":"2026-07-29T14:07:45.258161Z","shell.execute_reply":"2026-07-29T14:07:45.938403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:45.939992Z","iopub.execute_input":"2026-07-29T14:07:45.940281Z","iopub.status.idle":"2026-07-29T14:07:45.950139Z","shell.execute_reply.started":"2026-07-29T14:07:45.940249Z","shell.execute_reply":"2026-07-29T14:07:45.949349Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter(x = X_train_trf[:, 0],\n                y = X_train_trf[:, 1],\n                color = y_train_trf,\n                color_discrete_sequence = px.colors.qualitative.G10\n                )\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:45.952582Z","iopub.execute_input":"2026-07-29T14:07:45.954436Z","iopub.status.idle":"2026-07-29T14:07:51.135137Z","shell.execute_reply.started":"2026-07-29T14:07:45.954373Z","shell.execute_reply":"2026-07-29T14:07:51.134151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# transforming in 3D\npca = PCA(n_components=3)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:51.136500Z","iopub.execute_input":"2026-07-29T14:07:51.136947Z","iopub.status.idle":"2026-07-29T14:07:51.830105Z","shell.execute_reply.started":"2026-07-29T14:07:51.136906Z","shell.execute_reply":"2026-07-29T14:07:51.829214Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train_trf","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:51.836930Z","iopub.execute_input":"2026-07-29T14:07:51.838439Z","iopub.status.idle":"2026-07-29T14:07:51.845121Z","shell.execute_reply.started":"2026-07-29T14:07:51.838396Z","shell.execute_reply":"2026-07-29T14:07:51.844484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x=X_train_trf[:,0], y=X_train_trf[:,1], z=X_train_trf[:,2],\n              color=y_train_trf)\nfig.update_layout(\n    margin=dict(l=20, r=20, t=20, b=20),\n    paper_bgcolor=\"LightSteelBlue\",\n)\nfig.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:51.845942Z","iopub.execute_input":"2026-07-29T14:07:51.846216Z","iopub.status.idle":"2026-07-29T14:07:52.043639Z","shell.execute_reply.started":"2026-07-29T14:07:51.846185Z","shell.execute_reply":"2026-07-29T14:07:52.042143Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Eigen Values\n# It shows contribution in explaining the variance of original data\n\npca.explained_variance_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:52.044841Z","iopub.execute_input":"2026-07-29T14:07:52.045113Z","iopub.status.idle":"2026-07-29T14:07:52.051074Z","shell.execute_reply.started":"2026-07-29T14:07:52.045088Z","shell.execute_reply":"2026-07-29T14:07:52.050175Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Eigen vectors\npca.components_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:52.052237Z","iopub.execute_input":"2026-07-29T14:07:52.052655Z","iopub.status.idle":"2026-07-29T14:07:52.069628Z","shell.execute_reply.started":"2026-07-29T14:07:52.052611Z","shell.execute_reply":"2026-07-29T14:07:52.068775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:52.070740Z","iopub.execute_input":"2026-07-29T14:07:52.071162Z","iopub.status.idle":"2026-07-29T14:07:52.088785Z","shell.execute_reply.started":"2026-07-29T14:07:52.071133Z","shell.execute_reply":"2026-07-29T14:07:52.088010Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca = PCA(n_components = None)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:52.089878Z","iopub.execute_input":"2026-07-29T14:07:52.090231Z","iopub.status.idle":"2026-07-29T14:07:53.712124Z","shell.execute_reply.started":"2026-07-29T14:07:52.090201Z","shell.execute_reply":"2026-07-29T14:07:53.711195Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# contribution of each Eigen Value explaining the variance of original data\npca.explained_variance_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:53.713199Z","iopub.execute_input":"2026-07-29T14:07:53.713894Z","iopub.status.idle":"2026-07-29T14:07:53.719845Z","shell.execute_reply.started":"2026-07-29T14:07:53.713858Z","shell.execute_reply":"2026-07-29T14:07:53.719031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pca.components_.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:53.720714Z","iopub.execute_input":"2026-07-29T14:07:53.721047Z","iopub.status.idle":"2026-07-29T14:07:53.745842Z","shell.execute_reply.started":"2026-07-29T14:07:53.721014Z","shell.execute_reply":"2026-07-29T14:07:53.745123Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Ratio/Percent Explained variance by each Eigen value\npca.explained_variance_ratio_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:53.748772Z","iopub.execute_input":"2026-07-29T14:07:53.750401Z","iopub.status.idle":"2026-07-29T14:07:53.773280Z","shell.execute_reply.started":"2026-07-29T14:07:53.750358Z","shell.execute_reply":"2026-07-29T14:07:53.772511Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Cumulative sum percent of %_explained_variance starting from Lambda_1\nnp.cumsum(pca.explained_variance_ratio_*100)\n\n# The goal is to find where we get atleast 90% of cumulative sum percent\n# Till there Principle components will be used (here lambda_1 to lambda_222)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:20:15.617858Z","iopub.execute_input":"2026-07-29T14:20:15.618186Z","iopub.status.idle":"2026-07-29T14:20:15.631599Z","shell.execute_reply.started":"2026-07-29T14:20:15.618159Z","shell.execute_reply":"2026-07-29T14:20:15.630556Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.plot(np.cumsum(pca.explained_variance_ratio_))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-07-29T14:07:53.791661Z","iopub.execute_input":"2026-07-29T14:07:53.792276Z","iopub.status.idle":"2026-07-29T14:07:53.932033Z","shell.execute_reply.started":"2026-07-29T14:07:53.792230Z","shell.execute_reply":"2026-07-29T14:07:53.931234Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}