{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-24T04:15:17.043256Z","iopub.execute_input":"2024-09-24T04:15:17.043693Z","iopub.status.idle":"2024-09-24T04:15:18.226535Z","shell.execute_reply.started":"2024-09-24T04:15:17.043653Z","shell.execute_reply":"2024-09-24T04:15:18.225332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" ### Dataset contains pixel values of images, 1 image has 784 pixels that means 784 features per image","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:52:36.081953Z","iopub.execute_input":"2024-09-24T04:52:36.082434Z","iopub.status.idle":"2024-09-24T04:52:39.457367Z","shell.execute_reply.started":"2024-09-24T04:52:36.082390Z","shell.execute_reply":"2024-09-24T04:52:39.455829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:17:03.977076Z","iopub.execute_input":"2024-09-24T04:17:03.977711Z","iopub.status.idle":"2024-09-24T04:17:04.016628Z","shell.execute_reply.started":"2024-09-24T04:17:03.977649Z","shell.execute_reply":"2024-09-24T04:17:04.015022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:22:07.410961Z","iopub.execute_input":"2024-09-24T04:22:07.411453Z","iopub.status.idle":"2024-09-24T04:22:07.417233Z","shell.execute_reply.started":"2024-09-24T04:22:07.411406Z","shell.execute_reply":"2024-09-24T04:22:07.415948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# image plotting\nplt.imshow(df.iloc[10010, 1:].values.reshape(28,28))","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:23:13.862973Z","iopub.execute_input":"2024-09-24T04:23:13.863489Z","iopub.status.idle":"2024-09-24T04:23:14.438220Z","shell.execute_reply.started":"2024-09-24T04:23:13.863429Z","shell.execute_reply":"2024-09-24T04:23:14.436934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df.iloc[:, 1:] # pixel values will be the training data\ny = df.iloc[:, 0] # label will be the target","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:52:45.507729Z","iopub.execute_input":"2024-09-24T04:52:45.508172Z","iopub.status.idle":"2024-09-24T04:52:45.514387Z","shell.execute_reply.started":"2024-09-24T04:52:45.508130Z","shell.execute_reply":"2024-09-24T04:52:45.513100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train test split\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 20% data will be testing data rest will be used for training","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:52:47.205623Z","iopub.execute_input":"2024-09-24T04:52:47.206113Z","iopub.status.idle":"2024-09-24T04:52:47.447859Z","shell.execute_reply.started":"2024-09-24T04:52:47.206055Z","shell.execute_reply":"2024-09-24T04:52:47.446558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:28:56.070180Z","iopub.execute_input":"2024-09-24T04:28:56.071244Z","iopub.status.idle":"2024-09-24T04:28:56.079277Z","shell.execute_reply.started":"2024-09-24T04:28:56.071196Z","shell.execute_reply":"2024-09-24T04:28:56.077660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# knn\n# finds the similar images to given image and majority similar results will be returned\nfrom sklearn.neighbors import KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:30:12.662955Z","iopub.execute_input":"2024-09-24T04:30:12.663536Z","iopub.status.idle":"2024-09-24T04:30:12.669222Z","shell.execute_reply.started":"2024-09-24T04:30:12.663455Z","shell.execute_reply":"2024-09-24T04:30:12.667769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:30:20.845581Z","iopub.execute_input":"2024-09-24T04:30:20.846067Z","iopub.status.idle":"2024-09-24T04:30:20.851311Z","shell.execute_reply.started":"2024-09-24T04:30:20.846023Z","shell.execute_reply":"2024-09-24T04:30:20.850089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:32:30.776108Z","iopub.execute_input":"2024-09-24T04:32:30.777088Z","iopub.status.idle":"2024-09-24T04:32:30.919249Z","shell.execute_reply.started":"2024-09-24T04:32:30.777033Z","shell.execute_reply":"2024-09-24T04:32:30.917952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this step will consume much more time because because we're dealing with 784 dimensional data and that's why we're using PCA to reduce computation time\nimport time\nstart = time.time()\ny_pred = knn.predict(X_test)\nprint(time.time() - start)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:34:26.973138Z","iopub.execute_input":"2024-09-24T04:34:26.973649Z","iopub.status.idle":"2024-09-24T04:34:40.159503Z","shell.execute_reply.started":"2024-09-24T04:34:26.973601Z","shell.execute_reply":"2024-09-24T04:34:40.158212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Didn't took much time (13 secs) due to nice hardware, but on local machine it will consume lot more time and 13 secs also considered as a big value in ML","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\naccuracy_score(y_test, y_pred)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:38:14.113061Z","iopub.execute_input":"2024-09-24T04:38:14.113552Z","iopub.status.idle":"2024-09-24T04:38:14.124359Z","shell.execute_reply.started":"2024-09-24T04:38:14.113509Z","shell.execute_reply":"2024-09-24T04:38:14.123221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Using PCA","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:39:22.512012Z","iopub.execute_input":"2024-09-24T04:39:22.512498Z","iopub.status.idle":"2024-09-24T04:39:22.518345Z","shell.execute_reply.started":"2024-09-24T04:39:22.512438Z","shell.execute_reply":"2024-09-24T04:39:22.517088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# scaling data: mean centering\nX_train = scaler.fit_transform(X_train) # transformer learns about the parameters in data then applies the data transformation\nX_test = scaler.transform(X_test) # applying same transformation on testing data","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:53:48.644675Z","iopub.execute_input":"2024-09-24T04:53:48.645171Z","iopub.status.idle":"2024-09-24T04:53:49.154393Z","shell.execute_reply.started":"2024-09-24T04:53:48.645123Z","shell.execute_reply":"2024-09-24T04:53:49.152543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# PCA\nfrom sklearn.decomposition import PCA\npca = PCA(n_components=200) # n_components: no. of principle components","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:59:42.367222Z","iopub.execute_input":"2024-09-24T04:59:42.368117Z","iopub.status.idle":"2024-09-24T04:59:42.373779Z","shell.execute_reply.started":"2024-09-24T04:59:42.368067Z","shell.execute_reply":"2024-09-24T04:59:42.372507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dimensionality reduction\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:59:43.631258Z","iopub.execute_input":"2024-09-24T04:59:43.632068Z","iopub.status.idle":"2024-09-24T04:59:48.714283Z","shell.execute_reply.started":"2024-09-24T04:59:43.632019Z","shell.execute_reply":"2024-09-24T04:59:48.712777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:59:50.196335Z","iopub.execute_input":"2024-09-24T04:59:50.196830Z","iopub.status.idle":"2024-09-24T04:59:50.204522Z","shell.execute_reply.started":"2024-09-24T04:59:50.196770Z","shell.execute_reply":"2024-09-24T04:59:50.203217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_trf.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:59:51.421928Z","iopub.execute_input":"2024-09-24T04:59:51.422378Z","iopub.status.idle":"2024-09-24T04:59:51.429828Z","shell.execute_reply.started":"2024-09-24T04:59:51.422339Z","shell.execute_reply":"2024-09-24T04:59:51.428691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()\nknn.fit(X_train_trf, y_train)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:59:52.370711Z","iopub.execute_input":"2024-09-24T04:59:52.371462Z","iopub.status.idle":"2024-09-24T04:59:52.405973Z","shell.execute_reply.started":"2024-09-24T04:59:52.371411Z","shell.execute_reply":"2024-09-24T04:59:52.404725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = knn.predict(X_test_trf)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:59:53.848918Z","iopub.execute_input":"2024-09-24T04:59:53.849355Z","iopub.status.idle":"2024-09-24T04:59:56.674005Z","shell.execute_reply.started":"2024-09-24T04:59:53.849316Z","shell.execute_reply":"2024-09-24T04:59:56.672387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_test, y_pred)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T04:59:58.072253Z","iopub.execute_input":"2024-09-24T04:59:58.073044Z","iopub.status.idle":"2024-09-24T04:59:58.082655Z","shell.execute_reply.started":"2024-09-24T04:59:58.072995Z","shell.execute_reply":"2024-09-24T04:59:58.081456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"best_score = 0.0\nindex = -1\nfor i in range(1, 785):\n    pca = PCA(n_components=i)\n    \n    # dimensionality reduction\n    X_train_trf = pca.fit_transform(X_train)\n    X_test_trf = pca.transform(X_test)\n    \n    # train model\n    knn = KNeighborsClassifier()\n    knn.fit(X_train_trf, y_train)\n    \n    y_pred = knn.predict(X_test_trf)\n    score = accuracy_score(y_test, y_pred)\n    print(i, \" \", score)\n    if(score > best_score):\n        best_score = score\n        index = i","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:07:26.724002Z","iopub.execute_input":"2024-09-24T05:07:26.724438Z","iopub.status.idle":"2024-09-24T05:22:01.041292Z","shell.execute_reply.started":"2024-09-24T05:07:26.724399Z","shell.execute_reply":"2024-09-24T05:22:01.038481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(index)\nprint(best_score)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:22:07.822775Z","iopub.execute_input":"2024-09-24T05:22:07.823283Z","iopub.status.idle":"2024-09-24T05:22:07.830648Z","shell.execute_reply.started":"2024-09-24T05:22:07.823234Z","shell.execute_reply":"2024-09-24T05:22:07.829248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## PCA used for Visualizations","metadata":{}},{"cell_type":"code","source":"# transoforming to 2d system, only 2 principle components\npca = PCA(n_components=2)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:22:10.058339Z","iopub.execute_input":"2024-09-24T05:22:10.059417Z","iopub.status.idle":"2024-09-24T05:22:11.609050Z","shell.execute_reply.started":"2024-09-24T05:22:10.059365Z","shell.execute_reply":"2024-09-24T05:22:11.607255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_trf","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:22:13.679887Z","iopub.execute_input":"2024-09-24T05:22:13.681292Z","iopub.status.idle":"2024-09-24T05:22:13.691304Z","shell.execute_reply.started":"2024-09-24T05:22:13.681235Z","shell.execute_reply":"2024-09-24T05:22:13.689934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter(x=X_train_trf[:,0], y=X_train_trf[:, 1], color=y_train_trf, color_discrete_sequence = px.colors.qualitative.G10)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:22:14.965825Z","iopub.execute_input":"2024-09-24T05:22:14.966366Z","iopub.status.idle":"2024-09-24T05:22:17.824694Z","shell.execute_reply.started":"2024-09-24T05:22:14.966320Z","shell.execute_reply":"2024-09-24T05:22:17.823200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transoforming to 3d system, only 3 principle components\npca = PCA(n_components=3)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:23:55.201026Z","iopub.execute_input":"2024-09-24T05:23:55.201491Z","iopub.status.idle":"2024-09-24T05:23:56.851356Z","shell.execute_reply.started":"2024-09-24T05:23:55.201435Z","shell.execute_reply":"2024-09-24T05:23:56.849843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x=X_train_trf[:, 0], y=X_train_trf[:, 1], z=X_train_trf[:, 2], color=y_train_trf)\n\nfig.update_layout(\nmargin=dict(l=20, r=20, t=20, b=20)\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:23:58.507889Z","iopub.execute_input":"2024-09-24T05:23:58.508708Z","iopub.status.idle":"2024-09-24T05:23:58.742294Z","shell.execute_reply.started":"2024-09-24T05:23:58.508661Z","shell.execute_reply":"2024-09-24T05:23:58.740516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# explained variance: Eigen Values\npca.explained_variance_","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:29:07.020445Z","iopub.execute_input":"2024-09-24T05:29:07.021528Z","iopub.status.idle":"2024-09-24T05:29:07.030824Z","shell.execute_reply.started":"2024-09-24T05:29:07.021476Z","shell.execute_reply":"2024-09-24T05:29:07.029406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Eigen Vectors - 3 vectors of 784-D\n\n# pca.components_\npca.components_.shape","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:30:15.254315Z","iopub.execute_input":"2024-09-24T05:30:15.254735Z","iopub.status.idle":"2024-09-24T05:30:15.261883Z","shell.execute_reply.started":"2024-09-24T05:30:15.254692Z","shell.execute_reply":"2024-09-24T05:30:15.260680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca.explained_variance_ratio_","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:41:30.051386Z","iopub.execute_input":"2024-09-24T05:41:30.051960Z","iopub.status.idle":"2024-09-24T05:41:30.060825Z","shell.execute_reply.started":"2024-09-24T05:41:30.051913Z","shell.execute_reply":"2024-09-24T05:41:30.059579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# checking variance sum\npca = PCA(n_components=None)\nX_train_trf = pca.fit_transform(X_train)\nX_test_trf = pca.transform(X_test)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:44:05.701663Z","iopub.execute_input":"2024-09-24T05:44:05.702211Z","iopub.status.idle":"2024-09-24T05:44:12.937327Z","shell.execute_reply.started":"2024-09-24T05:44:05.702155Z","shell.execute_reply":"2024-09-24T05:44:12.935952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.cumsum(pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:44:41.170219Z","iopub.execute_input":"2024-09-24T05:44:41.170715Z","iopub.status.idle":"2024-09-24T05:44:41.190323Z","shell.execute_reply.started":"2024-09-24T05:44:41.170671Z","shell.execute_reply":"2024-09-24T05:44:41.188684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(np.cumsum(pca.explained_variance_ratio_))","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:56:44.140902Z","iopub.execute_input":"2024-09-24T05:56:44.141375Z","iopub.status.idle":"2024-09-24T05:56:44.373206Z","shell.execute_reply.started":"2024-09-24T05:56:44.141331Z","shell.execute_reply":"2024-09-24T05:56:44.371953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}