{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-03-30T13:46:41.034632Z","iopub.execute_input":"2023-03-30T13:46:41.035016Z","iopub.status.idle":"2023-03-30T13:46:41.045153Z","shell.execute_reply.started":"2023-03-30T13:46:41.034981Z","shell.execute_reply":"2023-03-30T13:46:41.043862Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# What is PCA (Principle Component Analysis)?\n## PCA is a technique which can transform higher dimentional data to lower dimential data while keep the essence of the data.\n# benifits\n## *) Faster exicution of algo.\n## *) We can Visulize higher dimentional data by using PCA to connvert 2D and 3D.","metadata":{}},{"cell_type":"markdown","source":"# 1) working with digital recognition train data set.\n# 2) use KNN algo.\n# 3) create ML Model without using PCA.\n# 4) create ML Model with using PCA.\n# 5) 2D and 3D visualization.\n# 6) find Eigen vectors and Eigen Vlaues.\n# 7) finding optimum number of principle components.","metadata":{}},{"cell_type":"code","source":"# import training data set\ndf=pd.read_csv('/kaggle/input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:18.917844Z","iopub.execute_input":"2023-03-30T14:20:18.918267Z","iopub.status.idle":"2023-03-30T14:20:21.084113Z","shell.execute_reply.started":"2023-03-30T14:20:18.918230Z","shell.execute_reply":"2023-03-30T14:20:21.082887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:21.833049Z","iopub.execute_input":"2023-03-30T14:20:21.833508Z","iopub.status.idle":"2023-03-30T14:20:21.841114Z","shell.execute_reply.started":"2023-03-30T14:20:21.833467Z","shell.execute_reply":"2023-03-30T14:20:21.839817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:22.394808Z","iopub.execute_input":"2023-03-30T14:20:22.395811Z","iopub.status.idle":"2023-03-30T14:20:22.413117Z","shell.execute_reply.started":"2023-03-30T14:20:22.395765Z","shell.execute_reply":"2023-03-30T14:20:22.411900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:22.421346Z","iopub.execute_input":"2023-03-30T14:20:22.421721Z","iopub.status.idle":"2023-03-30T14:20:22.427071Z","shell.execute_reply.started":"2023-03-30T14:20:22.421689Z","shell.execute_reply":"2023-03-30T14:20:22.425778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(df.iloc[7506,1:].values.reshape(28,28))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:23.302895Z","iopub.execute_input":"2023-03-30T14:20:23.303677Z","iopub.status.idle":"2023-03-30T14:20:23.504688Z","shell.execute_reply.started":"2023-03-30T14:20:23.303635Z","shell.execute_reply":"2023-03-30T14:20:23.503476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  without PCA","metadata":{}},{"cell_type":"code","source":"x = df.iloc[:,1:]\ny = df.iloc[:,0]","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:27.240745Z","iopub.execute_input":"2023-03-30T14:20:27.241167Z","iopub.status.idle":"2023-03-30T14:20:27.246217Z","shell.execute_reply.started":"2023-03-30T14:20:27.241128Z","shell.execute_reply":"2023-03-30T14:20:27.245404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:27.737323Z","iopub.execute_input":"2023-03-30T14:20:27.737961Z","iopub.status.idle":"2023-03-30T14:20:27.742427Z","shell.execute_reply.started":"2023-03-30T14:20:27.737921Z","shell.execute_reply":"2023-03-30T14:20:27.741343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train,x_test,y_train,y_test = train_test_split(x,y,test_size=.2,random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:27.822989Z","iopub.execute_input":"2023-03-30T14:20:27.823432Z","iopub.status.idle":"2023-03-30T14:20:28.048788Z","shell.execute_reply.started":"2023-03-30T14:20:27.823392Z","shell.execute_reply":"2023-03-30T14:20:28.047608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train.shape,x_test.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:33.211633Z","iopub.execute_input":"2023-03-30T14:20:33.212062Z","iopub.status.idle":"2023-03-30T14:20:33.220855Z","shell.execute_reply.started":"2023-03-30T14:20:33.212019Z","shell.execute_reply":"2023-03-30T14:20:33.219459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use knn algo\nfrom sklearn.neighbors import KNeighborsClassifier","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:34.096840Z","iopub.execute_input":"2023-03-30T14:20:34.097665Z","iopub.status.idle":"2023-03-30T14:20:34.103318Z","shell.execute_reply.started":"2023-03-30T14:20:34.097616Z","shell.execute_reply":"2023-03-30T14:20:34.101957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:34.891195Z","iopub.execute_input":"2023-03-30T14:20:34.892263Z","iopub.status.idle":"2023-03-30T14:20:34.896555Z","shell.execute_reply.started":"2023-03-30T14:20:34.892218Z","shell.execute_reply":"2023-03-30T14:20:34.895746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn.fit(x_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:35.309344Z","iopub.execute_input":"2023-03-30T14:20:35.309953Z","iopub.status.idle":"2023-03-30T14:20:35.329436Z","shell.execute_reply.started":"2023-03-30T14:20:35.309917Z","shell.execute_reply":"2023-03-30T14:20:35.328174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nstart = time.time()\ny_pred = knn.predict(x_test)\nprint(time.time() - start)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:35.773265Z","iopub.execute_input":"2023-03-30T14:20:35.773651Z","iopub.status.idle":"2023-03-30T14:20:48.514062Z","shell.execute_reply.started":"2023-03-30T14:20:35.773616Z","shell.execute_reply":"2023-03-30T14:20:48.511782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom sklearn.metrics import accuracy_score\naccuracy_score(y_test,y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:48.515871Z","iopub.execute_input":"2023-03-30T14:20:48.516243Z","iopub.status.idle":"2023-03-30T14:20:48.524982Z","shell.execute_reply.started":"2023-03-30T14:20:48.516207Z","shell.execute_reply":"2023-03-30T14:20:48.523769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  With PCA ","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:20:55.902299Z","iopub.execute_input":"2023-03-30T14:20:55.903027Z","iopub.status.idle":"2023-03-30T14:20:55.907076Z","shell.execute_reply.started":"2023-03-30T14:20:55.902985Z","shell.execute_reply":"2023-03-30T14:20:55.906116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:21:05.103073Z","iopub.execute_input":"2023-03-30T14:21:05.104269Z","iopub.status.idle":"2023-03-30T14:21:05.109691Z","shell.execute_reply.started":"2023-03-30T14:21:05.104208Z","shell.execute_reply":"2023-03-30T14:21:05.108734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = scaler.fit_transform(x_train)\nx_test =  scaler.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:21:05.112027Z","iopub.execute_input":"2023-03-30T14:21:05.112472Z","iopub.status.idle":"2023-03-30T14:21:05.610602Z","shell.execute_reply.started":"2023-03-30T14:21:05.112427Z","shell.execute_reply":"2023-03-30T14:21:05.609432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\npca = PCA(n_components = 100)\n","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:22:42.071554Z","iopub.execute_input":"2023-03-30T14:22:42.071991Z","iopub.status.idle":"2023-03-30T14:22:42.077467Z","shell.execute_reply.started":"2023-03-30T14:22:42.071945Z","shell.execute_reply":"2023-03-30T14:22:42.076166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_trf = pca.fit_transform(x_train)\nx_test_trf = pca.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:22:42.702344Z","iopub.execute_input":"2023-03-30T14:22:42.702774Z","iopub.status.idle":"2023-03-30T14:22:45.223347Z","shell.execute_reply.started":"2023-03-30T14:22:42.702737Z","shell.execute_reply":"2023-03-30T14:22:45.221440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_trf.shape,x_train.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:22:45.226195Z","iopub.execute_input":"2023-03-30T14:22:45.227230Z","iopub.status.idle":"2023-03-30T14:22:45.239651Z","shell.execute_reply.started":"2023-03-30T14:22:45.227162Z","shell.execute_reply":"2023-03-30T14:22:45.237780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"knn = KNeighborsClassifier()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:22:48.616193Z","iopub.execute_input":"2023-03-30T14:22:48.616947Z","iopub.status.idle":"2023-03-30T14:22:48.621535Z","shell.execute_reply.started":"2023-03-30T14:22:48.616905Z","shell.execute_reply":"2023-03-30T14:22:48.620376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use KNN ALGO\nknn.fit(x_train_trf,y_train)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:23:12.492972Z","iopub.execute_input":"2023-03-30T14:23:12.493437Z","iopub.status.idle":"2023-03-30T14:23:12.507979Z","shell.execute_reply.started":"2023-03-30T14:23:12.493398Z","shell.execute_reply":"2023-03-30T14:23:12.506836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = knn.predict(x_test_trf)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:23:12.864293Z","iopub.execute_input":"2023-03-30T14:23:12.864913Z","iopub.status.idle":"2023-03-30T14:23:19.360391Z","shell.execute_reply.started":"2023-03-30T14:23:12.864876Z","shell.execute_reply":"2023-03-30T14:23:19.359340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score(y_test,y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:23:19.362385Z","iopub.execute_input":"2023-03-30T14:23:19.362721Z","iopub.status.idle":"2023-03-30T14:23:19.371482Z","shell.execute_reply.started":"2023-03-30T14:23:19.362689Z","shell.execute_reply":"2023-03-30T14:23:19.370374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for i in range(1,785):\n#     pca = PCA(n_components = i)\n#     x_train_trf = pca.fit_transform(x_train)\n#     x_test_trf = pca.transform(x_test)\n#     knn = KNeighborsClassifier()\n#     knn.fit(x_train_trf,y_train)\n#     y_pred = knn.predict(x_test_trf)\n#     print(accuracy_score(y_test,y_pred))\n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#transformaing to a 2D coordinate system\nx_train_trf = pca.fit_transform(x_train)\nx_test_trf = pca.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:37:19.215944Z","iopub.execute_input":"2023-03-30T14:37:19.216428Z","iopub.status.idle":"2023-03-30T14:37:21.426407Z","shell.execute_reply.started":"2023-03-30T14:37:19.216382Z","shell.execute_reply":"2023-03-30T14:37:21.424585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_trf","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:37:23.389175Z","iopub.execute_input":"2023-03-30T14:37:23.390340Z","iopub.status.idle":"2023-03-30T14:37:23.397624Z","shell.execute_reply.started":"2023-03-30T14:37:23.390288Z","shell.execute_reply":"2023-03-30T14:37:23.396710Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 2D visulization\nimport plotly.express as px\ny_train_trf = y_train.astype(str)\nfig = px.scatter(x = x_train_trf[:,0],\n                 y = x_train_trf[:,1],\n                 color = y_train_trf,\n                 color_discrete_sequence = px.colors.qualitative.G10)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T14:39:04.759510Z","iopub.execute_input":"2023-03-30T14:39:04.760584Z","iopub.status.idle":"2023-03-30T14:39:06.243102Z","shell.execute_reply.started":"2023-03-30T14:39:04.760535Z","shell.execute_reply":"2023-03-30T14:39:06.241533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transforming 3D\npca = PCA(n_components = 3)\nx_train_trf = pca.fit_transform(x_train)\nx_test_trf =pca.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:10:52.041824Z","iopub.execute_input":"2023-03-30T15:10:52.042283Z","iopub.status.idle":"2023-03-30T15:10:53.663904Z","shell.execute_reply.started":"2023-03-30T15:10:52.042241Z","shell.execute_reply":"2023-03-30T15:10:53.662209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train_trf","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:11:11.465326Z","iopub.execute_input":"2023-03-30T15:11:11.465777Z","iopub.status.idle":"2023-03-30T15:11:11.474126Z","shell.execute_reply.started":"2023-03-30T15:11:11.465737Z","shell.execute_reply":"2023-03-30T15:11:11.472612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 3D visulaization\ny_train_trf = y_train.astype(str)\nfig = px.scatter_3d(df, x = x_train_trf[:,0], y=x_train_trf[:,1],z=x_train_trf[:,2],\n                   color = y_train_trf)\nfig.update_layout(\nmargin = dict(l = 20,t=20,b=20))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:16:30.196018Z","iopub.execute_input":"2023-03-30T15:16:30.197091Z","iopub.status.idle":"2023-03-30T15:16:30.402468Z","shell.execute_reply.started":"2023-03-30T15:16:30.197030Z","shell.execute_reply":"2023-03-30T15:16:30.401130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Eigen values\npca.explained_variance_","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:18:10.394715Z","iopub.execute_input":"2023-03-30T15:18:10.395172Z","iopub.status.idle":"2023-03-30T15:18:10.403536Z","shell.execute_reply.started":"2023-03-30T15:18:10.395134Z","shell.execute_reply":"2023-03-30T15:18:10.402220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Eigen Vectors\npca.components_.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:19:38.170886Z","iopub.execute_input":"2023-03-30T15:19:38.172267Z","iopub.status.idle":"2023-03-30T15:19:38.179533Z","shell.execute_reply.started":"2023-03-30T15:19:38.172215Z","shell.execute_reply":"2023-03-30T15:19:38.178447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# finding optimum number of principle components\npca = PCA(n_components = None)\nx_train_trf = pca.fit_transform(x_train)\nx_test_trf =pca.transform(x_test)","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:32:20.749134Z","iopub.execute_input":"2023-03-30T15:32:20.749511Z","iopub.status.idle":"2023-03-30T15:32:25.177847Z","shell.execute_reply.started":"2023-03-30T15:32:20.749478Z","shell.execute_reply":"2023-03-30T15:32:25.176193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca.explained_variance_","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:32:50.113182Z","iopub.execute_input":"2023-03-30T15:32:50.113605Z","iopub.status.idle":"2023-03-30T15:32:50.130464Z","shell.execute_reply.started":"2023-03-30T15:32:50.113565Z","shell.execute_reply":"2023-03-30T15:32:50.129127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca.components_.shape","metadata":{"execution":{"iopub.status.busy":"2023-03-30T15:34:22.616863Z","iopub.execute_input":"2023-03-30T15:34:22.618064Z","iopub.status.idle":"2023-03-30T15:34:22.625858Z","shell.execute_reply.started":"2023-03-30T15:34:22.618006Z","shell.execute_reply":"2023-03-30T15:34:22.624605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plt.plot(np.cumsum(pca.explained_variance_ratio_))\n# plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}