{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **IMPORTING LIBRARY**","metadata":{}},{"cell_type":"code","source":"#importing library\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:24:05.559446Z","iopub.execute_input":"2022-05-29T06:24:05.559879Z","iopub.status.idle":"2022-05-29T06:24:05.567917Z","shell.execute_reply.started":"2022-05-29T06:24:05.559844Z","shell.execute_reply":"2022-05-29T06:24:05.566655Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **READING THE DATASET**","metadata":{}},{"cell_type":"code","source":"#redaing the dataset\n\ndf = pd.read_csv('../input/digit-recognizer/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:24:27.604551Z","iopub.execute_input":"2022-05-29T06:24:27.604986Z","iopub.status.idle":"2022-05-29T06:24:31.323427Z","shell.execute_reply.started":"2022-05-29T06:24:27.604954Z","shell.execute_reply":"2022-05-29T06:24:31.322411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#first 5 records\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:24:51.327770Z","iopub.execute_input":"2022-05-29T06:24:51.328924Z","iopub.status.idle":"2022-05-29T06:24:51.353051Z","shell.execute_reply.started":"2022-05-29T06:24:51.328873Z","shell.execute_reply":"2022-05-29T06:24:51.351839Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#no. of rows and columns\n\ndf.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:24:36.664290Z","iopub.execute_input":"2022-05-29T06:24:36.665035Z","iopub.status.idle":"2022-05-29T06:24:36.670341Z","shell.execute_reply.started":"2022-05-29T06:24:36.664992Z","shell.execute_reply":"2022-05-29T06:24:36.669649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The data files train.csv contain gray-scale images of hand-drawn digits, from zero through nine.\n\nEach image is 28 pixels in height and 28 pixels in width, for a total of 784 pixels in total. Each pixel has a single pixel-value associated with it, indicating the lightness or darkness of that pixel, with higher numbers meaning darker. This pixel-value is an integer between 0 and 255, inclusive.\n\nThe training data set has 785 columns. The first column, called \"label\", is the digit that was drawn by the user. The rest of the columns contain the pixel-values of the associated image.","metadata":{}},{"cell_type":"code","source":"#splitting the features and label\n\nx = df.drop(columns=['label'])\ny = df['label']","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:26:55.859168Z","iopub.execute_input":"2022-05-29T06:26:55.860139Z","iopub.status.idle":"2022-05-29T06:26:55.978436Z","shell.execute_reply.started":"2022-05-29T06:26:55.860097Z","shell.execute_reply":"2022-05-29T06:26:55.977369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#features\n\nx","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:26:58.932825Z","iopub.execute_input":"2022-05-29T06:26:58.933726Z","iopub.status.idle":"2022-05-29T06:26:58.959338Z","shell.execute_reply.started":"2022-05-29T06:26:58.933669Z","shell.execute_reply":"2022-05-29T06:26:58.958416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:27:14.964874Z","iopub.execute_input":"2022-05-29T06:27:14.965281Z","iopub.status.idle":"2022-05-29T06:27:14.971850Z","shell.execute_reply.started":"2022-05-29T06:27:14.965246Z","shell.execute_reply":"2022-05-29T06:27:14.970726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#label\n\ny","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:27:02.832955Z","iopub.execute_input":"2022-05-29T06:27:02.833642Z","iopub.status.idle":"2022-05-29T06:27:02.841266Z","shell.execute_reply.started":"2022-05-29T06:27:02.833608Z","shell.execute_reply":"2022-05-29T06:27:02.840035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:27:18.517013Z","iopub.execute_input":"2022-05-29T06:27:18.517709Z","iopub.status.idle":"2022-05-29T06:27:18.524633Z","shell.execute_reply.started":"2022-05-29T06:27:18.517664Z","shell.execute_reply":"2022-05-29T06:27:18.523679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **FEATURE SELECTION**","metadata":{}},{"cell_type":"code","source":"# Picking only first 15K data-points to work on for time-effeciency.\n\ny =  y.head(15000)           #labels with 15k data points\nx = x.head(15000)       #data with 15k data points\nx.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:29:57.214272Z","iopub.execute_input":"2022-05-29T06:29:57.214690Z","iopub.status.idle":"2022-05-29T06:29:57.221327Z","shell.execute_reply.started":"2022-05-29T06:29:57.214655Z","shell.execute_reply":"2022-05-29T06:29:57.220462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **STANDARDIZATION**","metadata":{}},{"cell_type":"code","source":"# import standard scalar from sklearn\n\nfrom sklearn.preprocessing import StandardScaler\n","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:31:31.378406Z","iopub.execute_input":"2022-05-29T06:31:31.379069Z","iopub.status.idle":"2022-05-29T06:31:31.514273Z","shell.execute_reply.started":"2022-05-29T06:31:31.379030Z","shell.execute_reply":"2022-05-29T06:31:31.513463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#model building\n\nss = StandardScaler()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:31:31.646888Z","iopub.execute_input":"2022-05-29T06:31:31.647323Z","iopub.status.idle":"2022-05-29T06:31:31.651433Z","shell.execute_reply.started":"2022-05-29T06:31:31.647289Z","shell.execute_reply":"2022-05-29T06:31:31.650753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#fit transform data\n\nss_x = ss.fit_transform(x)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:31:31.778933Z","iopub.execute_input":"2022-05-29T06:31:31.779569Z","iopub.status.idle":"2022-05-29T06:31:32.018231Z","shell.execute_reply.started":"2022-05-29T06:31:31.779533Z","shell.execute_reply":"2022-05-29T06:31:32.017170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print shape of the standardized data\n\nss_x.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:31:45.553647Z","iopub.execute_input":"2022-05-29T06:31:45.554035Z","iopub.status.idle":"2022-05-29T06:31:45.560096Z","shell.execute_reply.started":"2022-05-29T06:31:45.554006Z","shell.execute_reply":"2022-05-29T06:31:45.559075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#final dataset\n\nsample_data = ss_x","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:34:17.190174Z","iopub.execute_input":"2022-05-29T06:34:17.190885Z","iopub.status.idle":"2022-05-29T06:34:17.194754Z","shell.execute_reply.started":"2022-05-29T06:34:17.190852Z","shell.execute_reply":"2022-05-29T06:34:17.193833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **PCA - PRINCIPAL COMPONENT ANALYSIS**","metadata":{}},{"cell_type":"markdown","source":"> Large datasets are increasingly widespread in many disciplines. In order to interpret such datasets, methods are required to drastically reduce their dimensionality in an interpretable way, such that most of the information in the data is preserved. Many techniques have been developed for this purpose, but principal component analysis (PCA) is one of the oldest and most widely used. Its idea is simple—reduce the dimensionality of a dataset, while preserving as much ‘variability’ (i.e. statistical information) as possible.","metadata":{}},{"cell_type":"markdown","source":"\n**PCA is a method that brings together:**\n1. A measure of how each variable is associated with one another- (Covariance matrix)\n\n2. The directions in which our data are dispersed- (Eigenvectors)\n\n3. The relative importance of these different directions- (Eigenvalues)\n\n4. PCA combines our predictors and allows us to drop the Eigenvectors that are relatively unimportant","metadata":{}},{"cell_type":"code","source":"# import decomposition from sklearn\n\nfrom sklearn import decomposition","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:34:25.347532Z","iopub.execute_input":"2022-05-29T06:34:25.348276Z","iopub.status.idle":"2022-05-29T06:34:25.351506Z","shell.execute_reply.started":"2022-05-29T06:34:25.348244Z","shell.execute_reply":"2022-05-29T06:34:25.350843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**NO. OF FEATURES = 2**","metadata":{}},{"cell_type":"code","source":"# initializing the pca\n\npca = decomposition.PCA()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:34:25.588053Z","iopub.execute_input":"2022-05-29T06:34:25.588660Z","iopub.status.idle":"2022-05-29T06:34:25.592065Z","shell.execute_reply.started":"2022-05-29T06:34:25.588622Z","shell.execute_reply":"2022-05-29T06:34:25.591444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#changing to 2 features (from 784)\n\npca.n_components = 2","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:34:25.663669Z","iopub.execute_input":"2022-05-29T06:34:25.664306Z","iopub.status.idle":"2022-05-29T06:34:25.668971Z","shell.execute_reply.started":"2022-05-29T06:34:25.664255Z","shell.execute_reply":"2022-05-29T06:34:25.668193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#variance in pca\n\npca.explained_variance_","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:38:46.631305Z","iopub.execute_input":"2022-05-29T06:38:46.631724Z","iopub.status.idle":"2022-05-29T06:38:46.647143Z","shell.execute_reply.started":"2022-05-29T06:38:46.631690Z","shell.execute_reply":"2022-05-29T06:38:46.646205Z"},"_kg_hide-input":false,"_kg_hide-output":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit transform sample data using pca \n\npca_data = pca.fit_transform(sample_data)\npca_data","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:42:27.512451Z","iopub.execute_input":"2022-05-29T06:42:27.513299Z","iopub.status.idle":"2022-05-29T06:42:29.526224Z","shell.execute_reply.started":"2022-05-29T06:42:27.513263Z","shell.execute_reply":"2022-05-29T06:42:29.525337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pca_reduced will contain the 2-d projects of simple data\n\npca_data.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:34:27.262059Z","iopub.execute_input":"2022-05-29T06:34:27.262417Z","iopub.status.idle":"2022-05-29T06:34:27.268052Z","shell.execute_reply.started":"2022-05-29T06:34:27.262376Z","shell.execute_reply":"2022-05-29T06:34:27.266992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**NO. OF FEATURES = 784**","metadata":{}},{"cell_type":"code","source":"# the number of components = 784\n\npca.n_components = 784","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:41:18.534485Z","iopub.execute_input":"2022-05-29T06:41:18.535074Z","iopub.status.idle":"2022-05-29T06:41:18.540385Z","shell.execute_reply.started":"2022-05-29T06:41:18.535027Z","shell.execute_reply":"2022-05-29T06:41:18.539328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit transform sample data using pca \n\npca_data = pca.fit_transform(sample_data)\npca_data","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:42:37.867399Z","iopub.execute_input":"2022-05-29T06:42:37.868260Z","iopub.status.idle":"2022-05-29T06:42:39.877083Z","shell.execute_reply.started":"2022-05-29T06:42:37.868217Z","shell.execute_reply":"2022-05-29T06:42:39.876160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#calculating percentage of variance explained in the data\n\npercentage_var_explained = pca.explained_variance_/np.sum(pca.explained_variance_)\npercentage_var_explained","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:42:03.016989Z","iopub.execute_input":"2022-05-29T06:42:03.017393Z","iopub.status.idle":"2022-05-29T06:42:03.036623Z","shell.execute_reply.started":"2022-05-29T06:42:03.017361Z","shell.execute_reply":"2022-05-29T06:42:03.035917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#cumulative sum of the percentage_var_explained\n\ncum_var_explained = np.cumsum(percentage_var_explained)\ncum_var_explained","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:42:10.660029Z","iopub.execute_input":"2022-05-29T06:42:10.660952Z","iopub.status.idle":"2022-05-29T06:42:10.677421Z","shell.execute_reply.started":"2022-05-29T06:42:10.660906Z","shell.execute_reply":"2022-05-29T06:42:10.676691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plottig the PCA\n\nplt.figure(1,figsize=(10,7))\nplt.plot(cum_var_explained,linewidth=2)\nplt.grid()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-29T06:43:55.248174Z","iopub.execute_input":"2022-05-29T06:43:55.248549Z","iopub.status.idle":"2022-05-29T06:43:55.444754Z","shell.execute_reply.started":"2022-05-29T06:43:55.248520Z","shell.execute_reply":"2022-05-29T06:43:55.443765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From above you can see that if we take 200-dimensions, approx. 90% of variance is expalined.\n\nOur intention with the princpal component analysis is to reduce the high-dimensional input to a low-dimensional input. Ultimately that low-dimensional input is intended for use in a model, since adding more components increases the cost and the accuracy.","metadata":{}},{"cell_type":"markdown","source":"**PLS UPVOTE IF YOU FOUND THE CONTENT HELPFUL :)**","metadata":{}}]}