{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Principal Component Analysis on MNIST Data","metadata":{}},{"cell_type":"markdown","source":"Principal component analysis (PCA) is the process of computing the principal components and using them to perform a change of basis on the data, sometimes using only the first few principal components and ignoring the rest.","metadata":{}},{"cell_type":"code","source":"# Functions to read and show images.\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n   \nd0 = pd.read_csv('../input/digit-recognizer/train.csv')\n\nprint(d0.head(5)) # print first five rows of d0.\n\n# save the labels into a variable l.\nl = d0['label']\n\n# Drop the label feature and store the pixel data in d.\nd = d0.drop(\"label\",axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:46.795353Z","iopub.execute_input":"2022-04-17T11:37:46.796183Z","iopub.status.idle":"2022-04-17T11:37:49.961777Z","shell.execute_reply.started":"2022-04-17T11:37:46.796117Z","shell.execute_reply":"2022-04-17T11:37:49.960758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(d.shape)\nprint(l.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:49.964740Z","iopub.execute_input":"2022-04-17T11:37:49.965880Z","iopub.status.idle":"2022-04-17T11:37:49.971570Z","shell.execute_reply.started":"2022-04-17T11:37:49.965807Z","shell.execute_reply":"2022-04-17T11:37:49.970656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# display or plot a number.\nplt.figure(figsize=(7,7))\nidx = 1\n\ngrid_data = d.iloc[idx].to_numpy().reshape(28,28)  # reshape from 1d to 2d pixel array\nplt.imshow(grid_data, interpolation = \"none\", cmap = \"gray\")\nplt.show()\n\nprint(l[idx])","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:49.972798Z","iopub.execute_input":"2022-04-17T11:37:49.973039Z","iopub.status.idle":"2022-04-17T11:37:50.132143Z","shell.execute_reply.started":"2022-04-17T11:37:49.973009Z","shell.execute_reply":"2022-04-17T11:37:50.131311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  2D Visualization using PCA ","metadata":{}},{"cell_type":"code","source":"#Pick first 15k data-points to work for time-efficiency\n\nlabels = l.head(15000)\ndata = d.head(15000)\n\nprint(\"The shape of sample data = \", data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:50.135004Z","iopub.execute_input":"2022-04-17T11:37:50.135747Z","iopub.status.idle":"2022-04-17T11:37:50.144101Z","shell.execute_reply.started":"2022-04-17T11:37:50.135694Z","shell.execute_reply":"2022-04-17T11:37:50.142727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Data-preprocessing: Standardizing the data\n\nfrom sklearn.preprocessing import StandardScaler\nstandardized_data  = StandardScaler().fit_transform(data)\nprint(standardized_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:50.146429Z","iopub.execute_input":"2022-04-17T11:37:50.147290Z","iopub.status.idle":"2022-04-17T11:37:50.462377Z","shell.execute_reply.started":"2022-04-17T11:37:50.147214Z","shell.execute_reply":"2022-04-17T11:37:50.461365Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Find the co-variance matrix which is : A^T * A\n\nsample_data = standardized_data\n\n#Matrix multiplication using numpy\ncovar_matrix = np.matmul(sample_data.T, sample_data)\n\nprint(\"The shape of covariance matrix = \", covar_matrix.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:50.463789Z","iopub.execute_input":"2022-04-17T11:37:50.464035Z","iopub.status.idle":"2022-04-17T11:37:50.664700Z","shell.execute_reply.started":"2022-04-17T11:37:50.464003Z","shell.execute_reply":"2022-04-17T11:37:50.663765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Finding the top twp eigen-values and corrosponding eigen-vectors\n#for projecting onto a 2-Dim space\n\nfrom scipy.linalg import eigh\n\nvalues, vectors = eigh(covar_matrix, eigvals=(782, 783))\nprint(\"Shape of eigen vectors = \", vectors.shape)\n\n#Converting the eigen vectors into (2,d) shape for easyness for further computation\nvectors = vectors.T\n\nprint(\"Updated shape of eigen vectors = \", vectors.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:50.666623Z","iopub.execute_input":"2022-04-17T11:37:50.667180Z","iopub.status.idle":"2022-04-17T11:37:50.802138Z","shell.execute_reply.started":"2022-04-17T11:37:50.667127Z","shell.execute_reply":"2022-04-17T11:37:50.801083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Projecting the original data sample on plane\n#formed by two principal eigen vectors by vector-vector multiplication. \nimport matplotlib.pyplot as plt\nnew_coordinates = np.matmul(vectors, sample_data.T)\n\nprint (\" resultant new data points' shape \", vectors.shape, \"X\", sample_data.T.shape,\" = \", new_coordinates.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:50.808686Z","iopub.execute_input":"2022-04-17T11:37:50.810052Z","iopub.status.idle":"2022-04-17T11:37:50.871096Z","shell.execute_reply.started":"2022-04-17T11:37:50.809985Z","shell.execute_reply":"2022-04-17T11:37:50.870040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\n\n#appending label to tge 2d projected data\nnew_coordinates = np.vstack((new_coordinates, labels)).T\n\n#creating a new dataFrame for ploting the labeled points. \ndataframe = pd.DataFrame(data= new_coordinates, columns=(\"1st_principal\", \"2nd_principal\", \"label\"))\nprint(dataframe.head())","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:50.877969Z","iopub.execute_input":"2022-04-17T11:37:50.882737Z","iopub.status.idle":"2022-04-17T11:37:50.912425Z","shell.execute_reply.started":"2022-04-17T11:37:50.882665Z","shell.execute_reply":"2022-04-17T11:37:50.911463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Ploting the 2d datapoints with seaborn\nimport seaborn as sns\nsns.FacetGrid(dataframe, hue='label', size= 6).map(plt.scatter, '1st_principal', '2nd_principal').add_legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:50.920291Z","iopub.execute_input":"2022-04-17T11:37:50.924370Z","iopub.status.idle":"2022-04-17T11:37:51.779177Z","shell.execute_reply.started":"2022-04-17T11:37:50.920867Z","shell.execute_reply":"2022-04-17T11:37:51.778200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PCA using Scikit-Learn","metadata":{}},{"cell_type":"code","source":"from sklearn import decomposition\npca = decomposition.PCA()","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:51.780375Z","iopub.execute_input":"2022-04-17T11:37:51.780617Z","iopub.status.idle":"2022-04-17T11:37:51.785471Z","shell.execute_reply.started":"2022-04-17T11:37:51.780586Z","shell.execute_reply":"2022-04-17T11:37:51.784603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Configuring the parameters\n#the number of components = 2\n\npca.n_components = 2\npca_data = pca.fit_transform(sample_data)\n\n#pca_reduced will contain the 2-d projects of simple data\nprint(\"Shape of PCA reduced = \", pca_data.shape)","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:51.786700Z","iopub.execute_input":"2022-04-17T11:37:51.787298Z","iopub.status.idle":"2022-04-17T11:37:53.171573Z","shell.execute_reply.started":"2022-04-17T11:37:51.787244Z","shell.execute_reply":"2022-04-17T11:37:53.170576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca_data = np.vstack((pca_data.T, labels)).T\n\n#creating a new dataFrame for ploting the labeled points. \npca_df = pd.DataFrame(data= pca_data, columns=(\"1st_principal\", \"2nd_principal\", \"label\"))\nsns.FacetGrid(pca_df, hue='label', size= 6).map(plt.scatter, '1st_principal', '2nd_principal').add_legend()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:53.173029Z","iopub.execute_input":"2022-04-17T11:37:53.173447Z","iopub.status.idle":"2022-04-17T11:37:54.039116Z","shell.execute_reply.started":"2022-04-17T11:37:53.173403Z","shell.execute_reply":"2022-04-17T11:37:54.038417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PCA for dimensionality Reduction: Variance Explained by dimensions","metadata":{}},{"cell_type":"code","source":"pca.n_components = 784\npca_data = pca.fit_transform(sample_data)\n\npercentage_var_explained = pca.explained_variance_ / np.sum(pca.explained_variance_)\n\ncum_var_explained = np.cumsum(percentage_var_explained)\n\n#Plot PCA spectrun\n\nplt.figure(1, figsize=(6,4))\nplt.clf()\nplt.plot(cum_var_explained, linewidth = 2)\nplt.axis(\"tight\")\nplt.grid()\nplt.xlabel('n_components')\nplt.ylabel('Cumulative_explained_variance')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:40:41.173455Z","iopub.execute_input":"2022-04-17T11:40:41.173966Z","iopub.status.idle":"2022-04-17T11:40:43.554489Z","shell.execute_reply.started":"2022-04-17T11:40:41.173931Z","shell.execute_reply":"2022-04-17T11:40:43.553580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#End of Notebook","metadata":{"execution":{"iopub.status.busy":"2022-04-17T11:37:54.040066Z","iopub.execute_input":"2022-04-17T11:37:54.040951Z","iopub.status.idle":"2022-04-17T11:37:54.045588Z","shell.execute_reply.started":"2022-04-17T11:37:54.040911Z","shell.execute_reply":"2022-04-17T11:37:54.044605Z"},"trusted":true},"execution_count":null,"outputs":[]}]}