{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Applying dimension reduction on digit recognizer and fashion Mnist dataset\n* purpose of this notebook is to find wich dimension reduction method work better on Mnist datasets\n\n* I used PCA, LDA, MDS, IsoMap, TSNE and UMAP for reducing diminsion\n\n* then I used seaborn scatterplot for visualization","metadata":{}},{"cell_type":"code","source":"#separate images from labels and turning them to numpy array\ndata_digit = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nX_digit = data_digit.drop(columns=['label']).to_numpy()\ny_digit = data_digit['label'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-10-26T11:59:19.571372Z","iopub.execute_input":"2022-10-26T11:59:19.572013Z","iopub.status.idle":"2022-10-26T11:59:22.408531Z","shell.execute_reply.started":"2022-10-26T11:59:19.571962Z","shell.execute_reply":"2022-10-26T11:59:22.407387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.decomposition import PCA\nfrom sklearn.manifold import Isomap , MDS, TSNE\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\nfrom umap import UMAP","metadata":{"execution":{"iopub.status.busy":"2022-10-26T12:46:07.049375Z","iopub.execute_input":"2022-10-26T12:46:07.050352Z","iopub.status.idle":"2022-10-26T12:46:07.057046Z","shell.execute_reply.started":"2022-10-26T12:46:07.050308Z","shell.execute_reply":"2022-10-26T12:46:07.055504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#using PCA\npca = PCA(n_components=2)\nX_digit_pca = pca.fit_transform(X_digit)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_digit_pca[:,0],y=X_digit_pca[:,1],hue=y_digit,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Digit recognition dimension reduction with PCA',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T11:59:22.420205Z","iopub.execute_input":"2022-10-26T11:59:22.420724Z","iopub.status.idle":"2022-10-26T11:59:28.385237Z","shell.execute_reply.started":"2022-10-26T11:59:22.420678Z","shell.execute_reply":"2022-10-26T11:59:28.384227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PCA is super fast and efficient but couldn't separate our classes","metadata":{}},{"cell_type":"code","source":"#using IsoMap\nisomap = Isomap(n_components=2,n_neighbors=5)\nX_digit_isomap = isomap.fit_transform(X_digit[:5000])\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_digit_isomap[:,0],y=X_digit_isomap[:,1],hue=y_digit[:5000],palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Digit recognition dimension reduction with Isomap',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T11:59:28.386730Z","iopub.execute_input":"2022-10-26T11:59:28.387455Z","iopub.status.idle":"2022-10-26T11:59:51.670064Z","shell.execute_reply.started":"2022-10-26T11:59:28.387414Z","shell.execute_reply":"2022-10-26T11:59:51.666222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"IsoMap using lots of RAM so i just consider 5000 points and I assumed in whole dataset we should get similar result\nas we see IsoMap couldn't separate our classes as well and we have similar result as PCA ","metadata":{}},{"cell_type":"code","source":"mds = MDS(n_components=2)\nX_digit_mds = mds.fit_transform(X_digit[:2000])\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_digit_mds[:,0],y=X_digit_mds[:,1],hue=y_digit[:2000],palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Digit recognition dimension reduction with MDS',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T12:01:54.352976Z","iopub.execute_input":"2022-10-26T12:01:54.354398Z","iopub.status.idle":"2022-10-26T12:04:15.464540Z","shell.execute_reply.started":"2022-10-26T12:01:54.354339Z","shell.execute_reply":"2022-10-26T12:04:15.462894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MDS is using lots of RAM too so i just consider 5000 points and also I assumed in whole dataset we should get similar result and as we could see it didn't do a better job than PCA and IsoMap","metadata":{}},{"cell_type":"code","source":"tsne = TSNE(n_components=2,early_exaggeration=8)\nX_digit_tsne = tsne.fit_transform(X_digit)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_digit_tsne[:,0],y=X_digit_tsne[:,1],hue=y_digit,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Digit recognition dimension reduction with TSNE',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T12:58:45.460665Z","iopub.execute_input":"2022-10-26T12:58:45.461221Z","iopub.status.idle":"2022-10-26T13:06:04.526781Z","shell.execute_reply.started":"2022-10-26T12:58:45.461178Z","shell.execute_reply":"2022-10-26T13:06:04.525215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TSNE did a beautiful job here and separate our classes almost perfect but it was a little bit time consuming","metadata":{}},{"cell_type":"code","source":"lda = LDA(n_components=2)\nX_digit_lda = lda.fit_transform(X_digit,y_digit)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_digit_lda[:,0],y=X_digit_lda[:,1],hue=y_digit,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Digit recognition dimension reduction with Linear Discriminant Analysis',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T12:40:56.917097Z","iopub.execute_input":"2022-10-26T12:40:56.917569Z","iopub.status.idle":"2022-10-26T12:41:10.264595Z","shell.execute_reply.started":"2022-10-26T12:40:56.917535Z","shell.execute_reply":"2022-10-26T12:41:10.262659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"LDA was fast but it didn't give a better result than PCA","metadata":{}},{"cell_type":"code","source":"umap = UMAP()\nX_digit_umap = umap.fit_transform(X_digit)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_digit_umap[:,0],y=X_digit_umap[:,1],hue=y_digit,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Digit recognition dimension reduction with UMAP',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T12:47:41.348113Z","iopub.execute_input":"2022-10-26T12:47:41.352254Z","iopub.status.idle":"2022-10-26T12:48:33.530271Z","shell.execute_reply.started":"2022-10-26T12:47:41.352030Z","shell.execute_reply":"2022-10-26T12:48:33.529245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"UMAP could give us very good result and was faster than TSNE but I think TSNE give slightly better result","metadata":{}},{"cell_type":"code","source":"#separate images from labels and turning them to numpy array\ndata_fashion = pd.read_csv('/kaggle/input/fashionmnist/fashion-mnist_train.csv')\nX_fashion = data_fashion.drop(columns=['label']).to_numpy()\ny_fashion = data_fashion['label'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-10-26T13:20:47.223299Z","iopub.execute_input":"2022-10-26T13:20:47.223814Z","iopub.status.idle":"2022-10-26T13:20:50.956079Z","shell.execute_reply.started":"2022-10-26T13:20:47.223772Z","shell.execute_reply":"2022-10-26T13:20:50.954762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_fashion_pca = pca.fit_transform(X_fashion)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_fashion_pca[:,0],y=X_fashion_pca[:,1],hue=y_fashion,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Fashion Mnist recognition dimension reduction with PCA',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T13:23:05.613083Z","iopub.execute_input":"2022-10-26T13:23:05.613585Z","iopub.status.idle":"2022-10-26T13:23:13.015818Z","shell.execute_reply.started":"2022-10-26T13:23:05.613548Z","shell.execute_reply":"2022-10-26T13:23:13.014405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_fashion_isomap = isomap.fit_transform(X_fashion[:5000])\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_fashion_isomap[:,0],y=X_fashion_isomap[:,1],hue=y_fashion[:5000],palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Fashion Mnist recognition dimension reduction with Isomap',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T13:25:40.796975Z","iopub.execute_input":"2022-10-26T13:25:40.797875Z","iopub.status.idle":"2022-10-26T13:26:02.797068Z","shell.execute_reply.started":"2022-10-26T13:25:40.797827Z","shell.execute_reply":"2022-10-26T13:26:02.795187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_fashion_mds = mds.fit_transform(X_digit[:2000])\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_fashion_mds[:,0],y=X_fashion_mds[:,1],hue=y_fashion[:2000],palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Fashion Mnist recognition dimension reduction with MDS',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T13:27:25.881062Z","iopub.execute_input":"2022-10-26T13:27:25.881551Z","iopub.status.idle":"2022-10-26T13:29:44.202439Z","shell.execute_reply.started":"2022-10-26T13:27:25.881507Z","shell.execute_reply":"2022-10-26T13:29:44.201062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_fashion_lda = lda.fit_transform(X_fashion,y_fashion)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_fashion_lda[:,0],y=X_fashion_lda[:,1],hue=y_fashion,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Fashion Mnist recognition dimension reduction with Linear Discriminant Analysis',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"as we expected PCA, MDS, LDA and IsoMap give us similar result as digit recognizer","metadata":{}},{"cell_type":"code","source":"tsne = TSNE(n_components=2,early_exaggeration=8)\nX_fashion_tsne = tsne.fit_transform(X_fashion)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_fashion_tsne[:,0],y=X_fashion_tsne[:,1],hue=y_fashion,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Fashion Mnist recognition dimension reduction with TSNE',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T13:31:40.485512Z","iopub.execute_input":"2022-10-26T13:31:40.486069Z","iopub.status.idle":"2022-10-26T13:43:15.146086Z","shell.execute_reply.started":"2022-10-26T13:31:40.486027Z","shell.execute_reply":"2022-10-26T13:43:15.145083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"TSNE did better job but it seems Fashion Mnist it is harder dataset than digit recognizer in terns of classification\n","metadata":{}},{"cell_type":"code","source":"\nX_fashion_umap = umap.fit_transform(X_fashion)\nplt.figure(figsize=(12,12))\nsns.scatterplot(x=X_fashion_umap[:,0],y=X_fashion_umap[:,1],hue=y_fashion,palette='Paired')\nplt.xticks([])\nplt.yticks([])\nplt.title('Fashion Mnist dimension reduction with UMAP',fontdict = {'family':'serif','color':'darkred','size':20});","metadata":{"execution":{"iopub.status.busy":"2022-10-26T13:45:25.511076Z","iopub.execute_input":"2022-10-26T13:45:25.511638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"UMAP gives similar result as TSNE","metadata":{}}]}