{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-13T04:18:39.302443Z","iopub.execute_input":"2022-09-13T04:18:39.302891Z","iopub.status.idle":"2022-09-13T04:18:39.311647Z","shell.execute_reply.started":"2022-09-13T04:18:39.302855Z","shell.execute_reply":"2022-09-13T04:18:39.310774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2022-09-13T04:18:39.317087Z","iopub.execute_input":"2022-09-13T04:18:39.317834Z","iopub.status.idle":"2022-09-13T04:18:39.323153Z","shell.execute_reply.started":"2022-09-13T04:18:39.317792Z","shell.execute_reply":"2022-09-13T04:18:39.321987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')\nX = data.drop(columns='label').to_numpy()\ny = data['label'].to_numpy()","metadata":{"execution":{"iopub.status.busy":"2022-09-13T04:18:39.325285Z","iopub.execute_input":"2022-09-13T04:18:39.325965Z","iopub.status.idle":"2022-09-13T04:18:42.054951Z","shell.execute_reply.started":"2022-09-13T04:18:39.325912Z","shell.execute_reply":"2022-09-13T04:18:42.053719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)\nX_train.shape, X_test.shape, y_train.shape, y_test.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-13T04:18:42.059470Z","iopub.execute_input":"2022-09-13T04:18:42.059858Z","iopub.status.idle":"2022-09-13T04:18:42.488227Z","shell.execute_reply.started":"2022-09-13T04:18:42.059823Z","shell.execute_reply":"2022-09-13T04:18:42.486985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.imshow(X_train[0,:].reshape((28,28)))","metadata":{"execution":{"iopub.status.busy":"2022-09-13T04:18:42.489618Z","iopub.execute_input":"2022-09-13T04:18:42.490299Z","iopub.status.idle":"2022-09-13T04:18:42.686959Z","shell.execute_reply.started":"2022-09-13T04:18:42.490253Z","shell.execute_reply":"2022-09-13T04:18:42.685809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# first I'm using SVD for decomposition and calssification","metadata":{}},{"cell_type":"code","source":"digit_container = {}\nfor i in range(len(y_train)):\n    digit_container[i] = X_train[y_train == i,:]\n    digit_container[i] = digit_container[i].T","metadata":{"execution":{"iopub.status.busy":"2022-09-13T04:18:42.689187Z","iopub.execute_input":"2022-09-13T04:18:42.689608Z","iopub.status.idle":"2022-09-13T04:18:43.469267Z","shell.execute_reply.started":"2022-09-13T04:18:42.689575Z","shell.execute_reply":"2022-09-13T04:18:43.468137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"U_container = []\nfor i in range(10):\n    U, S, VT = np.linalg.svd(digit_container[i],full_matrices=False)\n    U_container.append(U)\n    plt.figure(i)\n    plt.plot(np.cumsum(S)/np.sum(S),c='red')\n    plt.title(f'Singular values U{i} : cumulative sum U{i}')\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-13T04:18:43.470574Z","iopub.execute_input":"2022-09-13T04:18:43.470933Z","iopub.status.idle":"2022-09-13T04:18:54.375780Z","shell.execute_reply.started":"2022-09-13T04:18:43.470900Z","shell.execute_reply":"2022-09-13T04:18:54.374563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score","metadata":{"execution":{"iopub.status.busy":"2022-09-13T04:18:54.377162Z","iopub.execute_input":"2022-09-13T04:18:54.377587Z","iopub.status.idle":"2022-09-13T04:18:54.382445Z","shell.execute_reply.started":"2022-09-13T04:18:54.377554Z","shell.execute_reply":"2022-09-13T04:18:54.381180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ny_pred1 = np.array([])\nfor i in range(len(X_test)):\n    dis = np.inf\n    yp = 0\n    for k in range(10):\n        d = np.linalg.norm((np.eye(28*28) - U_container[k][:,:20] @ U_container[k][:,:20].T) @ X_test[i,:])\n        if d < dis:\n            dis = d\n            yp = k\n    y_pred1 = np.concatenate((y_pred1,[yp]))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:32:44.905571Z","iopub.execute_input":"2022-09-13T05:32:44.906035Z","iopub.status.idle":"2022-09-13T05:36:59.264334Z","shell.execute_reply.started":"2022-09-13T05:32:44.905998Z","shell.execute_reply":"2022-09-13T05:36:59.262956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Now I'm using PCA for decomposition and Three diffrent classifire (andomForestClassifier, KNeighborsClassifier,SVM) for classify my data.","metadata":{}},{"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.decomposition import PCA, KernelPCA\nfrom sklearn.ensemble import RandomForestClassifier as RFC\nfrom sklearn.neighbors import KNeighborsClassifier as KNC \nfrom sklearn.svm import SVC","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:38:16.725885Z","iopub.execute_input":"2022-09-13T05:38:16.726329Z","iopub.status.idle":"2022-09-13T05:38:16.871154Z","shell.execute_reply.started":"2022-09-13T05:38:16.726295Z","shell.execute_reply":"2022-09-13T05:38:16.869857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"step2 = [\n    ('transformer',PCA(n_components=100)),\n    ('classifier',RFC(n_estimators=10))\n        ]\n\nstep3 = [\n    ('transformer',PCA(n_components=100)),\n    ('classifier',KNC(n_neighbors=5))    \n]\n\nstep4 = [\n    ('transformer',PCA(n_components=100)),\n    ('classifier',SVC())\n]\n","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:51:16.825000Z","iopub.execute_input":"2022-09-13T05:51:16.825809Z","iopub.status.idle":"2022-09-13T05:51:16.833493Z","shell.execute_reply.started":"2022-09-13T05:51:16.825737Z","shell.execute_reply":"2022-09-13T05:51:16.832324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf2 = Pipeline(steps=step2)\nclf3 = Pipeline(steps=step3)\nclf4 = Pipeline(steps=step4)\n\nclf2.fit(X_train,y_train)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:53:25.548648Z","iopub.execute_input":"2022-09-13T05:53:25.549098Z","iopub.status.idle":"2022-09-13T05:53:33.286172Z","shell.execute_reply.started":"2022-09-13T05:53:25.549058Z","shell.execute_reply":"2022-09-13T05:53:33.284801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf3.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:53:57.336552Z","iopub.execute_input":"2022-09-13T05:53:57.336984Z","iopub.status.idle":"2022-09-13T05:54:00.516517Z","shell.execute_reply.started":"2022-09-13T05:53:57.336939Z","shell.execute_reply":"2022-09-13T05:54:00.515450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf4.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:54:24.716229Z","iopub.execute_input":"2022-09-13T05:54:24.717132Z","iopub.status.idle":"2022-09-13T05:54:44.202277Z","shell.execute_reply.started":"2022-09-13T05:54:24.717082Z","shell.execute_reply":"2022-09-13T05:54:44.201079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred2 = clf2.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:56:04.140861Z","iopub.execute_input":"2022-09-13T05:56:04.141587Z","iopub.status.idle":"2022-09-13T05:56:04.257921Z","shell.execute_reply.started":"2022-09-13T05:56:04.141536Z","shell.execute_reply":"2022-09-13T05:56:04.256591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred3 = clf3.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:57:08.754233Z","iopub.execute_input":"2022-09-13T05:57:08.755038Z","iopub.status.idle":"2022-09-13T05:57:14.795265Z","shell.execute_reply.started":"2022-09-13T05:57:08.754980Z","shell.execute_reply":"2022-09-13T05:57:14.794032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred4 = clf4.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T05:57:48.838052Z","iopub.execute_input":"2022-09-13T05:57:48.838473Z","iopub.status.idle":"2022-09-13T05:57:57.840708Z","shell.execute_reply.started":"2022-09-13T05:57:48.838438Z","shell.execute_reply":"2022-09-13T05:57:57.839505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'SVD digits accuracy : {accuracy_score(y_test,y_pred1)}')\nprint(f'Random Forest Classifier accuracy : {accuracy_score(y_test,y_pred2)}')\nprint(f'KNeighbors Classifier accuracy : {accuracy_score(y_test,y_pred3)}')\nprint(f'SVM accuracy : {accuracy_score(y_test,y_pred4)}')","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:01:54.652040Z","iopub.execute_input":"2022-09-13T06:01:54.653007Z","iopub.status.idle":"2022-09-13T06:01:54.665949Z","shell.execute_reply.started":"2022-09-13T06:01:54.652945Z","shell.execute_reply":"2022-09-13T06:01:54.664727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Apparently SVM is the best classifier","metadata":{}},{"cell_type":"code","source":"data_test = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')\nXtest = data.drop(columns='label').to_numpy()\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:28:38.450911Z","iopub.execute_input":"2022-09-13T06:28:38.452087Z","iopub.status.idle":"2022-09-13T06:28:41.096780Z","shell.execute_reply.started":"2022-09-13T06:28:38.452035Z","shell.execute_reply":"2022-09-13T06:28:41.095627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ypred = clf4.predict(Xtest)","metadata":{"execution":{"iopub.status.busy":"2022-09-13T06:29:42.995241Z","iopub.execute_input":"2022-09-13T06:29:42.995676Z","iopub.status.idle":"2022-09-13T06:30:28.427249Z","shell.execute_reply.started":"2022-09-13T06:29:42.995637Z","shell.execute_reply":"2022-09-13T06:30:28.426135Z"},"trusted":true},"execution_count":null,"outputs":[]}]}