{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Table of Contents

\n
","metadata":{"toc":true}},{"cell_type":"markdown","source":"# Libraries import","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport os\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.preprocessing import StandardScaler\nimport math\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn import preprocessing\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import roc_curve\nfrom sklearn import svm\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.dummy import DummyClassifier\nfrom sklearn.model_selection import train_test_split\nfrom matplotlib import pyplot\nimport numpy as np\nfrom sklearn.metrics import f1_score\nfrom datetime import datetime\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:15.287135Z","iopub.execute_input":"2021-11-08T02:41:15.287457Z","iopub.status.idle":"2021-11-08T02:41:15.304281Z","shell.execute_reply.started":"2021-11-08T02:41:15.287424Z","shell.execute_reply":"2021-11-08T02:41:15.303134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data import","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"/kaggle/input/digital-recognizer/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:15.306574Z","iopub.execute_input":"2021-11-08T02:41:15.307079Z","iopub.status.idle":"2021-11-08T02:41:19.320582Z","shell.execute_reply.started":"2021-11-08T02:41:15.307005Z","shell.execute_reply":"2021-11-08T02:41:19.319442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import os\n#os.listdir(\"/kaggle/input/digital-recognizer/train.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:19.321871Z","iopub.execute_input":"2021-11-08T02:41:19.322114Z","iopub.status.idle":"2021-11-08T02:41:19.326380Z","shell.execute_reply.started":"2021-11-08T02:41:19.322083Z","shell.execute_reply":"2021-11-08T02:41:19.325344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#os. getcwd()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:19.327979Z","iopub.execute_input":"2021-11-08T02:41:19.328412Z","iopub.status.idle":"2021-11-08T02:41:19.342881Z","shell.execute_reply.started":"2021-11-08T02:41:19.328363Z","shell.execute_reply":"2021-11-08T02:41:19.341700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:19.346211Z","iopub.execute_input":"2021-11-08T02:41:19.346814Z","iopub.status.idle":"2021-11-08T02:41:19.392635Z","shell.execute_reply.started":"2021-11-08T02:41:19.346764Z","shell.execute_reply":"2021-11-08T02:41:19.391469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"/kaggle/input/digital-recognizer/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:19.394112Z","iopub.execute_input":"2021-11-08T02:41:19.394945Z","iopub.status.idle":"2021-11-08T02:41:19.746933Z","shell.execute_reply.started":"2021-11-08T02:41:19.394898Z","shell.execute_reply":"2021-11-08T02:41:19.746232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:19.748175Z","iopub.execute_input":"2021-11-08T02:41:19.748536Z","iopub.status.idle":"2021-11-08T02:41:19.771056Z","shell.execute_reply.started":"2021-11-08T02:41:19.748506Z","shell.execute_reply":"2021-11-08T02:41:19.770085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dig = pd.read_csv(\"/kaggle/input/digital-recognizer/Dig-MNIST.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:19.772352Z","iopub.execute_input":"2021-11-08T02:41:19.772729Z","iopub.status.idle":"2021-11-08T02:41:20.454227Z","shell.execute_reply.started":"2021-11-08T02:41:19.772699Z","shell.execute_reply":"2021-11-08T02:41:20.453314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_dig","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:20.455807Z","iopub.execute_input":"2021-11-08T02:41:20.456109Z","iopub.status.idle":"2021-11-08T02:41:20.479360Z","shell.execute_reply.started":"2021-11-08T02:41:20.456074Z","shell.execute_reply":"2021-11-08T02:41:20.478430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os. getcwd()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:20.480808Z","iopub.execute_input":"2021-11-08T02:41:20.481215Z","iopub.status.idle":"2021-11-08T02:41:20.486455Z","shell.execute_reply.started":"2021-11-08T02:41:20.481164Z","shell.execute_reply":"2021-11-08T02:41:20.485865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df = pd.concat([df_train, df_test], axis=0,sort=False)\ndf = df_train.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:20.487720Z","iopub.execute_input":"2021-11-08T02:41:20.488140Z","iopub.status.idle":"2021-11-08T02:41:20.709058Z","shell.execute_reply.started":"2021-11-08T02:41:20.488095Z","shell.execute_reply":"2021-11-08T02:41:20.708061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:20.710880Z","iopub.execute_input":"2021-11-08T02:41:20.711203Z","iopub.status.idle":"2021-11-08T02:41:20.718928Z","shell.execute_reply.started":"2021-11-08T02:41:20.711165Z","shell.execute_reply":"2021-11-08T02:41:20.717781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:20.720431Z","iopub.execute_input":"2021-11-08T02:41:20.721263Z","iopub.status.idle":"2021-11-08T02:41:23.370302Z","shell.execute_reply.started":"2021-11-08T02:41:20.721204Z","shell.execute_reply":"2021-11-08T02:41:23.369494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.373931Z","iopub.execute_input":"2021-11-08T02:41:23.374415Z","iopub.status.idle":"2021-11-08T02:41:23.426031Z","shell.execute_reply.started":"2021-11-08T02:41:23.374361Z","shell.execute_reply":"2021-11-08T02:41:23.425273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum(0).sort_values(ascending=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.427610Z","iopub.execute_input":"2021-11-08T02:41:23.428121Z","iopub.status.idle":"2021-11-08T02:41:23.505418Z","shell.execute_reply.started":"2021-11-08T02:41:23.428068Z","shell.execute_reply":"2021-11-08T02:41:23.504405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## for i in df.columns:\n## print(df[i].value_counts())","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.506873Z","iopub.execute_input":"2021-11-08T02:41:23.507271Z","iopub.status.idle":"2021-11-08T02:41:23.511259Z","shell.execute_reply.started":"2021-11-08T02:41:23.507222Z","shell.execute_reply":"2021-11-08T02:41:23.510373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Test Split","metadata":{}},{"cell_type":"code","source":"df.isnull()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.512587Z","iopub.execute_input":"2021-11-08T02:41:23.513294Z","iopub.status.idle":"2021-11-08T02:41:23.563686Z","shell.execute_reply.started":"2021-11-08T02:41:23.513259Z","shell.execute_reply":"2021-11-08T02:41:23.562989Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df.iloc[:, 1:]\nX = pd.get_dummies(data=X, columns=X.columns[X.dtypes == object], drop_first=False)\nX.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.565125Z","iopub.execute_input":"2021-11-08T02:41:23.565868Z","iopub.status.idle":"2021-11-08T02:41:23.910778Z","shell.execute_reply.started":"2021-11-08T02:41:23.565819Z","shell.execute_reply":"2021-11-08T02:41:23.909891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = df.iloc[:, 0]\ny","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.911864Z","iopub.execute_input":"2021-11-08T02:41:23.912123Z","iopub.status.idle":"2021-11-08T02:41:23.923039Z","shell.execute_reply.started":"2021-11-08T02:41:23.912093Z","shell.execute_reply":"2021-11-08T02:41:23.921966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.924689Z","iopub.execute_input":"2021-11-08T02:41:23.925634Z","iopub.status.idle":"2021-11-08T02:41:23.938282Z","shell.execute_reply.started":"2021-11-08T02:41:23.925593Z","shell.execute_reply":"2021-11-08T02:41:23.937321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_dig = df_dig.iloc[:, 1:]\n\nX_dig.head(2)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-08T02:41:23.939813Z","iopub.execute_input":"2021-11-08T02:41:23.940671Z","iopub.status.idle":"2021-11-08T02:41:23.973297Z","shell.execute_reply.started":"2021-11-08T02:41:23.940619Z","shell.execute_reply":"2021-11-08T02:41:23.972158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_dig = df_dig.iloc[:, 0]","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.975178Z","iopub.execute_input":"2021-11-08T02:41:23.975535Z","iopub.status.idle":"2021-11-08T02:41:23.980980Z","shell.execute_reply.started":"2021-11-08T02:41:23.975490Z","shell.execute_reply":"2021-11-08T02:41:23.979951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tr, X_ts, y_tr, y_ts=train_test_split(X,y,test_size=0.2,random_state=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:23.982617Z","iopub.execute_input":"2021-11-08T02:41:23.983157Z","iopub.status.idle":"2021-11-08T02:41:24.363922Z","shell.execute_reply.started":"2021-11-08T02:41:23.983071Z","shell.execute_reply":"2021-11-08T02:41:24.363174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tr_dig, X_ts_dig, y_tr_dig, y_ts_dig=train_test_split(X_dig,y_dig,test_size=0.2,random_state=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:24.365203Z","iopub.execute_input":"2021-11-08T02:41:24.365636Z","iopub.status.idle":"2021-11-08T02:41:24.420535Z","shell.execute_reply.started":"2021-11-08T02:41:24.365603Z","shell.execute_reply":"2021-11-08T02:41:24.419802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_scale = StandardScaler()\nx_scale.fit(X_tr)\nX_trScaled = x_scale.transform(X_tr)\nX_tsScaled = x_scale.transform(X_ts)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:24.421855Z","iopub.execute_input":"2021-11-08T02:41:24.422273Z","iopub.status.idle":"2021-11-08T02:41:25.441384Z","shell.execute_reply.started":"2021-11-08T02:41:24.422239Z","shell.execute_reply":"2021-11-08T02:41:25.440724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(nrows=10, ncols=10, figsize=(10,10))\n\nfor i in range(10): \n num_i = X_tr[y_tr == i]\n ax[0][i].set_title(i)\n for j in range(10):\n ax[j][i].axis('off')\n ax[j][i].imshow(num_i.iloc[j, :].to_numpy().astype(np.uint8).reshape(28, 28), cmap='gray')\n","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:25.442601Z","iopub.execute_input":"2021-11-08T02:41:25.442953Z","iopub.status.idle":"2021-11-08T02:41:30.163573Z","shell.execute_reply.started":"2021-11-08T02:41:25.442924Z","shell.execute_reply":"2021-11-08T02:41:30.162856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Standard Scaler ","metadata":{}},{"cell_type":"code","source":"x_scale = StandardScaler()","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:30.164746Z","iopub.execute_input":"2021-11-08T02:41:30.165079Z","iopub.status.idle":"2021-11-08T02:41:30.169430Z","shell.execute_reply.started":"2021-11-08T02:41:30.165044Z","shell.execute_reply":"2021-11-08T02:41:30.168827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_scale.fit_transform(X_tr)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:30.171245Z","iopub.execute_input":"2021-11-08T02:41:30.171494Z","iopub.status.idle":"2021-11-08T02:41:31.131606Z","shell.execute_reply.started":"2021-11-08T02:41:30.171463Z","shell.execute_reply":"2021-11-08T02:41:31.130916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_trScaled = x_scale.transform(X_tr)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.132948Z","iopub.execute_input":"2021-11-08T02:41:31.133423Z","iopub.status.idle":"2021-11-08T02:41:31.337283Z","shell.execute_reply.started":"2021-11-08T02:41:31.133350Z","shell.execute_reply":"2021-11-08T02:41:31.336464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_tsScaled = x_scale.transform(X_ts)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.338998Z","iopub.execute_input":"2021-11-08T02:41:31.339502Z","iopub.status.idle":"2021-11-08T02:41:31.402366Z","shell.execute_reply.started":"2021-11-08T02:41:31.339449Z","shell.execute_reply":"2021-11-08T02:41:31.401344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Principal component analysis","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.404478Z","iopub.execute_input":"2021-11-08T02:41:31.405497Z","iopub.status.idle":"2021-11-08T02:41:31.427105Z","shell.execute_reply.started":"2021-11-08T02:41:31.405452Z","shell.execute_reply":"2021-11-08T02:41:31.426072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=1, svd_solver='arpack')\n#.fit(X_trScaled)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.428630Z","iopub.execute_input":"2021-11-08T02:41:31.429563Z","iopub.status.idle":"2021-11-08T02:41:31.434777Z","shell.execute_reply.started":"2021-11-08T02:41:31.429508Z","shell.execute_reply":"2021-11-08T02:41:31.433907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.436252Z","iopub.execute_input":"2021-11-08T02:41:31.436523Z","iopub.status.idle":"2021-11-08T02:41:31.453251Z","shell.execute_reply.started":"2021-11-08T02:41:31.436492Z","shell.execute_reply":"2021-11-08T02:41:31.452112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom sklearn.decomposition import PCA","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.455143Z","iopub.execute_input":"2021-11-08T02:41:31.455487Z","iopub.status.idle":"2021-11-08T02:41:31.463057Z","shell.execute_reply.started":"2021-11-08T02:41:31.455453Z","shell.execute_reply":"2021-11-08T02:41:31.461883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_pca = datetime.now()\nstart_pca","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.465047Z","iopub.execute_input":"2021-11-08T02:41:31.466159Z","iopub.status.idle":"2021-11-08T02:41:31.478742Z","shell.execute_reply.started":"2021-11-08T02:41:31.466109Z","shell.execute_reply":"2021-11-08T02:41:31.478099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(n_components=237,svd_solver='full')\npca.fit(X)\n\nmm = pca.explained_variance_ratio_\nsum(pca.explained_variance_ratio_)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:31.479933Z","iopub.execute_input":"2021-11-08T02:41:31.480681Z","iopub.status.idle":"2021-11-08T02:41:39.681175Z","shell.execute_reply.started":"2021-11-08T02:41:31.480635Z","shell.execute_reply":"2021-11-08T02:41:39.680195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_pca = datetime.now()\nend_pca","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:39.682597Z","iopub.execute_input":"2021-11-08T02:41:39.682912Z","iopub.status.idle":"2021-11-08T02:41:39.690263Z","shell.execute_reply.started":"2021-11-08T02:41:39.682869Z","shell.execute_reply":"2021-11-08T02:41:39.689380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(end_pca-start_pca)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:39.691816Z","iopub.execute_input":"2021-11-08T02:41:39.692155Z","iopub.status.idle":"2021-11-08T02:41:39.702284Z","shell.execute_reply.started":"2021-11-08T02:41:39.692109Z","shell.execute_reply":"2021-11-08T02:41:39.701592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mm[0]","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:39.703629Z","iopub.execute_input":"2021-11-08T02:41:39.704083Z","iopub.status.idle":"2021-11-08T02:41:39.717340Z","shell.execute_reply.started":"2021-11-08T02:41:39.704035Z","shell.execute_reply":"2021-11-08T02:41:39.716598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#sum0=0\n#for k,i in enumerate(mm):\n# sum0+=i\n# if sum0>0.95:\n# print(k)\n# break","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:39.718671Z","iopub.execute_input":"2021-11-08T02:41:39.718914Z","iopub.status.idle":"2021-11-08T02:41:39.729521Z","shell.execute_reply.started":"2021-11-08T02:41:39.718884Z","shell.execute_reply":"2021-11-08T02:41:39.728578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_trScaler = pca.transform(X_trScaled)\nX_tsScaler = pca.transform(X_tsScaled)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:39.735706Z","iopub.execute_input":"2021-11-08T02:41:39.736039Z","iopub.status.idle":"2021-11-08T02:41:40.517428Z","shell.execute_reply.started":"2021-11-08T02:41:39.735988Z","shell.execute_reply":"2021-11-08T02:41:40.516332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modeling","metadata":{}},{"cell_type":"markdown","source":"## Random Forest","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:40.529696Z","iopub.execute_input":"2021-11-08T02:41:40.534093Z","iopub.status.idle":"2021-11-08T02:41:40.611958Z","shell.execute_reply.started":"2021-11-08T02:41:40.533996Z","shell.execute_reply":"2021-11-08T02:41:40.610945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfc = RandomForestClassifier(max_depth=10, criterion='entropy', random_state=0, n_estimators=100,max_features=15)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:40.628784Z","iopub.execute_input":"2021-11-08T02:41:40.629494Z","iopub.status.idle":"2021-11-08T02:41:40.637240Z","shell.execute_reply.started":"2021-11-08T02:41:40.629440Z","shell.execute_reply":"2021-11-08T02:41:40.636339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_rf = datetime.now()\nstart_rf","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:40.638783Z","iopub.execute_input":"2021-11-08T02:41:40.640233Z","iopub.status.idle":"2021-11-08T02:41:40.648901Z","shell.execute_reply.started":"2021-11-08T02:41:40.640185Z","shell.execute_reply":"2021-11-08T02:41:40.648294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfc.fit(X_tr, y_tr)","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2021-11-08T02:41:40.650335Z","iopub.execute_input":"2021-11-08T02:41:40.650804Z","iopub.status.idle":"2021-11-08T02:41:57.391647Z","shell.execute_reply.started":"2021-11-08T02:41:40.650757Z","shell.execute_reply":"2021-11-08T02:41:57.390877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"end_rf = datetime.now()\nend_rf","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:57.393250Z","iopub.execute_input":"2021-11-08T02:41:57.393729Z","iopub.status.idle":"2021-11-08T02:41:57.400839Z","shell.execute_reply.started":"2021-11-08T02:41:57.393679Z","shell.execute_reply":"2021-11-08T02:41:57.399935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(end_rf - start_rf)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:57.402309Z","iopub.execute_input":"2021-11-08T02:41:57.403447Z","iopub.status.idle":"2021-11-08T02:41:57.416563Z","shell.execute_reply.started":"2021-11-08T02:41:57.403277Z","shell.execute_reply":"2021-11-08T02:41:57.415292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_tr_rfc = rfc.predict(X_tr)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:57.418148Z","iopub.execute_input":"2021-11-08T02:41:57.418976Z","iopub.status.idle":"2021-11-08T02:41:58.765640Z","shell.execute_reply.started":"2021-11-08T02:41:57.418920Z","shell.execute_reply":"2021-11-08T02:41:58.764806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('\\t\\tThe Classification Report is: '+'\\n\\n', classification_report(y_tr, y_tr_rfc), '\\n')\nprint('The Confusion Matrix is: '+'\\n\\n', confusion_matrix(y_tr, y_tr_rfc), '\\n')\nprint('The accuracy score is: ', accuracy_score(y_tr, y_tr_rfc))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:58.767142Z","iopub.execute_input":"2021-11-08T02:41:58.768387Z","iopub.status.idle":"2021-11-08T02:41:58.955247Z","shell.execute_reply.started":"2021-11-08T02:41:58.768349Z","shell.execute_reply":"2021-11-08T02:41:58.954646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_ts_rfc = rfc.predict(df_test.iloc[:, 1:])\ny_ts_rfc.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:58.956675Z","iopub.execute_input":"2021-11-08T02:41:58.956933Z","iopub.status.idle":"2021-11-08T02:41:59.113333Z","shell.execute_reply.started":"2021-11-08T02:41:58.956902Z","shell.execute_reply":"2021-11-08T02:41:59.112371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(y_ts_rfc)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.114538Z","iopub.execute_input":"2021-11-08T02:41:59.114777Z","iopub.status.idle":"2021-11-08T02:41:59.122011Z","shell.execute_reply.started":"2021-11-08T02:41:59.114749Z","shell.execute_reply":"2021-11-08T02:41:59.121070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_series = list(pd.Series(df_test['id']))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.123217Z","iopub.execute_input":"2021-11-08T02:41:59.123461Z","iopub.status.idle":"2021-11-08T02:41:59.137641Z","shell.execute_reply.started":"2021-11-08T02:41:59.123425Z","shell.execute_reply":"2021-11-08T02:41:59.136843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#id_series.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.139565Z","iopub.execute_input":"2021-11-08T02:41:59.140184Z","iopub.status.idle":"2021-11-08T02:41:59.152431Z","shell.execute_reply.started":"2021-11-08T02:41:59.140150Z","shell.execute_reply":"2021-11-08T02:41:59.151409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict1_df = pd.DataFrame([id_series,list(y_ts_rfc)]).transpose()\n","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.153990Z","iopub.execute_input":"2021-11-08T02:41:59.154358Z","iopub.status.idle":"2021-11-08T02:41:59.385305Z","shell.execute_reply.started":"2021-11-08T02:41:59.154314Z","shell.execute_reply":"2021-11-08T02:41:59.384332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict1_df.columns = ['id','label']","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.386819Z","iopub.execute_input":"2021-11-08T02:41:59.387086Z","iopub.status.idle":"2021-11-08T02:41:59.392416Z","shell.execute_reply.started":"2021-11-08T02:41:59.387052Z","shell.execute_reply":"2021-11-08T02:41:59.391621Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict1_df.reset_index(drop=True, inplace=True)\npredict1_df","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.393746Z","iopub.execute_input":"2021-11-08T02:41:59.394400Z","iopub.status.idle":"2021-11-08T02:41:59.419838Z","shell.execute_reply.started":"2021-11-08T02:41:59.394356Z","shell.execute_reply":"2021-11-08T02:41:59.418542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sb = pd.read_csv(\"/kaggle/input/sbsbsb/sb.csv\")\n\n#combinedArray = np.concatenate((id_series,list(y_ts_rfc)))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.422087Z","iopub.execute_input":"2021-11-08T02:41:59.422945Z","iopub.status.idle":"2021-11-08T02:41:59.437572Z","shell.execute_reply.started":"2021-11-08T02:41:59.422887Z","shell.execute_reply":"2021-11-08T02:41:59.436677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#combinedArray","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.438973Z","iopub.execute_input":"2021-11-08T02:41:59.439756Z","iopub.status.idle":"2021-11-08T02:41:59.443644Z","shell.execute_reply.started":"2021-11-08T02:41:59.439720Z","shell.execute_reply":"2021-11-08T02:41:59.442818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_output = pd.DataFrame(combinedArray)\n#df_output","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.445386Z","iopub.execute_input":"2021-11-08T02:41:59.445706Z","iopub.status.idle":"2021-11-08T02:41:59.456556Z","shell.execute_reply.started":"2021-11-08T02:41:59.445665Z","shell.execute_reply":"2021-11-08T02:41:59.455345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#y_ts.index","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.458254Z","iopub.execute_input":"2021-11-08T02:41:59.458619Z","iopub.status.idle":"2021-11-08T02:41:59.471389Z","shell.execute_reply.started":"2021-11-08T02:41:59.458571Z","shell.execute_reply":"2021-11-08T02:41:59.470324Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_test['id']","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.472894Z","iopub.execute_input":"2021-11-08T02:41:59.473734Z","iopub.status.idle":"2021-11-08T02:41:59.485497Z","shell.execute_reply.started":"2021-11-08T02:41:59.473684Z","shell.execute_reply":"2021-11-08T02:41:59.484536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#conda install openpyxl","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.487370Z","iopub.execute_input":"2021-11-08T02:41:59.488631Z","iopub.status.idle":"2021-11-08T02:41:59.502909Z","shell.execute_reply.started":"2021-11-08T02:41:59.488577Z","shell.execute_reply":"2021-11-08T02:41:59.501825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#output_directory = 'predict1.xlsx'\n#writer = pd.ExcelWriter(output_directory)\n#predict1_df.to_excel(writer,sheet_name='predict',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.504921Z","iopub.execute_input":"2021-11-08T02:41:59.505744Z","iopub.status.idle":"2021-11-08T02:41:59.514981Z","shell.execute_reply.started":"2021-11-08T02:41:59.505692Z","shell.execute_reply":"2021-11-08T02:41:59.513845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sb.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.516917Z","iopub.execute_input":"2021-11-08T02:41:59.517554Z","iopub.status.idle":"2021-11-08T02:41:59.540781Z","shell.execute_reply.started":"2021-11-08T02:41:59.517516Z","shell.execute_reply":"2021-11-08T02:41:59.540041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Forest with PCA","metadata":{}},{"cell_type":"code","source":"## rfc_pca = RandomForestClassifier(max_depth=10, criterion='entropy', random_state=0, n_estimators=100,max_features=15)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.542818Z","iopub.execute_input":"2021-11-08T02:41:59.543852Z","iopub.status.idle":"2021-11-08T02:41:59.547730Z","shell.execute_reply.started":"2021-11-08T02:41:59.543811Z","shell.execute_reply":"2021-11-08T02:41:59.547087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## start_rf_pca = datetime.now()\n## start_rf_pca","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.549640Z","iopub.execute_input":"2021-11-08T02:41:59.550586Z","iopub.status.idle":"2021-11-08T02:41:59.562542Z","shell.execute_reply.started":"2021-11-08T02:41:59.550549Z","shell.execute_reply":"2021-11-08T02:41:59.561754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## rfc_pca.fit(X_trScaler, y_tr)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.564140Z","iopub.execute_input":"2021-11-08T02:41:59.564589Z","iopub.status.idle":"2021-11-08T02:41:59.575641Z","shell.execute_reply.started":"2021-11-08T02:41:59.564539Z","shell.execute_reply":"2021-11-08T02:41:59.574934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## end_rf_pca = datetime.now()\n## end_rf_pca","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.577123Z","iopub.execute_input":"2021-11-08T02:41:59.577562Z","iopub.status.idle":"2021-11-08T02:41:59.589952Z","shell.execute_reply.started":"2021-11-08T02:41:59.577515Z","shell.execute_reply":"2021-11-08T02:41:59.589153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## print(end_rf_pca - start_rf_pca)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.591446Z","iopub.execute_input":"2021-11-08T02:41:59.592260Z","iopub.status.idle":"2021-11-08T02:41:59.603559Z","shell.execute_reply.started":"2021-11-08T02:41:59.592213Z","shell.execute_reply":"2021-11-08T02:41:59.602524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## y_ts_rfc_pca = rfc_pca.predict(X_tsScaler)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.605104Z","iopub.execute_input":"2021-11-08T02:41:59.605535Z","iopub.status.idle":"2021-11-08T02:41:59.616348Z","shell.execute_reply.started":"2021-11-08T02:41:59.605483Z","shell.execute_reply":"2021-11-08T02:41:59.615104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## print('\\t\\tThe Classification Report is: '+'\\n\\n', classification_report(y_ts, y_ts_rfc_pca), '\\n')\n## print('The Confusion Matrix is: '+'\\n\\n', confusion_matrix(y_ts, y_ts_rfc_pca), '\\n')\n## print('The accuracy score is: ', accuracy_score(y_ts, y_ts_rfc_pca))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.617826Z","iopub.execute_input":"2021-11-08T02:41:59.618206Z","iopub.status.idle":"2021-11-08T02:41:59.627840Z","shell.execute_reply.started":"2021-11-08T02:41:59.618160Z","shell.execute_reply":"2021-11-08T02:41:59.627241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## K Means","metadata":{}},{"cell_type":"code","source":"## import sys\n## import sklearn\n## import matplotlib\n## import numpy as np\n## import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.629414Z","iopub.execute_input":"2021-11-08T02:41:59.629881Z","iopub.status.idle":"2021-11-08T02:41:59.639751Z","shell.execute_reply.started":"2021-11-08T02:41:59.629847Z","shell.execute_reply":"2021-11-08T02:41:59.639112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert each image to 1 dimensional array\n# normalize the data to 0 - 1\n## X_dig = X_dig.astype(float) / 255\n## print(X_dig.shape)\n#print(X[0].shape)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.641503Z","iopub.execute_input":"2021-11-08T02:41:59.642260Z","iopub.status.idle":"2021-11-08T02:41:59.652107Z","shell.execute_reply.started":"2021-11-08T02:41:59.641927Z","shell.execute_reply":"2021-11-08T02:41:59.651200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## from sklearn.cluster import MiniBatchKMeans\n## n_digits = len(np.unique(y_ts_dig))\n## print(n_digits)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.653980Z","iopub.execute_input":"2021-11-08T02:41:59.654631Z","iopub.status.idle":"2021-11-08T02:41:59.663902Z","shell.execute_reply.started":"2021-11-08T02:41:59.654585Z","shell.execute_reply":"2021-11-08T02:41:59.663115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## kmeans = MiniBatchKMeans(n_clusters = n_digits)\n# Fit the model to the training data\n## kmeans.fit(X_dig)\n## kmeans.labels_","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.665456Z","iopub.execute_input":"2021-11-08T02:41:59.666170Z","iopub.status.idle":"2021-11-08T02:41:59.677271Z","shell.execute_reply.started":"2021-11-08T02:41:59.666111Z","shell.execute_reply":"2021-11-08T02:41:59.676238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n## def infer_cluster_labels(kmeans, actual_labels):\n## inferred_labels = {}\n\n## for i in range(kmeans.n_clusters):\n\n # find index of points in cluster\n## labels = []\n## index = np.where(kmeans.labels_ == i)\n\n # append actual labels for each point in cluster\n## labels.append(actual_labels[index])\n\n # determine most common label\n## if len(labels[0]) == 1:\n ## counts = np.bincount(labels[0])\n ## else:\n## counts = np.bincount(np.squeeze(labels))\n\n # assign the cluster to a value in the inferred_labels dictionary\n ## if np.argmax(counts) in inferred_labels:\n ## # append the new number to the existing array at this slot\n ## inferred_labels[np.argmax(counts)].append(i)\n ## else:\n # create a new array in this slot\n ## inferred_labels[np.argmax(counts)] = [i]\n\n #print(labels)\n #print('Cluster: {}, label: {}'.format(i, np.argmax(counts)))\n\n## return inferred_labels\n\n## def infer_data_labels(X_labels, cluster_labels):\n # empty array of len(X)\n## predicted_labels = np.zeros(len(X_labels)).astype(np.uint8)\n\n## for i, cluster in enumerate(X_labels):\n ## for key, value in cluster_labels.items():\n ## if cluster in value:\n ## predicted_labels[i] = key\n\n## return predicted_labels\n\n# test the infer_cluster_labels() and infer_data_labels() functions\n\n## cluster_labels = infer_cluster_labels(kmeans, Y)\n## X_clusters = kmeans.predict(X)\n## predicted_labels = infer_data_labels(X_clusters, cluster_labels)\n## print(predicted_labels[:20])\n## print(Y[:20])\n","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.679185Z","iopub.execute_input":"2021-11-08T02:41:59.679502Z","iopub.status.idle":"2021-11-08T02:41:59.690919Z","shell.execute_reply.started":"2021-11-08T02:41:59.679467Z","shell.execute_reply":"2021-11-08T02:41:59.690003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Initialize and fit KMeans algorithm\n## kmeans = MiniBatchKMeans(n_clusters = 10)\n## kmeans.fit(X_tr_dig)\n\n# record centroid values\n## ## centroids = kmeans.cluster_centers_\n## ## centroids[0].shape","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.692410Z","iopub.execute_input":"2021-11-08T02:41:59.693769Z","iopub.status.idle":"2021-11-08T02:41:59.704267Z","shell.execute_reply.started":"2021-11-08T02:41:59.693733Z","shell.execute_reply":"2021-11-08T02:41:59.703561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## kmeans = MiniBatchKMeans(n_clusters = 36)\n## kmeans.fit(X_tr_dig)\n\n# record centroid values\n## centroids = kmeans.cluster_centers_\n\n## images = centroids.reshape(36, 28, 28)\n## images *= 255\n## images = images.astype(np.uint8)\n","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.705496Z","iopub.execute_input":"2021-11-08T02:41:59.706009Z","iopub.status.idle":"2021-11-08T02:41:59.718177Z","shell.execute_reply.started":"2021-11-08T02:41:59.705973Z","shell.execute_reply":"2021-11-08T02:41:59.717203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## y_clusters = kmeans.predict(X_ts_dig)","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.719551Z","iopub.execute_input":"2021-11-08T02:41:59.720440Z","iopub.status.idle":"2021-11-08T02:41:59.733537Z","shell.execute_reply.started":"2021-11-08T02:41:59.720395Z","shell.execute_reply":"2021-11-08T02:41:59.732347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## y_clusters","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.735380Z","iopub.execute_input":"2021-11-08T02:41:59.736128Z","iopub.status.idle":"2021-11-08T02:41:59.748303Z","shell.execute_reply.started":"2021-11-08T02:41:59.736085Z","shell.execute_reply":"2021-11-08T02:41:59.747265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## print('\\t\\tThe Classification Report is: '+'\\n\\n', classification_report(y_ts_dig, y_clusters), '\\n')\n## print('The Confusion Matrix is: '+'\\n\\n', confusion_matrix(y_ts_dig, y_clusters), '\\n')\n## print('The accuracy score is: ', accuracy_score(y_ts_dig, y_clusters))","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.749842Z","iopub.execute_input":"2021-11-08T02:41:59.750704Z","iopub.status.idle":"2021-11-08T02:41:59.761797Z","shell.execute_reply.started":"2021-11-08T02:41:59.750647Z","shell.execute_reply":"2021-11-08T02:41:59.760778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## list(y_tr_dig)[:10]","metadata":{"execution":{"iopub.status.busy":"2021-11-08T02:41:59.763376Z","iopub.execute_input":"2021-11-08T02:41:59.764184Z","iopub.status.idle":"2021-11-08T02:41:59.775153Z","shell.execute_reply.started":"2021-11-08T02:41:59.764134Z","shell.execute_reply":"2021-11-08T02:41:59.774253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}