{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import sys\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n#!conda install --yes xgboost\nimport xgboost\n\n%matplotlib inline\nimport matplotlib\nimport matplotlib as mpl\nimport matplotlib.pyplot as plt\nmpl.rc('axes', labelsize=14)\nmpl.rc('xtick', labelsize=12)\nmpl.rc('ytick', labelsize=12)\n\n#features\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, LabelEncoder, MinMaxScaler\n\n#stats model\nfrom sklearn import datasets, linear_model\nimport statsmodels.formula.api as smf\n\n# Model Selection\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.tree import DecisionTreeClassifier\n#!conda install --yes lightgbm\nfrom lightgbm import LGBMClassifier\nfrom sklearn.linear_model import LassoCV, LinearRegression\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV, cross_val_score\n\n# Model Accuracy\nfrom sklearn import metrics\nfrom sklearn.metrics import r2_score, accuracy_score, mean_poisson_deviance, mean_squared_error \n\n# Import librairies\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.model_selection import KFold, cross_val_score, train_test_split\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\n\nfrom sklearn.svm import SVC\nfrom sklearn.model_selection import GridSearchCV, LeaveOneOut\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import GaussianNB, BernoulliNB\nfrom sklearn.ensemble import RandomForestClassifier\n\nimport sklearn.metrics as metrics\nfrom sklearn.metrics import roc_curve\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score\n\nfrom sklearn.tree import export_graphviz\nimport six\nimport sys\nsys.modules['sklearn.externals.six'] = six\nfrom sklearn.externals.six import StringIO  \nfrom IPython.display import Image  \n\n\n# gradient boosting for classification in scikit-learn\nfrom numpy import mean\nfrom numpy import std\nfrom sklearn.datasets import make_classification\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import RepeatedStratifiedKFold\nfrom matplotlib import pyplot\n\nfrom sklearn.ensemble import BaggingClassifier\nfrom sklearn.ensemble import ExtraTreesClassifier\n\npd.pandas.set_option('display.max_columns', None)\npd.pandas.set_option('display.max_rows', None)\n\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\nwarnings.simplefilter(action='ignore', category=UserWarning)\n\nfrom datetime import datetime \n\nimport sys\nimport sklearn\nimport matplotlib\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\nfrom sklearn.cluster import MiniBatchKMeans\nfrom sklearn.cluster import KMeans\n\nimport tensorflow as tf\nprint(\"You are using TensorFlow version\", tf.__version__)\n#if len(tf.config.list_physical_devices('GPU')) > 0:\n#  print(\"You have a GPU enabled.\")\n#else:\n#  print(\"Enable a GPU before running this notebook.\")\nimport tensorflow as tf\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.models import Sequential  # initial NN\nfrom tensorflow.keras.layers import Input, Dense, Activation, Dropout, InputLayer, BatchNormalization # construct each layer\nfrom tensorflow.keras.layers import Conv2D # swipe across the image by 1\nfrom tensorflow.keras.layers import MaxPool2D, GlobalMaxPool2D # swipe across by pool size\nfrom tensorflow.keras.layers import Flatten, GlobalAveragePooling2D\n\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau,LearningRateScheduler, TensorBoard, EarlyStopping, ModelCheckpoint\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.losses import categorical_crossentropy, binary_crossentropy, mean_squared_error,sparse_categorical_crossentropy\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport cv2\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.image as mpimg \n%matplotlib inline\nfrom tqdm.notebook import trange,tqdm\nfrom IPython.display import Image, display, Markdown, clear_output\n\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn import svm, metrics\nfrom sklearn.decomposition import PCA\nfrom skimage import io\n\n# # tensorflow libraries\n# import tensorflow as tf\n# from keras.models import Sequential\n# from keras.layers import Input,Conv2D,Activation,Dense,Lambda,Flatten,Embedding,PReLU,BatchNormalization,Dropout,MaxPooling2D,GlobalMaxPooling2D,GlobalAveragePooling2D\n# from sklearn.model_selection import train_test_split\n# from keras.models import Model\n# from keras import backend as K","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:17.163252Z","iopub.execute_input":"2022-05-23T13:06:17.164044Z","iopub.status.idle":"2022-05-23T13:06:17.433379Z","shell.execute_reply.started":"2022-05-23T13:06:17.164007Z","shell.execute_reply":"2022-05-23T13:06:17.432279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tf.config.list_physical_devices()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:17.435129Z","iopub.execute_input":"2022-05-23T13:06:17.435522Z","iopub.status.idle":"2022-05-23T13:06:17.447977Z","shell.execute_reply.started":"2022-05-23T13:06:17.435489Z","shell.execute_reply":"2022-05-23T13:06:17.447134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train =pd.read_csv('../input/digit-recognizer/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:17.449423Z","iopub.execute_input":"2022-05-23T13:06:17.450273Z","iopub.status.idle":"2022-05-23T13:06:20.5572Z","shell.execute_reply.started":"2022-05-23T13:06:17.450238Z","shell.execute_reply":"2022-05-23T13:06:20.55662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('../input/digit-recognizer/test.csv')\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:20.558204Z","iopub.execute_input":"2022-05-23T13:06:20.558855Z","iopub.status.idle":"2022-05-23T13:06:22.500461Z","shell.execute_reply.started":"2022-05-23T13:06:20.558824Z","shell.execute_reply":"2022-05-23T13:06:22.499581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum().any()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:22.502338Z","iopub.execute_input":"2022-05-23T13:06:22.502553Z","iopub.status.idle":"2022-05-23T13:06:22.553574Z","shell.execute_reply.started":"2022-05-23T13:06:22.502528Z","shell.execute_reply":"2022-05-23T13:06:22.552761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum().any()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:22.554874Z","iopub.execute_input":"2022-05-23T13:06:22.555135Z","iopub.status.idle":"2022-05-23T13:06:22.605132Z","shell.execute_reply.started":"2022-05-23T13:06:22.555104Z","shell.execute_reply":"2022-05-23T13:06:22.604166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:22.606215Z","iopub.execute_input":"2022-05-23T13:06:22.606453Z","iopub.status.idle":"2022-05-23T13:06:22.612379Z","shell.execute_reply.started":"2022-05-23T13:06:22.606424Z","shell.execute_reply":"2022-05-23T13:06:22.610985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ntrain = train.shape[0]\ny_train = train['label'].values\ntrain=train.drop(['label'], axis=1)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:22.613539Z","iopub.execute_input":"2022-05-23T13:06:22.61379Z","iopub.status.idle":"2022-05-23T13:06:22.736877Z","shell.execute_reply.started":"2022-05-23T13:06:22.613719Z","shell.execute_reply":"2022-05-23T13:06:22.735961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Scalled data:\")\n# Drawing box plots\nplt.figure(figsize = (130,15))\ntrain.boxplot()\nplt.figure(figsize = (130,15))\ntest.boxplot()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:06:22.739998Z","iopub.execute_input":"2022-05-23T13:06:22.740249Z","iopub.status.idle":"2022-05-23T13:08:01.836905Z","shell.execute_reply.started":"2022-05-23T13:06:22.74022Z","shell.execute_reply":"2022-05-23T13:08:01.835926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train[train.columns]=preprocessing.scale(train[train.columns])\n#test[test.columns]=preprocessing.scale(test[test.columns])\n# Drawing box plots\n# plt.figure(figsize = (130,15))\n# train.boxplot()\n# plt.figure(figsize = (130,15))\n# test.boxplot()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:08:01.838337Z","iopub.execute_input":"2022-05-23T13:08:01.839084Z","iopub.status.idle":"2022-05-23T13:08:01.843306Z","shell.execute_reply.started":"2022-05-23T13:08:01.839039Z","shell.execute_reply":"2022-05-23T13:08:01.84233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#splitting data in to train and validation sets\ntrain['label'] = y_train\n# train-test 80-20 split\ntrain_reg, test_reg = train_test_split(train, \n                                     train_size = 0.8, \n                                     test_size = 0.2, \n                                     random_state = 13)\n\n# Divide into X_train, y_train, X_test, y_test\ny_train_1 = train_reg.pop('label')\nX_train_1 = train_reg\n\ny_test_1 = test_reg.pop('label')\nX_test_1 = test_reg\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:08:01.84463Z","iopub.execute_input":"2022-05-23T13:08:01.844912Z","iopub.status.idle":"2022-05-23T13:08:02.339713Z","shell.execute_reply.started":"2022-05-23T13:08:01.844877Z","shell.execute_reply":"2022-05-23T13:08:02.33879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(X_train_1.shape)\nprint(y_train_1.shape)\nprint(X_test_1.shape)\nprint(y_test_1.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:08:02.34093Z","iopub.execute_input":"2022-05-23T13:08:02.341154Z","iopub.status.idle":"2022-05-23T13:08:02.346649Z","shell.execute_reply.started":"2022-05-23T13:08:02.341126Z","shell.execute_reply":"2022-05-23T13:08:02.345796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_columns=X_train_1.columns\nmodel_columns","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:08:02.347774Z","iopub.execute_input":"2022-05-23T13:08:02.347975Z","iopub.status.idle":"2022-05-23T13:08:02.360514Z","shell.execute_reply.started":"2022-05-23T13:08:02.347949Z","shell.execute_reply":"2022-05-23T13:08:02.359654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Ramdom forest with entropy and other\nrf = RandomForestClassifier(n_estimators=100,max_features='auto',max_depth=200,criterion=\"entropy\")\nstart=datetime.now()\n# Train model\nrfc= rf.fit(X_train_1, y_train_1)\nend=datetime.now()\nprint(end-start)\n## Performance Matrix on test data set\ny_pred_rfc_e = rfc.predict(X_test_1)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:08:02.364922Z","iopub.execute_input":"2022-05-23T13:08:02.365253Z","iopub.status.idle":"2022-05-23T13:08:28.718111Z","shell.execute_reply.started":"2022-05-23T13:08:02.365215Z","shell.execute_reply":"2022-05-23T13:08:28.717252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a KFold object with 10 splits \nfolds = KFold(n_splits = 3, shuffle = True, random_state = 13)\nscores = cross_val_score(rfc, X_train_1, y_train_1, scoring='r2', cv=folds)\nprint(\"R^2 Scores: \",scores)\n\n# create a KFold object with 10 splits \nfolds = KFold(n_splits = 3, shuffle = True, random_state = 13)\nscores = cross_val_score(rfc, X_train_1, y_train_1, scoring='neg_root_mean_squared_error', cv=folds)\nprint(\"\")","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:08:28.719519Z","iopub.execute_input":"2022-05-23T13:08:28.720081Z","iopub.status.idle":"2022-05-23T13:10:12.102035Z","shell.execute_reply.started":"2022-05-23T13:08:28.720029Z","shell.execute_reply":"2022-05-23T13:10:12.101153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#extract single tree\nestimator = rfc.estimators_[5]\nprint(estimator)\nn_nodes = rfc.estimators_[0].tree_.node_count\nprint(\"n_nodes={}\".format(n_nodes))\nprint(metrics.confusion_matrix(y_test_1, y_pred_rfc_e))         ## confusion_matrix\nprint(metrics.classification_report(y_test_1, y_pred_rfc_e))    ## classification_report\n#Visualize graph\ndot_data = StringIO()\nexport_graphviz(estimator, out_file=dot_data,  \n                filled=True, rounded=True,\n                special_characters=True,\n                feature_names = model_columns,\n                class_names=['0','1','2','3','4','5','6','7','8','9'])\n\ngraph = pydotplus.graph_from_dot_data(dot_data.getvalue())  \ngraph.write_png('RandomForestClassifer_entropy.png')\nImage(graph.create_png())","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:12.103304Z","iopub.execute_input":"2022-05-23T13:10:12.103541Z","iopub.status.idle":"2022-05-23T13:10:13.021213Z","shell.execute_reply.started":"2022-05-23T13:10:12.103511Z","shell.execute_reply":"2022-05-23T13:10:13.020085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.shape[0]+1","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.022268Z","iopub.status.idle":"2022-05-23T13:10:13.022612Z","shell.execute_reply.started":"2022-05-23T13:10:13.022431Z","shell.execute_reply":"2022-05-23T13:10:13.022455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Predicting on test data\npred=rf.predict(test)\npred = pd.Series(pred,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,test.shape[0]+1),name = \"ImageId\"),pred],axis = 1)\nsubmission.to_csv(\"digit_rfc_without_scale.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.024152Z","iopub.status.idle":"2022-05-23T13:10:13.02497Z","shell.execute_reply.started":"2022-05-23T13:10:13.024781Z","shell.execute_reply":"2022-05-23T13:10:13.024802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Combining train and test data for PCA\ntrain =pd.read_csv('digit-recognizer/train.csv')\ntest = pd.read_csv('digit-recognizer/test.csv')\nprint(train.shape)\nprint(test.shape)\n\n#Combinig train and test data.\nntrain = train.shape[0]\nntest = test.shape[0]\ny_train = train.label.values\nall_data = pd.concat((train,test)).reset_index(drop=True)\nall_data.drop(['label'], axis=1, inplace=True)\nprint(\"all_data size is : {}\".format(all_data.shape))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.025759Z","iopub.status.idle":"2022-05-23T13:10:13.026407Z","shell.execute_reply.started":"2022-05-23T13:10:13.026204Z","shell.execute_reply":"2022-05-23T13:10:13.026246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Doing PCA\nstart=datetime.now()\npca = PCA(n_components=0.95, svd_solver='full',whiten=True).fit(all_data)\n\n# Check the time to run the model\n\nall_data_pca = pca.transform(all_data)\nend=datetime.now()\nprint(end-start)\nall_data_pca.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.027321Z","iopub.status.idle":"2022-05-23T13:10:13.02816Z","shell.execute_reply.started":"2022-05-23T13:10:13.027932Z","shell.execute_reply":"2022-05-23T13:10:13.027973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_data_pca","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.029047Z","iopub.status.idle":"2022-05-23T13:10:13.029799Z","shell.execute_reply.started":"2022-05-23T13:10:13.029549Z","shell.execute_reply":"2022-05-23T13:10:13.029576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Spilling train and test after PCA\ntrain_pca=all_data_pca[0:ntrain]\nprint(train_pca.shape)\ntest_pca=all_data_pca[ntrain:]\nprint(test_pca.shape)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.030709Z","iopub.status.idle":"2022-05-23T13:10:13.031016Z","shell.execute_reply.started":"2022-05-23T13:10:13.030859Z","shell.execute_reply":"2022-05-23T13:10:13.030875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Ramdom forest with entropy and other\nrf = RandomForestClassifier(n_estimators=100,max_features='auto',max_depth=200,criterion=\"entropy\")\nstart=datetime.now()\n# Train model\nrfc= rf.fit(train_pca, y_train)\nend=datetime.now()\nprint(end-start)\n## Performance Matrix on test data set\ny_pred_rfc_pca = rfc.predict(test_pca)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.031724Z","iopub.status.idle":"2022-05-23T13:10:13.032019Z","shell.execute_reply.started":"2022-05-23T13:10:13.031867Z","shell.execute_reply":"2022-05-23T13:10:13.031883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a KFold object with 10 splits \nfolds = KFold(n_splits = 3, shuffle = True, random_state = 13)\nscores = cross_val_score(rfc, train_pca, y_train, scoring='r2', cv=folds)\nprint(\"R^2 Scores: \",scores)\n\n# create a KFold object with 10 splits \nfolds = KFold(n_splits = 3, shuffle = True, random_state = 13)\nscores = cross_val_score(rfc, train_pca, y_train, scoring='neg_root_mean_squared_error', cv=folds)\nprint(\"\")\nprint(\"RMSE Scores: \",scores)  \n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.03317Z","iopub.status.idle":"2022-05-23T13:10:13.03348Z","shell.execute_reply.started":"2022-05-23T13:10:13.033327Z","shell.execute_reply":"2022-05-23T13:10:13.033343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Final submission data\ntest_pca.shape[0]+1\npred = pd.Series(pred,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,test.shape[0]+1),name = \"ImageId\"),pred],axis = 1)\nsubmission.to_csv(\"digit_rfc_pca.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.034399Z","iopub.status.idle":"2022-05-23T13:10:13.034726Z","shell.execute_reply.started":"2022-05-23T13:10:13.034541Z","shell.execute_reply":"2022-05-23T13:10:13.034557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#K-Means Clustering\ntrain =pd.read_csv('digit-recognizer/train.csv')\ntest = pd.read_csv('digit-recognizer/test.csv')\nprint(train.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.035477Z","iopub.status.idle":"2022-05-23T13:10:13.035768Z","shell.execute_reply.started":"2022-05-23T13:10:13.035609Z","shell.execute_reply":"2022-05-23T13:10:13.035624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ntrain=train.shape[0]\ny_train=train['label'].values\nX_train=train.drop(['label'], axis=1)\nprint('Training Data: {}'.format(X_train.shape))\nprint('Training Labels: {}'.format(y_train.shape))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.03672Z","iopub.status.idle":"2022-05-23T13:10:13.037036Z","shell.execute_reply.started":"2022-05-23T13:10:13.036868Z","shell.execute_reply":"2022-05-23T13:10:13.036884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train=X_train.values.reshape(-1,28,28)\nX_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.037852Z","iopub.status.idle":"2022-05-23T13:10:13.038148Z","shell.execute_reply.started":"2022-05-23T13:10:13.03798Z","shell.execute_reply":"2022-05-23T13:10:13.037995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Visulize data\n# rows = 5 # defining no. of rows in figure\n# cols = 6 # defining no. of colums in figure\n\n# f = plt.figure(figsize=(2*cols,2*rows)) # defining a figure \n\n# for i in range(rows*cols): \n#     f.add_subplot(rows,cols,i+1) # adding sub plot to figure on each iteration\n#     plt.imshow(X_train[i].reshape([28,28]), cmap=\"Blues\") \n#     plt.axis(\"off\")","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.040022Z","iopub.status.idle":"2022-05-23T13:10:13.040362Z","shell.execute_reply.started":"2022-05-23T13:10:13.040168Z","shell.execute_reply":"2022-05-23T13:10:13.040184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # convert each image to 1 dimensional array\n# X = X_train.reshape(len(X_train),-1)\n# Y = y_train\n\n# # normalize the data to 0 - 1\n# X = X.astype(float) / 255.\n\n# print(X.shape)\n# print(X[0].shape)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.041091Z","iopub.status.idle":"2022-05-23T13:10:13.041385Z","shell.execute_reply.started":"2022-05-23T13:10:13.041234Z","shell.execute_reply":"2022-05-23T13:10:13.04125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# n_digits = len(np.unique(y_train))\n# print(n_digits)\n\n# # Initialize KMeans model\n\n# kmeans = MiniBatchKMeans(n_clusters = n_digits)\n\n# # Fit the model to the training data\n\n# kmeans.fit(X)\n\n# kmeans.labels_","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.042348Z","iopub.status.idle":"2022-05-23T13:10:13.042652Z","shell.execute_reply.started":"2022-05-23T13:10:13.042481Z","shell.execute_reply":"2022-05-23T13:10:13.042496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# def infer_cluster_labels(kmeans, actual_labels):\n#     inferred_labels = {}\n#     for i in range(kmeans.n_clusters):\n#         # find index of points in cluster\n#         labels = []\n#         index = np.where(kmeans.labels_ == i)\n#         # append actual labels for each point in cluster\n#         labels.append(actual_labels[index])\n#         # determine most common label\n#         if len(labels[0]) == 1:\n#             counts = np.bincount(labels[0])\n#         else:\n#             counts = np.bincount(np.squeeze(labels))\n#             # assign the cluster to a value in the inferred_labels dictionary\n#         if np.argmax(counts) in inferred_labels:\n#             # append the new number to the existing array at this slot\n#             inferred_labels[np.argmax(counts)].append(i)\n#         else:\n#             inferred_labels[np.argmax(counts)] = [i]\n#     return inferred_labels\n\n# def infer_data_labels(X_labels, cluster_labels):\n#     # empty array of len(X)\n#     predicted_labels = np.zeros(len(X_labels)).astype(np.uint8)\n#     for i, cluster in enumerate(X_labels):\n#         for key, value in cluster_labels.items():\n#             if cluster in value:\n#                 predicted_labels[i] = key\n#     return predicted_labels\n\n# # test the infer_cluster_labels() and infer_data_labels() functions\n\n# cluster_labels = infer_cluster_labels(kmeans, Y)\n# X_clusters = kmeans.predict(X)\n# predicted_labels = infer_data_labels(X_clusters, cluster_labels)\n# print(predicted_labels[:20])\n# print(Y[:20])","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.043406Z","iopub.status.idle":"2022-05-23T13:10:13.043711Z","shell.execute_reply.started":"2022-05-23T13:10:13.043535Z","shell.execute_reply":"2022-05-23T13:10:13.04355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # Initialize and fit KMeans algorithm\n# kmeans = MiniBatchKMeans(n_clusters = 10)\n# kmeans.fit(X)\n\n# # record centroid values\n# centroids = kmeans.cluster_centers_\n\n# # reshape centroids into images\n# images = centroids.reshape(10, 28, 28)\n# images *= 255\n# images = images.astype(np.uint8)\n\n# # determine cluster labels\n# cluster_labels = infer_cluster_labels(kmeans, Y)\n\n# # create figure with subplots using matplotlib.pyplot\n# fig, axs = plt.subplots(2, 5, figsize = (10, 5))\n# plt.gray()\n\n# # loop through subplots and add centroid images\n# for i, ax in enumerate(axs.flat):\n    \n#     # determine inferred label using cluster_labels dictionary\n#     for key, value in cluster_labels.items():\n#         if i in value:\n#             ax.set_title('Inferred Label: {}'.format(key))\n    \n#     # add image to subplot\n#     ax.matshow(images[i])\n#     ax.axis('off')\n    \n# # display the figure\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.044802Z","iopub.status.idle":"2022-05-23T13:10:13.045081Z","shell.execute_reply.started":"2022-05-23T13:10:13.044934Z","shell.execute_reply":"2022-05-23T13:10:13.044948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(metrics.accuracy_score(y_train, predicted_labels))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.046197Z","iopub.status.idle":"2022-05-23T13:10:13.046572Z","shell.execute_reply.started":"2022-05-23T13:10:13.046409Z","shell.execute_reply":"2022-05-23T13:10:13.046427Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train=X_train.reshape(len(X_train),-1)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.047466Z","iopub.status.idle":"2022-05-23T13:10:13.047764Z","shell.execute_reply.started":"2022-05-23T13:10:13.047601Z","shell.execute_reply":"2022-05-23T13:10:13.047617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scaler = StandardScaler()\nX_train = scaler.fit_transform(X_train)\nX_test = scaler.fit_transform(test)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.048569Z","iopub.status.idle":"2022-05-23T13:10:13.048858Z","shell.execute_reply.started":"2022-05-23T13:10:13.048709Z","shell.execute_reply":"2022-05-23T13:10:13.048724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kmeans = KMeans(n_clusters=10, init='random',n_jobs=-1)\n\n# fitting the k means algorithm on scaled data\nkmeans.fit(X_train,y_train)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.049691Z","iopub.status.idle":"2022-05-23T13:10:13.049977Z","shell.execute_reply.started":"2022-05-23T13:10:13.049826Z","shell.execute_reply":"2022-05-23T13:10:13.049841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kmeans.inertia_","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.051173Z","iopub.status.idle":"2022-05-23T13:10:13.051499Z","shell.execute_reply.started":"2022-05-23T13:10:13.051331Z","shell.execute_reply":"2022-05-23T13:10:13.051347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_labels = kmeans.predict(X_train)\npredicted_labels","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.052463Z","iopub.status.idle":"2022-05-23T13:10:13.052802Z","shell.execute_reply.started":"2022-05-23T13:10:13.052596Z","shell.execute_reply":"2022-05-23T13:10:13.052611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.accuracy_score(y_train, predicted_labels))","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.053733Z","iopub.status.idle":"2022-05-23T13:10:13.054031Z","shell.execute_reply.started":"2022-05-23T13:10:13.05387Z","shell.execute_reply":"2022-05-23T13:10:13.053886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create a KFold object with 10 splits \nfolds = KFold(n_splits = 3, shuffle = True, random_state = 13)\nscores = cross_val_score(kmeans, X_train, y_train, scoring='r2', cv=folds)\nprint(\"R^2 Scores: \",scores)\n\n# create a KFold object with 10 splits \nfolds = KFold(n_splits = 3, shuffle = True, random_state = 13)\nscores = cross_val_score(kmeans, X_train, y_train, scoring='neg_root_mean_squared_error', cv=folds)\nprint(\"\")\nprint(\"RMSE Scores: \",scores)  ","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.055628Z","iopub.status.idle":"2022-05-23T13:10:13.056092Z","shell.execute_reply.started":"2022-05-23T13:10:13.055838Z","shell.execute_reply":"2022-05-23T13:10:13.055863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Final Submission\ny_pred=kmeans.predict(X_test)\n\ntest_pca.shape[0]+1\ny_pred = pd.Series(y_pred,name=\"Label\")\nsubmission = pd.concat([pd.Series(range(1,test.shape[0]+1),name = \"ImageId\"),y_pred],axis = 1)\nsubmission.to_csv(\"digit_kmeans.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-23T13:10:13.057667Z","iopub.status.idle":"2022-05-23T13:10:13.058119Z","shell.execute_reply.started":"2022-05-23T13:10:13.057873Z","shell.execute_reply":"2022-05-23T13:10:13.057898Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}