{"cells":[{"metadata":{"_uuid":"4f02161787744df3db0644e4e4b91506cec7c61e"},"cell_type":"markdown","source":"In this kernel I have tried to find the most suitable algorithms(leaving Deep learning) by comparing the performance across various and by further fine tuning algoritms.\n**Load all required preprocessing and model liberaries**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom collections import Counter\n#Importing Preprocessing liberaries\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, ExtraTreesClassifier\nfrom sklearn.preprocessing import StandardScaler, Normalizer, MinMaxScaler, Binarizer, OneHotEncoder\nfrom sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV, RandomizedSearchCV\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import accuracy_score\n#Importing model liberaries \nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.svm import SVC\nimport warnings\nwarnings.filterwarnings(\"ignore\", category=FutureWarning)\nwarnings.filterwarnings(\"ignore\", category=UserWarning)\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9ca82a325fb093cb6b1b3227dee19e23e7e032d"},"cell_type":"markdown","source":"**Declare global hyperparameters and load the input data**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"seed = 1\nscoring='accuracy'\nnum_folds = 10\ntest_size = 0.3\nkfold = KFold(n_splits=num_folds, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b355274ab495597215e6eea3d6af5821da9a6ec"},"cell_type":"markdown","source":"Here I am choosing only a small amount of data 5000 records to compare the model as comparing with all the data \nwould involve so much time."},{"metadata":{"trusted":true,"_uuid":"4fa6033a6f326712970d7f517533bf75ab815f00"},"cell_type":"code","source":"filepath = '../input/train.csv'\nlabeled_images = pd.read_csv(filepath)\nimages = labeled_images.iloc[:5000, 1:].values.astype('float32')\nlabels = labeled_images.iloc[:5000, :1].values.astype('int32')\n\ntrain_images,test_images, train_labels, test_labels = train_test_split(images, labels, test_size=test_size, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fa5d62a2baae50cb051a0c0c3193a88cba3ad7d4"},"cell_type":"markdown","source":"Checking data for any NULL value and if it has normal distribution or any skewness."},{"metadata":{"trusted":true,"_uuid":"978cbf59cf4be3f11acc09577fccffd813db190d"},"cell_type":"code","source":"sns.countplot(labeled_images['label'])\nprint(labeled_images.isnull().any().any())\nCounter(labeled_images['label'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b81218189cd433c6a268998f2621984617ca3014"},"cell_type":"markdown","source":"Scale the data as we know that ML algorithms performs best when data range is small."},{"metadata":{"trusted":true,"_uuid":"899a5d087a554e6830379bacda8c69ac3461e673"},"cell_type":"code","source":"train_images /= 255.0\ntest_images /= 255.0","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf9848b7c81872afcd601bd2da22d943071e9a78"},"cell_type":"markdown","source":"Plot few images from training data to do sanity check i.e. that labels are marked correctly in training data."},{"metadata":{"trusted":true,"_uuid":"618a2d0d57116e2e3afdb5cc6df3d90817b92031"},"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor i in range(5):\n  plt.subplot(5, 5, i+1)\n  img = train_images[i].reshape(28,28)\n  plt.xticks([])\n  plt.yticks([])\n  plt.imshow(img, cmap='gray')\n  plt.colorbar()\n  plt.title(train_labels[i])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6cb84b3040ef027f6a331e37fbe9143fc7af9652"},"cell_type":"markdown","source":"Initialize all the models you want to check with their default hyperparameters."},{"metadata":{"trusted":true,"_uuid":"1c24c3d223559045981109a606549a1e64b82584"},"cell_type":"code","source":"models = []\nmodels.append(('LR', LogisticRegression()))\nmodels.append(('LDA', LinearDiscriminantAnalysis()))\nmodels.append(('KNN', KNeighborsClassifier()))\nmodels.append(('GB', GaussianNB()))\nmodels.append(('CART', DecisionTreeClassifier()))\nmodels.append(('SVM', SVC()))\nmodels.append(('ExtraTrees', ExtraTreesClassifier()))\nmodels.append(('GradientBoosting', GradientBoostingClassifier()))\nmodels.append(('RandomForest', RandomForestClassifier()))\nresults = []\nnames = []","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e62e325f1fea1e17c8055190812504fc8b20a547"},"cell_type":"markdown","source":"Compare all the models in loop through cross validation:"},{"metadata":{"trusted":true,"_uuid":"cf002220479c8046774cc9895bd27845fb5ee59a"},"cell_type":"code","source":"# for name, model in models:\n#   cv_results = cross_val_score(model, train_images, train_labels.ravel(), cv=kfold)\n#   results.append(cv_results)\n#   names.append(name)\n#   print(name, cv_results.mean(), cv_results.std())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"86338335b848fc1c5f2a043064785672078ba50d"},"cell_type":"markdown","source":"From the result it is clear that KNN, SVM and GradientBoosting models are the winners and are best choosen for further fine tuning the modes.\nHere I have done the fine tuning on KNN and SVM.(yet to do finish experiment on GradientBoosting)."},{"metadata":{"_uuid":"88b7c5e8be82e42c671fbb5bf8ceca8ade353c29"},"cell_type":"markdown","source":"**Fine tuning KNN Hyperparameters using GridSearchCV **"},{"metadata":{"trusted":true,"_uuid":"59090e96f50646066d30751bf67949d067ee90bb"},"cell_type":"code","source":"# k_range = list(range(1,10))\n# weight_options = ['uniform', 'distance']\n# algorithm = ['auto', 'brute']\n# knn_params_grid = dict(n_neighbors=k_range, weights=weight_options)\n\n# def knn_param_selection(X, y, params_grid, num_folds):\n#     knn = KNeighborsClassifier()\n#     knnGrid = GridSearchCV(knn, params_grid, cv=num_folds, n_jobs=-1, scoring='accuracy')\n#     knnGrid.fit(train_images, train_labels.ravel())\n#     print(knnGrid.best_estimator_, knnGrid.best_score_, knnGrid.best_params_)\n    \n# knn_param_selection(train_images, train_labels, knn_params_grid, num_folds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87c1d28a4c3cbcffe25bb589ff0766c60ade0d4f"},"cell_type":"code","source":"#Fine Tuning KNN Hyperparameters\nmodel = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n           metric_params=None, n_jobs=None, n_neighbors=6, p=2,\n           weights='distance')\nfinal_results = cross_val_score(model, train_images, train_labels, cv=kfold)\nprint(\"%s %f %f\" % ('KNN - FineTuned Grid', final_results.mean(), final_results.std()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf07639bd8809ad7fbe892bd19d6e1277d3c471f"},"cell_type":"markdown","source":"Fine tuning SVM Hyperparameters using RandomizedSearchCV. SVM takes much more time than KNN hence used RandomizedSearchCV for discovering hyperparameters. Used three iterations to zero in on final hyperparameters for SVM."},{"metadata":{"trusted":true,"_uuid":"7b725c986d7dc98ac71f3e37cf1ed1b118cb7141"},"cell_type":"code","source":"# kernels = ['rbf']\n# #first iteration params\n# #kernels = ['linear', 'rbf']\n# # Cs = [0.001, 0.01, 0.1, 1, 10]         \n# # gammas = [0.001, 0.01, 0.1, 1]\n# #Second iteration params\n# # Cs = [9, 10, 11, 12, 13, 14, 15]\n# # gammas = [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07]\n# #third iteration params\n# Cs = [13, 14, 15, 16, 17]\n# gammas = [0.018, 0.019, 0.020, 0.021, 0.022 ]\n# svcParams_grid = dict(C=Cs, gamma=gammas, kernel=kernels)\n# def svc_param_selection(X, y, params_grid, num_folds):\n#     svc = SVC()\n#     svcRand = RandomizedSearchCV(svc, svcParams_grid, cv=num_folds, n_jobs=-1, scoring='accuracy')\n#     svcRand.fit(train_images, train_labels.ravel())\n#     print(svcRand.best_estimator_, svcRand.best_score_, svcRand.best_params_)\n\n# svc_param_selection(train_images, train_labels, svcParams_grid, num_folds)\n# #First iteration: {'kernel': 'rbf', 'gamma': 0.01, 'C': 10}  Score: 0.950857\n# #Second iteration: {'kernel': 'rbf', 'gamma': 0.02, 'C': 15}  Score: 0.9588571428571429\n# #Third iteration: {'kernel': 'rbf', 'gamma': 0.022, 'C': 17} Score: 0.9588571428571429 ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5352aa4f95aaad49d19c2a2f1f2016406a51fb0"},"cell_type":"markdown","source":"As can be seen SVM accuracy is same for second and third iteration best hyperameters and hence choose best estimators discovered from second iteration only."},{"metadata":{"trusted":true,"_uuid":"39aa20b1b4ab1a90350c825a377918931d54f3be"},"cell_type":"code","source":"#Fine Tuning SVM Hyperparameters\nsvc = SVC(C=15, cache_size=200, class_weight=None, coef0=0.0,\n  decision_function_shape='ovr', degree=3, gamma=0.022, kernel='rbf',\n  max_iter=-1, probability=False, random_state=None, shrinking=True,\n  tol=0.001, verbose=False)\nfinal_results = cross_val_score(svc, train_images, train_labels, cv=kfold)\nprint(\"%s %f %f\" % ('SVM - FineTuned Random', final_results.mean(), final_results.std()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b9613662d8c5a111cba8a6abaf70748af83a2b0"},"cell_type":"markdown","source":"**Make predictions and Test the predictions**"},{"metadata":{"trusted":true,"_uuid":"ffcaf93fdc0316f3ced765179183f008794558f5"},"cell_type":"code","source":"#Predictions after tuning KNN Hyperparameter\nmodel = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n           metric_params=None, n_jobs=None, n_neighbors=6, p=2,\n           weights='distance')\nmodel.fit(train_images, train_labels)\npredictions = model.predict(test_images)\nprint(\"%s %f\" % ('KNN - Fine Tuned', accuracy_score(test_labels, predictions)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45b8da365d5fc3ada49199926a722dcc4b6a5587"},"cell_type":"code","source":"#Predictions after tuning SVM Hyperparameter\nsvc = SVC(C=15, cache_size=200, class_weight=None, coef0=0.0,\n  decision_function_shape='ovr', degree=3, gamma=0.022, kernel='rbf',\n  max_iter=-1, probability=False, random_state=None, shrinking=True,\n  tol=0.001, verbose=False)\nsvc.fit(train_images, train_labels)\npredictions = svc.predict(test_images)\nprint(\"%s %f\" % ('SVM - Fine Tuned', accuracy_score(test_labels, predictions)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4af65ea67c464b2fc15bec45ab74c2131053e58c"},"cell_type":"markdown","source":"****Submission result - Predictions on provided test data****"},{"metadata":{"trusted":true,"_uuid":"bf41244bdd7ca02374609de2fb12d20474d9ea0d"},"cell_type":"code","source":"filepath = '../input/test.csv'\ntest_data = pd.read_csv(filepath)\ntest_data_images = test_data.iloc[:, :].values.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8e224bea6ce814ef524bc1c79acea1d95d2171c"},"cell_type":"code","source":"test_data_images /= 255.0\ntest_data_images.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9ef7d3017cdede36c91ffffd02ddc7820926375a"},"cell_type":"markdown","source":"Taking full training data for training for predictions instead of partial data taken eararlier:"},{"metadata":{"trusted":true,"_uuid":"06f0685c26f05911cdcd00e1d8e2b28887176b86"},"cell_type":"code","source":"images = labeled_images.iloc[:, 1:].values.astype('float32')\nlabels = labeled_images.iloc[:, :1].values.astype('int32')\nimages /= 255.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67d51296662090e15707345732ea8a4fb7195a43"},"cell_type":"code","source":"# svc = SVC(C=15, cache_size=200, class_weight=None, coef0=0.0,\n#   decision_function_shape='ovr', degree=3, gamma=0.022, kernel='rbf',\n#   max_iter=-1, probability=False, random_state=None, shrinking=True,\n#   tol=0.001, verbose=False)\n# svc.fit(images, labels)\n# results = svc.predict(test_data_images)\n# df = pd.DataFrame(results)\n# df.index.name='ImageId'\n# df.index+=1\n# df.columns=['Label']\n# df.to_csv('results.csv', header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"603684f1c966b0cc5808f68a95f2fea551abe880"},"cell_type":"code","source":"#Trying to create an ensemble of few best performing \nfrom sklearn.ensemble import VotingClassifier\nlr_clf  = LogisticRegression()\nknn_clf = KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',\n           metric_params=None, n_jobs=None, n_neighbors=6, p=2,\n           weights='distance')\nsvm_clf = SVC(C=15, cache_size=200, class_weight=None, coef0=0.0,\n  decision_function_shape='ovr', degree=3, gamma=0.022, kernel='rbf',\n  max_iter=-1, random_state=None, shrinking=True,\n  tol=0.001, verbose=False, probability=True)\nextree_clf =  ExtraTreesClassifier()\n\n\nvoting_clf = VotingClassifier(\n              estimators= [('lr', lr_clf), ('knn', knn_clf), ('svm', svm_clf), ('extree', extree_clf)],\n              voting='soft')\nvoting_clf.fit(images, labels)\nresults = voting_clf.predict(test_data_images)\ndf = pd.DataFrame(results)\ndf.index.name='ImageId'\ndf.index+=1\ndf.columns=['Label']\ndf.to_csv('results.csv', header=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ae655870cabe7e3e50f588c6f082b2e6a43b551"},"cell_type":"markdown","source":"Doing a sanity check by plotting some 25 images and seeing that predicitions make sense."},{"metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"trusted":true,"_uuid":"96a7c2d0376a1fecca028a285fe2bdbd0f22f26e"},"cell_type":"code","source":"plt.figure(figsize=(15,15))\nfor i in range(25):\n  plt.subplot(5, 5, i+1)\n  img = test_data_images[i].reshape(28,28)\n  plt.xticks([])\n  plt.yticks([])\n  plt.imshow(img, cmap='gray')\n  plt.colorbar()\n  plt.title(results[i])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0836809fdc492cc40bd69d41a9e58f0905a0872"},"cell_type":"markdown","source":"Checking distribution of predicitons:"},{"metadata":{"trusted":true,"_uuid":"d42469a2fbdfefca3d31ae68a774d9f1765b3b4d"},"cell_type":"code","source":"sns.countplot(df['Label'])\nCounter(df['Label'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}