{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Digit Classifier\n    Problem statement: Predict digit value given handwriting picture (28 x 28 pixel)\n    Type: Classification\n    Performance metric: Accuracy\n\n    What I did in notebook:\n    1. Data pre-processing: Randomization\n    2. Feature engineering: Data Augmentation\n    3. Modelling: (a) Multiclass Classifier VS Ensemble Binary Classifier, (b) Measure improvement in data augmentation"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# Library\nimport numpy as np # linear algebra\nimport pandas as pd # data processing\nimport matplotlib as mpl # data visualization\nimport matplotlib.pyplot as plt # data visualization\nfrom sklearn.linear_model import LogisticRegression # logistic regression\nfrom sklearn.linear_model import RidgeClassifier # ridge classifier\nfrom sklearn.ensemble import RandomForestClassifier # random forest\nfrom sklearn.ensemble import VotingClassifier # Ensemble model\nfrom sklearn.model_selection import StratifiedKFold # Stratified K-Fold for cross-validation\nfrom sklearn.model_selection import KFold # Stratified K-Fold for cross-validation\nfrom sklearn.model_selection import GridSearchCV # to perform Grid search cross-validation \nfrom sklearn.model_selection import cross_val_score # to calculate cross-validation score\nfrom scipy.ndimage.interpolation import shift # shift image for data augmentation\nfrom sklearn import metrics\nimport random\nimport time\nfrom datetime import datetime\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape, test.shape\n# There are 42k training data and 28k test data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get sense in target value\n\ntrain['label'].value_counts()\n# Each label has more than 3.7k samples\n# Smallest sample is digit 5\n# Biggest samples is digit 1\n# We will make sure do sratified cross-validation since # of samples per label different","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define Functions\n\n# Series to Array\ndef series_to_array(series):\n    data = np.array(series)\n    return data\n\n# Plot Digit\ndef plot_digit(series):\n    data = series_to_array(series)\n    image = data.reshape(28, 28)\n    plt.imshow(image, cmap = mpl.cm.binary,\n               interpolation=\"nearest\")\n    plt.axis(\"off\")\n    \n# Plot Multiple Digits\ndef plot_digits(instances, images_per_row=10, **options):\n    size = 28\n    images_per_row = min(len(instances), images_per_row)\n    images = [instance.reshape(size,size) for instance in instances]\n    n_rows = (len(instances) - 1) // images_per_row + 1\n    row_images = []\n    n_empty = n_rows * images_per_row - len(instances)\n    images.append(np.zeros((size, size * n_empty)))\n    for row in range(n_rows):\n        rimages = images[row * images_per_row : (row + 1) * images_per_row]\n        row_images.append(np.concatenate(rimages, axis=1))\n    image = np.concatenate(row_images, axis=0)\n    plt.imshow(image, cmap = mpl.cm.binary, **options)\n    plt.axis(\"off\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot one random digit\nplot_digit(train.iloc[37694,1:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot multiple random digits\nplt.figure(figsize=(9,9))\nexample_images = np.r_[test[:12000:600], test[13000:30600:600], test[30600:60000:590]]\nplot_digits(example_images, images_per_row=10)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Modelling\n### Multiclass Classifier VS Ensemble Binary Classifier"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prepare Train and Test dataset\nX_train = pd.DataFrame(train.iloc[:, 1:])\ny_train = pd.DataFrame(train.iloc[:, 0]) # temporarily convert to DataFrame for randomized process\nX_test = test\n\nX_train.shape, y_train.shape, X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Randomized Train dataset\nrandom.seed(51)\nshuffle_index = np.arange(42000)\nrandom.shuffle(shuffle_index)\nX_train, y_train = X_train.iloc[shuffle_index], y_train.iloc[shuffle_index]\n\ny_train = y_train.iloc[:,0] # Revert back to series\n\n# Dataset ready to process","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stratified k-fold being used in order to get stratified split\nskfolds = StratifiedKFold(n_splits=2, random_state=41)\n\n# K-fold cross-validation\nkfolds = KFold(n_splits=2, random_state=41)\n\n# For simplicity in measuring performance\ndef cv_acc(model, X=X_train, y=y_train):\n    acc = cross_val_score(model, X, y_train, cv=3, scoring = \"accuracy\")\n    return (acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Multiclass Classifier\n# I will be using three Multiclass Classifier i.e. (1) Logistic Regression, (2) Ridge Classifier, and (3) Random Forest\n# Define Multiclass Classifier\n\nlogistic_multi = LogisticRegression(multi_class='multinomial', solver = 'saga', penalty = 'l1') # define as multiclass classifier\nridge_multi = RidgeClassifier(alpha = 0.7)\nrf_multi = RandomForestClassifier(n_estimators = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Do Cross Validation for MultiClass Classifier\nscore_log_multi = cv_acc(logistic_multi , X_train)\nscore_ridge_multi = cv_acc(ridge_multi , X_train)\nscore_rf_multi = cv_acc(rf_multi , X_train)\nprint(\"Finished!\\n\", datetime.now(), )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print Accuracy estimated using CV\nprint(\"Logistic MultiClass Accuracy: {:.4f} (with STD {:.4f})\\n\".format(score_log_multi.mean(), score_log_multi.std()))\nprint(\"Ridge MultiClass Accuracy: {:.4f} (with STD {:.4f})\\n\".format(score_ridge_multi.mean(), score_ridge_multi.std()))\nprint(\"Random Forest MultiClass Accuracy: {:.4f} (with STD {:.4f})\\n\".format(score_rf_multi.mean(), score_rf_multi.std()))\n\n# Random Forest & Logistic Multiclass give > 90% accuracy\n# Let's compare it with Ensemble Binary Classifier which consist of multiple Random Forest binary classifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Ensemble Binary Classifier\n\n# Let's create target for binary classification\ny_train_0 = (y_train == 0)\ny_train_1 = (y_train == 1)\ny_train_2 = (y_train == 2)\ny_train_3 = (y_train == 3)\ny_train_4 = (y_train == 4)\ny_train_5 = (y_train == 5)\ny_train_6 = (y_train == 6)\ny_train_7 = (y_train == 7)\ny_train_8 = (y_train == 8)\ny_train_9 = (y_train == 9) # Now we have separated target for each digit\n\n# Lets define Binary Classifier for each digit\nrf_0 = RandomForestClassifier(n_estimators = 10)\nrf_1 = RandomForestClassifier(n_estimators = 10)\nrf_2 = RandomForestClassifier(n_estimators = 10)\nrf_3 = RandomForestClassifier(n_estimators = 10)\nrf_4 = RandomForestClassifier(n_estimators = 10)\nrf_5 = RandomForestClassifier(n_estimators = 10)\nrf_6 = RandomForestClassifier(n_estimators = 10)\nrf_7 = RandomForestClassifier(n_estimators = 10)\nrf_8 = RandomForestClassifier(n_estimators = 10)\nrf_9 = RandomForestClassifier(n_estimators = 10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Estimate accuracy of each model using CV\nscore_rf_0 = cv_acc(rf_0 , X_train, y_train_0)\nscore_rf_1 = cv_acc(rf_1 , X_train, y_train_1)\nscore_rf_2 = cv_acc(rf_2 , X_train, y_train_2)\nscore_rf_3 = cv_acc(rf_3 , X_train, y_train_3)\nscore_rf_4 = cv_acc(rf_4 , X_train, y_train_4)\nscore_rf_5 = cv_acc(rf_5 , X_train, y_train_5)\nscore_rf_6 = cv_acc(rf_6 , X_train, y_train_6)\nscore_rf_7 = cv_acc(rf_7 , X_train, y_train_7)\nscore_rf_8 = cv_acc(rf_8 , X_train, y_train_8)\nscore_rf_9 = cv_acc(rf_9 , X_train, y_train_9)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print Accuracy estimated using CV\nprint(\"Random Forest 0: {:.4f} (with STD {:.4f})\\n\".format(score_rf_0.mean(), score_rf_0.std()))\nprint(\"Random Forest 1: {:.4f} (with STD {:.4f})\\n\".format(score_rf_1.mean(), score_rf_1.std()))\nprint(\"Random Forest 2: {:.4f} (with STD {:.4f})\\n\".format(score_rf_2.mean(), score_rf_2.std()))\nprint(\"Random Forest 3: {:.4f} (with STD {:.4f})\\n\".format(score_rf_3.mean(), score_rf_3.std()))\nprint(\"Random Forest 4: {:.4f} (with STD {:.4f})\\n\".format(score_rf_4.mean(), score_rf_4.std()))\nprint(\"Random Forest 5: {:.4f} (with STD {:.4f})\\n\".format(score_rf_5.mean(), score_rf_5.std()))\nprint(\"Random Forest 6: {:.4f} (with STD {:.4f})\\n\".format(score_rf_6.mean(), score_rf_6.std()))\nprint(\"Random Forest 7: {:.4f} (with STD {:.4f})\\n\".format(score_rf_7.mean(), score_rf_7.std()))\nprint(\"Random Forest 8: {:.4f} (with STD {:.4f})\\n\".format(score_rf_8.mean(), score_rf_8.std()))\nprint(\"Random Forest 9: {:.4f} (with STD {:.4f})\\n\".format(score_rf_9.mean(), score_rf_9.std()))\n\n# Great! Estimators have roughly 93% accuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit binary classifier\nrf_0.fit(X_train, y_train_0)\nrf_1.fit(X_train, y_train_1)\nrf_2.fit(X_train, y_train_2)\nrf_3.fit(X_train, y_train_3)\nrf_4.fit(X_train, y_train_4)\nrf_5.fit(X_train, y_train_5)\nrf_6.fit(X_train, y_train_6)\nrf_7.fit(X_train, y_train_7)\nrf_8.fit(X_train, y_train_8)\nrf_9.fit(X_train, y_train_9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Vote Classifier\n# Using vote classifier as method to combine binary classifier\n# Soft Voting will be used as we want the average class probabilities from each estimator\n\nbinary_classifiers = [('rf_0', rf_0), ('rf_1', rf_1), ('rf_2', rf_2), ('rf_3', rf_3), ('rf_4', rf_4), \n                      ('rf_5', rf_5), ('rf_6', rf_6), ('rf_7', rf_7), ('rf_8', rf_8), ('rf_9', rf_9)]\n\nvoting_cl = VotingClassifier( estimators = binary_classifiers, voting = 'soft')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Estimate accuracy of Voting Classifier\nscore_voting = cv_acc(voting_cl , X_train, y_train)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print estimated accuracy of Voting Classifier\nprint(\"Voting Classifier: {:.4f} (with STD {:.4f})\\n\".format(score_voting.mean(), score_voting.std()))\n\n# Significant improvement! We got estimation of 96.3% accuracy!","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Lets use this Voting Classifier for submission\n\nvoting_cl.fit(X_train, y_train)\ny_predicted = voting_cl.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Submission\nsubmission1 = pd.read_csv(\"../input/sample_submission.csv\")\nsubmission1.iloc[:,1] = (y_predicted)\nsubmission1.to_csv(\"submission1.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Augmentation\nGoal of data augmentation is to get additional number of samples using available data. Adding samples hopefully will increase model performance. How we will get additional samples? By shifting available images."},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_digit(series):\n    data = series_to_array(series)\n    image = data.reshape(28, 28)\n    plt.imshow(image, cmap = mpl.cm.binary,\n               interpolation=\"nearest\")\n    plt.axis(\"off\")\n\ndef shift_image(series, dx, dy):\n    data = series_to_array(series)\n    image = data.reshape((28, 28))\n    shifted_image = shift(image, [dy, dx], cval=0, mode=\"constant\")\n    return shifted_image.reshape([-1])\nimage = X_train.iloc[1050]\nshifted_image_down = shift_image(image, 0, 5)\nshifted_image_left = shift_image(image, -5, 0)\n\nplt.figure(figsize=(12,3))\nplt.subplot(131)\nplt.title(\"Original\", fontsize=14)\nplt.imshow(series_to_array(image).reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\nplt.subplot(132)\nplt.title(\"Shifted down\", fontsize=14)\nplt.imshow(series_to_array(shifted_image_down).reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\nplt.subplot(133)\nplt.title(\"Shifted left\", fontsize=14)\nplt.imshow(series_to_array(shifted_image_left).reshape(28, 28), interpolation=\"nearest\", cmap=\"Greys\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create augmented data\n\ndef shift_image2(image, dx, dy):\n    image = image.reshape((28, 28))\n    shifted_image = shift(image, [dy, dx], cval=0, mode=\"constant\")\n    return shifted_image.reshape([-1])\n\nX_train_array = X_train.values # Convert DataFrame into Array\n\nX_train_augmented = [ images for images in X_train_array]\ny_train_augmented = [ labels for labels in y_train]\n\n# We will create shifted left and right images by 1 pixel\nfor dx, dy in ((1, 0), (-1, 0)):\n    for images, labels in zip(X_train_array, y_train):\n        X_train_augmented.append(shift_image2(images, dx, dy))\n        y_train_augmented.append(labels)\n\nX_train_augmented = np.array(X_train_augmented)\ny_train_augmented = np.array(y_train_augmented)\n\nshuffle_idx = np.random.permutation(len(X_train_augmented))\nX_train_augmented = X_train_augmented[shuffle_idx]\ny_train_augmented = y_train_augmented[shuffle_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_augmented.shape, y_train_augmented.shape\n# We got 126000 samples!\n# Lets try to estimate accuracy using augmented dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's create target for binary classification\ny_train_0_agm = (y_train_augmented == 0)\ny_train_1_agm = (y_train_augmented == 1)\ny_train_2_agm = (y_train_augmented == 2)\ny_train_3_agm = (y_train_augmented == 3)\ny_train_4_agm = (y_train_augmented == 4)\ny_train_5_agm = (y_train_augmented == 5)\ny_train_6_agm = (y_train_augmented == 6)\ny_train_7_agm = (y_train_augmented == 7)\ny_train_8_agm = (y_train_augmented == 8)\ny_train_9_agm = (y_train_augmented == 9) # Now we have separated target for each digit\n\n# Create Binary Classifier\nrf_0_agm = RandomForestClassifier(n_estimators = 10)\nrf_1_agm = RandomForestClassifier(n_estimators = 10)\nrf_2_agm = RandomForestClassifier(n_estimators = 10)\nrf_3_agm = RandomForestClassifier(n_estimators = 10)\nrf_4_agm = RandomForestClassifier(n_estimators = 10)\nrf_5_agm = RandomForestClassifier(n_estimators = 10)\nrf_6_agm = RandomForestClassifier(n_estimators = 10)\nrf_7_agm = RandomForestClassifier(n_estimators = 10)\nrf_8_agm = RandomForestClassifier(n_estimators = 10)\nrf_9_agm = RandomForestClassifier(n_estimators = 10)\n\n# Fit Binary Classifier\nprint(\"Started! We are now building binary classifiers!\\n\", datetime.now(), )\nrf_0_agm.fit(X_train_augmented, y_train_0_agm)\nrf_1_agm.fit(X_train_augmented, y_train_1_agm)\nrf_2_agm.fit(X_train_augmented, y_train_2_agm)\nrf_3_agm.fit(X_train_augmented, y_train_3_agm)\nrf_4_agm.fit(X_train_augmented, y_train_4_agm)\nrf_5_agm.fit(X_train_augmented, y_train_5_agm)\nrf_6_agm.fit(X_train_augmented, y_train_6_agm)\nrf_7_agm.fit(X_train_augmented, y_train_7_agm)\nrf_8_agm.fit(X_train_augmented, y_train_8_agm)\nrf_9_agm.fit(X_train_augmented, y_train_9_agm)\nprint(\"Finished! binary classifiers created!\\n\", datetime.now(), )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Vote Classifier\nbinary_classifiers_2 = [('rf_0_agm', rf_0_agm), ('rf_1_agm', rf_1_agm), ('rf_2_agm', rf_2_agm), ('rf_3_agm', rf_3_agm), ('rf_4_agm', rf_4_agm), \n                      ('rf_5_agm', rf_5_agm), ('rf_6_agm', rf_6_agm), ('rf_7_agm', rf_7_agm), ('rf_8_agm', rf_8_agm), ('rf_9_agm', rf_9_agm)]\n\nvoting_cl_agm= VotingClassifier( estimators = binary_classifiers_2, voting = 'soft')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Estimate accuracy of Voting Classifier\nscore_voting_agm = cross_val_score(voting_cl_agm, X_train_augmented, y_train_augmented, cv=2, scoring = \"accuracy\")\nprint(\"Finished! cross-validation complete!\\n\", datetime.now(), )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print estimated accuracy of Voting Classifier\nprint(\"Voting Classifier - Augmented: {:.4f} (with STD {:.4f})\\n\".format(score_voting_agm.mean(), score_voting_agm.std()))\n\n# We got 96.74% Accuracy! Our model slightly improved with augmented data.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit Voting Classifier with augmented data\nvoting_cl_agm.fit(X_train_augmented, y_train_augmented)\ny_predicted_2 = voting_cl_agm.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Submission 2\nsubmission2 = pd.read_csv(\"../input/sample_submission.csv\")\nsubmission2.iloc[:,1] = (y_predicted_2)\nsubmission2.to_csv(\"submission2.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Turns out shifting our images to the left and right works to improve accuracy\n# We will try to create more shifted left and right by 2 pixel\n\n# We will create shifted left and right images by 2 pixel\n\nX_train_augmented_2 = [ images for images in X_train_array]\ny_train_augmented_2 = [ labels for labels in y_train]\n\nfor dx, dy in ((2, 0), (-2, 0), (1,0), (-1,0)):\n    for images, labels in zip(X_train_array, y_train):\n        X_train_augmented_2.append(shift_image2(images, dx, dy))\n        y_train_augmented_2.append(labels)\n\nX_train_augmented_2 = np.array(X_train_augmented_2)\ny_train_augmented_2 = np.array(y_train_augmented_2)\n\nshuffle_idx = np.random.permutation(len(X_train_augmented_2))\nX_train_augmented_2 = X_train_augmented_2[shuffle_idx]\ny_train_augmented_2 = y_train_augmented_2[shuffle_idx]\n\nX_train_augmented_2.shape, y_train_augmented_2.shape\n# Now we have 210000 rows data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Let's create target for binary classification\ny_train_0_agm_2 = (y_train_augmented_2 == 0)\ny_train_1_agm_2 = (y_train_augmented_2 == 1)\ny_train_2_agm_2 = (y_train_augmented_2 == 2)\ny_train_3_agm_2 = (y_train_augmented_2 == 3)\ny_train_4_agm_2 = (y_train_augmented_2 == 4)\ny_train_5_agm_2 = (y_train_augmented_2 == 5)\ny_train_6_agm_2 = (y_train_augmented_2 == 6)\ny_train_7_agm_2 = (y_train_augmented_2 == 7)\ny_train_8_agm_2 = (y_train_augmented_2 == 8)\ny_train_9_agm_2 = (y_train_augmented_2 == 9) # Now we have separated target for each digit\n\n# Create Binary Classifier\nrf_0_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_1_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_2_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_3_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_4_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_5_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_6_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_7_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_8_agm_2 = RandomForestClassifier(n_estimators = 10)\nrf_9_agm_2 = RandomForestClassifier(n_estimators = 10)\n\n# Fit Binary Classifier\nprint(\"Started! We are now building binary classifiers!\\n\", datetime.now(), )\nrf_0_agm_2.fit(X_train_augmented_2, y_train_0_agm_2)\nrf_1_agm_2.fit(X_train_augmented_2, y_train_1_agm_2)\nrf_2_agm_2.fit(X_train_augmented_2, y_train_2_agm_2)\nrf_3_agm_2.fit(X_train_augmented_2, y_train_3_agm_2)\nrf_4_agm_2.fit(X_train_augmented_2, y_train_4_agm_2)\nrf_5_agm_2.fit(X_train_augmented_2, y_train_5_agm_2)\nrf_6_agm_2.fit(X_train_augmented_2, y_train_6_agm_2)\nrf_7_agm_2.fit(X_train_augmented_2, y_train_7_agm_2)\nrf_8_agm_2.fit(X_train_augmented_2, y_train_8_agm_2)\nrf_9_agm_2.fit(X_train_augmented_2, y_train_9_agm_2)\nprint(\"Finished! binary classifiers created!\\n\", datetime.now(), )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Vote Classifier\nbinary_classifiers_3 = [('rf_0_agm_2', rf_0_agm_2), ('rf_1_agm_2', rf_1_agm_2), ('rf_2_agm_2', rf_2_agm_2), ('rf_3_agm_2', rf_3_agm_2), ('rf_4_agm_2', rf_4_agm_2), \n                      ('rf_5_agm_2', rf_5_agm_2), ('rf_6_agm_2', rf_6_agm_2), ('rf_7_agm_2', rf_7_agm_2), ('rf_8_agm_2', rf_8_agm_2), ('rf_9_agm_2', rf_9_agm_2)]\n\nvoting_cl_agm_2= VotingClassifier( estimators = binary_classifiers_3, voting = 'soft')\n\n# Estimate accuracy of Voting Classifier\nprint(\"Started! cross-validation start!\\n\", datetime.now(), )\nscore_voting_agm_2 = cross_val_score(voting_cl_agm_2, X_train_augmented_2, y_train_augmented_2, cv=2, scoring = \"accuracy\")\nprint(\"Finished! cross-validation complete!\\n\", datetime.now(), )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print estimated accuracy of Voting Classifier\nprint(\"Voting Classifier - Augmented 2: {:.4f} (with STD {:.4f})\\n\".format(score_voting_agm_2.mean(), score_voting_agm_2.std()))\n\n# We got 96.78% Accuracy! 0.02% improvement. Turns out at this level, additional augmented data not much improve our performance","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit Voting Classifier with augmented data - 2\nvoting_cl_agm_2.fit(X_train_augmented_2, y_train_augmented_2)\ny_predicted_3 = voting_cl_agm_2.predict(X_test)\n\n# Submission 3\nsubmission3 = pd.read_csv(\"../input/sample_submission.csv\")\nsubmission3.iloc[:,1] = (y_predicted_3)\nsubmission3.to_csv(\"submission3.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}