{"cells":[{"metadata":{},"cell_type":"markdown","source":"Original [kernel](https://github.com/Yorko/mlcourse.ai/blob/master/jupyter_russian/topic05_bagging_rf/topic5_part2_random_forest.ipynb)"},{"metadata":{"trusted":true},"cell_type":"code","source":"from __future__ import division, print_function\n# отключим всякие предупреждения Anaconda\nimport warnings\nimport numpy as np\nimport pandas as pd\nwarnings.filterwarnings('ignore')\n%matplotlib inline\nfrom matplotlib import pyplot as plt\nplt.style.use('ggplot')\nplt.rcParams['figure.figsize'] = 10, 6\nimport seaborn as sns\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import GridSearchCV","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/Kannada-MNIST/train.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"images = df_train.drop(\"label\", axis=1).values.astype('float32')\nlabels = df_train['label'].values.astype('float32')\ndel df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_csv('../input/Kannada-MNIST/test.csv')\ndf_test = df_test.drop(\"id\", axis=1).values.astype('float32')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Random Forest"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize the stratified breakdown of our dataset for validation\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize our classifier with default parameters\nrfc = RandomForestClassifier(random_state=42, n_jobs=-1, oob_score=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# teach on a training dataset\nresults = cross_val_score(rfc, images, labels, cv=skf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# evaluate the accuracy of the train dataset\nprint(\"CV accuracy score: {:.2f}%\".format(results.mean()*100))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Let's try to improve this result and see how the learning curves behave when changing the basic parameters.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initialize validation\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create lists to maintain accuracy on the training and test dataset\ntrain_acc = []\ntest_acc = []\ntemp_train_acc = []\ntemp_test_acc = []\ntrees_grid = [5, 10, 15, 20, 30, 50, 75, 100]\n\n# Teach on a training dataset\nfor ntrees in trees_grid:\n    rfc = RandomForestClassifier(n_estimators=ntrees, random_state=42, n_jobs=-1, oob_score=True)\n    temp_train_acc = []\n    temp_test_acc = []\n    for train_index, test_index in skf.split(images, labels):\n        X_train, X_test = images[train_index], images[test_index]\n        y_train, y_test = labels[train_index], labels[test_index]\n        rfc.fit(X_train, y_train)\n        temp_train_acc.append(rfc.score(X_train, y_train))\n        temp_test_acc.append(rfc.score(X_test, y_test))\n    train_acc.append(temp_train_acc)\n    test_acc.append(temp_test_acc)\n    \ntrain_acc, test_acc = np.asarray(train_acc), np.asarray(test_acc)\nprint(\"Best accuracy on CV is {:.2f}% with {} trees\".format(max(test_acc.mean(axis=1))*100, \n                                                        trees_grid[np.argmax(test_acc.mean(axis=1))]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('ggplot')\n\nfig, ax = plt.subplots(figsize=(8, 4))\nax.plot(trees_grid, train_acc.mean(axis=1), alpha=0.5, color='blue', label='train')\nax.plot(trees_grid, test_acc.mean(axis=1), alpha=0.5, color='red', label='cv')\nax.fill_between(trees_grid, test_acc.mean(axis=1) - test_acc.std(axis=1), test_acc.mean(axis=1) + test_acc.std(axis=1), color='#888888', alpha=0.4)\nax.fill_between(trees_grid, test_acc.mean(axis=1) - 2*test_acc.std(axis=1), test_acc.mean(axis=1) + 2*test_acc.std(axis=1), color='#888888', alpha=0.2)\nax.legend(loc='best')\nax.set_ylim([0.88,1.02])\nax.set_ylabel(\"Accuracy\")\nax.set_xlabel(\"N_estimators\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**As you can see, when a certain number of trees is reached, our accuracy on the test reaches the asymptote, and you can decide for yourself how many trees are optimal for your task. The figure also shows that in the training sample we were able to achieve 100% accuracy, this tells us about the retraining of our model. To avoid overtraining, we must add regularization parameters to the model.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Create lists to maintain accuracy on the training and test dataset\ntrain_acc = []\ntest_acc = []\ntemp_train_acc = []\ntemp_test_acc = []\nmax_depth_grid = [3, 5, 7, 9, 11, 13, 15, 17, 20, 22, 24]\n\n# Teach on a training dataset\nfor max_depth in max_depth_grid:\n    rfc = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1, oob_score=True, max_depth=max_depth)\n    temp_train_acc = []\n    temp_test_acc = []\n    for train_index, test_index in skf.split(images, labels):\n        X_train, X_test = images[train_index], images[test_index]\n        y_train, y_test = labels[train_index], labels[test_index]\n        rfc.fit(X_train, y_train)\n        temp_train_acc.append(rfc.score(X_train, y_train))\n        temp_test_acc.append(rfc.score(X_test, y_test))\n    train_acc.append(temp_train_acc)\n    test_acc.append(temp_test_acc)\n    \ntrain_acc, test_acc = np.asarray(train_acc), np.asarray(test_acc)\nprint(\"Best accuracy on CV is {:.2f}% with {} max_depth\".format(max(test_acc.mean(axis=1))*100, \n                                                        max_depth_grid[np.argmax(test_acc.mean(axis=1))]))\n\nfig, ax = plt.subplots(figsize=(8, 4))\nax.plot(max_depth_grid, train_acc.mean(axis=1), alpha=0.5, color='blue', label='train')\nax.plot(max_depth_grid, test_acc.mean(axis=1), alpha=0.5, color='red', label='cv')\nax.fill_between(max_depth_grid, test_acc.mean(axis=1) - test_acc.std(axis=1), test_acc.mean(axis=1) + test_acc.std(axis=1), color='#888888', alpha=0.4)\nax.fill_between(max_depth_grid, test_acc.mean(axis=1) - 2*test_acc.std(axis=1), test_acc.mean(axis=1) + 2*test_acc.std(axis=1), color='#888888', alpha=0.2)\nax.legend(loc='best')\nax.set_ylim([0.88,1.02])\nax.set_ylabel(\"Accuracy\")\nax.set_xlabel(\"Max_depth\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**The max_depth parameter does a good job of regularizing the model, and we are not so overtrained. The accuracy of our model has increased slightly.**\n\n**Another important parameter min_samples_leaf, it also serves as a regularizer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create lists to maintain accuracy on the training and test dataset\ntrain_acc = []\ntest_acc = []\ntemp_train_acc = []\ntemp_test_acc = []\nmin_samples_leaf_grid = [1, 3, 5, 7, 9, 11, 13, 15, 17, 20, 22, 24]\n\n# Teach on a training dataset\nfor min_samples_leaf in min_samples_leaf_grid:\n    rfc = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1, \n                                 oob_score=True, min_samples_leaf=min_samples_leaf)\n    temp_train_acc = []\n    temp_test_acc = []\n    for train_index, test_index in skf.split(images, labels):\n        X_train, X_test = images[train_index], images[test_index]\n        y_train, y_test = labels[train_index], labels[test_index]\n        rfc.fit(X_train, y_train)\n        temp_train_acc.append(rfc.score(X_train, y_train))\n        temp_test_acc.append(rfc.score(X_test, y_test))\n    train_acc.append(temp_train_acc)\n    test_acc.append(temp_test_acc)\n    \ntrain_acc, test_acc = np.asarray(train_acc), np.asarray(test_acc)\nprint(\"Best accuracy on CV is {:.2f}% with {} min_samples_leaf\".format(max(test_acc.mean(axis=1))*100, \n                                                        min_samples_leaf_grid[np.argmax(test_acc.mean(axis=1))]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 4))\nax.plot(min_samples_leaf_grid, train_acc.mean(axis=1), alpha=0.5, color='blue', label='train')\nax.plot(min_samples_leaf_grid, test_acc.mean(axis=1), alpha=0.5, color='red', label='cv')\nax.fill_between(min_samples_leaf_grid, test_acc.mean(axis=1) - test_acc.std(axis=1), test_acc.mean(axis=1) + test_acc.std(axis=1), color='#888888', alpha=0.4)\nax.fill_between(min_samples_leaf_grid, test_acc.mean(axis=1) - 2*test_acc.std(axis=1), test_acc.mean(axis=1) + 2*test_acc.std(axis=1), color='#888888', alpha=0.2)\nax.legend(loc='best')\nax.set_ylim([0.88,1.02])\nax.set_ylabel(\"Accuracy\")\nax.set_xlabel(\"Min_samples_leaf\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**In this case, we do not gain in accuracy on validation, but we can greatly reduce retraining while maintaining accuracy.**\n\n**Consider such a parameter as max_features. For classification problems, the default is $\\large \\sqrt{n}$, where n is the number of features. Let's check whether it is optimal in our case to use 4 signs or not.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create lists to maintain accuracy on the training and test dataset\ntrain_acc = []\ntest_acc = []\ntemp_train_acc = []\ntemp_test_acc = []\nmax_features_grid = [2, 4, 6, 8, 10, 12, 14, 16]\n\n# Teach on a training dataset\nfor max_features in max_features_grid:\n    rfc = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1, \n                                 oob_score=True, max_features=max_features)\n    temp_train_acc = []\n    temp_test_acc = []\n    for train_index, test_index in skf.split(images, labels):\n        X_train, X_test = images[train_index], images[test_index]\n        y_train, y_test = labels[train_index], labels[test_index]\n        rfc.fit(X_train, y_train)\n        temp_train_acc.append(rfc.score(X_train, y_train))\n        temp_test_acc.append(rfc.score(X_test, y_test))\n    train_acc.append(temp_train_acc)\n    test_acc.append(temp_test_acc)\n    \ntrain_acc, test_acc = np.asarray(train_acc), np.asarray(test_acc)\nprint(\"Best accuracy on CV is {:.2f}% with {} max_features\".format(max(test_acc.mean(axis=1))*100, \n                                                        max_features_grid[np.argmax(test_acc.mean(axis=1))]))\n\nfig, ax = plt.subplots(figsize=(8, 4))\nax.plot(max_features_grid, train_acc.mean(axis=1), alpha=0.5, color='blue', label='train')\nax.plot(max_features_grid, test_acc.mean(axis=1), alpha=0.5, color='red', label='cv')\nax.fill_between(max_features_grid, test_acc.mean(axis=1) - test_acc.std(axis=1), test_acc.mean(axis=1) + test_acc.std(axis=1), color='#888888', alpha=0.4)\nax.fill_between(max_features_grid, test_acc.mean(axis=1) - 2*test_acc.std(axis=1), test_acc.mean(axis=1) + 2*test_acc.std(axis=1), color='#888888', alpha=0.2)\nax.legend(loc='best')\nax.set_ylim([0.88,1.02])\nax.set_ylabel(\"Accuracy\")\nax.set_xlabel(\"Max_features\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**In our case, the optimal number of features is 16, it is with this value that the best result is achieved.**\n\n**We looked at how learning curves behave depending on changes in the basic parameters. Now let's use Grid Search to find the optimal parameters for our example.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Сделаем инициализацию параметров, по которым хотим сделать полный перебор\nparameters = {'max_features': [4, 7, 10, 13], 'min_samples_leaf': [1, 3, 5, 7], 'max_depth': [5,10,15,20]}\nrfc = RandomForestClassifier(n_estimators=100, random_state=42, \n                             n_jobs=-1, oob_score=True)\ngcv = GridSearchCV(rfc, parameters, n_jobs=-1, cv=skf, verbose=1)\ngcv.fit(images, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gcv.best_estimator_, gcv.best_score_","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**The best accuracy that we were able to achieve with brute force is 97.66% with 'max_depth': 20,' max_features': 13,' min_samples_leaf': 1.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"rfc = RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n                        max_depth=20, max_features=13, max_leaf_nodes=None,\n                        min_impurity_decrease=0.0, min_impurity_split=None,\n                        min_samples_leaf=1, min_samples_split=2,\n                        min_weight_fraction_leaf=0.0, n_estimators=100,\n                        n_jobs=-1, oob_score=True, random_state=42, verbose=0,\n                        warm_start=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rfc.fit(images, labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = rfc.predict(df_test).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def write_preds(preds, fname):\n    pd.DataFrame({\"id\": list(range(0,len(preds))), \"label\": preds}).to_csv(fname, index=False, header=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"write_preds(pred, \"samplesubmission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}