{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Visualize the Level of Randomness in the Random Forest Algo\n### A case study with the Categorical Feature Encoding Challenge II Dataset\n\n\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\n%matplotlib inline\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import rcParams\nfrom tqdm import tqdm_notebook\nfrom sklearn.tree import _tree, export_graphviz\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.preprocessing import OneHotEncoder, LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(file_path):\n    print('Loading datasets...')\n    X_train = pd.read_csv(file_path + 'train.csv', sep=',')\n    print('Datasets loaded')\n    return X_train\nPATH = '../input/cat-in-the-dat-ii/'\nX_train = read_data(PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head(25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = X_train.sample(n=50000, replace=False, random_state=42, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = X_train['target']\ndel X_train['target']\ndel X_train['id']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Missing Value Approaches\nprint('Missing Value Approaches')\ncolumns = X_train.columns\nfor cc in tqdm_notebook(columns):\n    X_train[cc] = X_train[cc].fillna(X_train[cc].mode()[0])\n\n#Label Encoding\nprint('Label Encoding')\nfor cc in tqdm_notebook(columns):\n    le = LabelEncoder()\n    le.fit(X_train[cc].values)\n    X_train[cc] = le.transform(X_train[cc].values)\n\n#OneHotEncoding\nprint('OneHotEncoding')\nOHE = OneHotEncoder(dtype='uint16', handle_unknown=\"ignore\")\nOHE.fit(X_train)\nX_train = OHE.transform(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def leaf__depths(estimator, nodeid = 0):\n     left__child = estimator.children_left[nodeid]\n     right__child = estimator.children_right[nodeid]\n     \n     if left__child == _tree.TREE_LEAF:\n         depths = np.array([0])\n     else:\n         left__depths = leaf__depths(estimator, left__child) + 1\n         right__depths = leaf__depths(estimator, right__child) + 1\n         depths = np.append(left__depths, right__depths)\n \n     return depths\n\ndef leaf__samples(estimator, nodeid = 0):  \n     left__child = estimator.children_left[nodeid]\n     right__child = estimator.children_right[nodeid]\n\n     if left__child == _tree.TREE_LEAF: \n         samples = np.array([estimator.n_node_samples[nodeid]])\n     else:\n         left__samples = leaf__samples(estimator, left__child)\n         right__samples = leaf__samples(estimator, right__child)\n         samples = np.append(left__samples, right__samples)\n\n     return samples\n\ndef visualization__estimator(ensemble, tree_id=0):\n\n     plt.figure(figsize=(20,20))\n     plt.subplot(211)\n\n     estimator = ensemble.estimators_[tree_id].tree_\n     depths = leaf__depths(estimator)\n     \n     plt.hist(depths, histtype='step', color='blue', bins=range(min(depths), max(depths)+1))\n     plt.grid(color='black', linestyle='dotted')\n     plt.xlabel(\"Depth of leaf nodes (tree %s)\" % tree_id)\n     plt.show()\n\ndef visualization__forest(ensemble):\n\n     plt.figure(figsize=(20,20))\n     plt.subplot(211)\n\n     depths__all = np.array([], dtype=int)\n\n     for x in ensemble.estimators_:\n         estimator = x.tree_\n         depths = leaf__depths(estimator)\n         depths__all = np.append(depths__all, depths)\n         plt.hist(depths, histtype='step', color='blue', \n                  bins=range(min(depths), max(depths)+1))\n\n     plt.hist(depths__all, histtype='step', color='blue', \n              bins=range(min(depths__all), max(depths__all)+1), \n              weights=np.ones(len(depths__all))/len(ensemble.estimators_), \n              linewidth=2)\n     plt.grid(color='black', linestyle='dotted')\n     plt.xlabel(\"Depth of leaf nodes\")\n    \n     plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seed = 2020","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.clock()\n\nmodel = RandomForestClassifier(n_jobs=-1,n_estimators=100, max_features=5, random_state=seed)\nmodel.fit(X_train, y_train)\n\nvisualization__estimator(model)\n\nend_time = time.clock()\nprint(\"\")\nprint(\"Total Estimation Running Time:\")\nprint(end_time - start_time, \"Seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.clock()\n\nmodel = RandomForestClassifier(n_jobs=-1,n_estimators=100, max_features=5, random_state=seed)\nmodel.fit(X_train, y_train)\n\nvisualization__forest(model)\n\nend_time = time.clock()\nprint(\"\")\nprint(\"Total Estimation Running Time:\")\nprint(end_time - start_time, \"Seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.clock()\n\nmodel = RandomForestClassifier(n_jobs=-1,n_estimators=100, max_features=5, max_depth=12, random_state=seed)\nmodel.fit(X_train, y_train)\n\nvisualization__forest(model)\n\nend_time = time.clock()\nprint(\"\")\nprint(\"Total Estimation Running Time:\")\nprint(end_time - start_time, \"Seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.clock()\n\nmodel = RandomForestClassifier(n_jobs=-1,n_estimators=100, max_features=5, min_samples_leaf=3, random_state=seed)\nmodel.fit(X_train, y_train)\n\nvisualization__forest(model)\n\nend_time = time.clock()\nprint(\"\")\nprint(\"Total Estimation Running Time:\")\nprint(end_time - start_time, \"Seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.clock()\n\nmodel = RandomForestClassifier(n_jobs=-1,n_estimators=100, max_features=3, min_samples_leaf=2, random_state=seed)\nmodel.fit(X_train, y_train)\n\nvisualization__forest(model)\n\nend_time = time.clock()\nprint(\"\")\nprint(\"Total Estimation Running Time:\")\nprint(end_time - start_time, \"Seconds\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.clock()\n\nmodel = RandomForestClassifier(n_jobs=-1,n_estimators=100, max_features=5, bootstrap=False, random_state=seed)\nmodel.fit(X_train, y_train)\n\nvisualization__forest(model)\n\nend_time = time.clock()\nprint(\"\")\nprint(\"Total Estimation Running Time:\")\nprint(end_time - start_time, \"Seconds\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}