{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Stacking Method"},{"metadata":{},"cell_type":"markdown","source":"### *Kaggle Ensembling Guide*\n\nModel ensembling is a very powerful technique to increase accuracy on a variety of ML tasks. The ensembling approaches for Kaggle Competitions are shared: https://mlwave.com/kaggle-ensembling-guide/"},{"metadata":{},"cell_type":"markdown","source":"### *ashrae-ensembling-1*\n\nsubmittion_01: https://www.kaggle.com/nz0722/aligned-timestamp-lgbm-by-meter-type\n\nsubmittion_02: https://www.kaggle.com/rohanrao/ashrae-half-and-half\n\nsubmittion_03: https://www.kaggle.com/isaienkov/keras-nn-with-embeddings-for-cat-features-1-15\n\nsubmittion_04: https://www.kaggle.com/kimtaegwan/what-s-your-cv-method\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nprint(os.listdir('/kaggle/input/ashrae-ensembling-1'))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step1: Stat Stack"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nfrom scipy.stats.mstats import gmean\nimport seaborn as sns\n%matplotlib inline\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step2"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub_path = \"../input/ashrae-ensembling-1\"\nall_files = os.listdir(sub_path)\nall_files","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step3"},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nouts = [pd.read_csv(os.path.join(sub_path, f), index_col=0) for f in all_files]\nconcat_sub = pd.concat(outs, axis=1)\ncols = list(map(lambda x: \"mol\" + str(x), range(len(concat_sub.columns))))\nconcat_sub.columns = cols\nconcat_sub.reset_index(inplace=True)\nconcat_sub.head()\nncol = concat_sub.shape[1]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step4: check correlation"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub.iloc[:,1:].corr()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step5"},{"metadata":{"trusted":true},"cell_type":"code","source":"corr = concat_sub.iloc[:,1:].corr()\nmask = np.zeros_like(corr, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr, mask=mask, cmap='prism', vmin=0.96, center=0, square=True, linewidths=1, annot=True, fmt='.4f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step6: get the data fields ready for stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['m_max'] = concat_sub.iloc[:, 1:].max(axis=1)\nconcat_sub['m_min'] = concat_sub.iloc[:, 1:].min(axis=1)\nconcat_sub['m_median'] = concat_sub.iloc[:, 1:].median(axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step7"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub.describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step8"},{"metadata":{"trusted":true},"cell_type":"code","source":"cutoff_lo = 0.8\ncutoff_hi = 0.2","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step9"},{"metadata":{"trusted":true},"cell_type":"code","source":"rank = np.tril(concat_sub.iloc[:,1:ncol].corr().values,-1)\nm_gmean = 0\nn = 8\nwhile rank.max()>0:\n    mx = np.unravel_index(rank.argmax(), rank.shape)\n    m_gmean += n*(np.log(concat_sub.iloc[:, mx[0]+1]) + np.log(concat_sub.iloc[:, mx[1]+1]))/2\n    rank[mx] = 0\n    n += 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step10"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['m_mean'] = np.exp(m_gmean/(n-1)**2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step11: Mean Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['meter_reading'] = concat_sub['m_mean']\nconcat_sub[['row_id', 'meter_reading']].to_csv('stack_mean.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step12: Median Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['meter_reading'] = concat_sub['m_median']\nconcat_sub[['row_id', 'meter_reading']].to_csv('stack_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step13: Pushout + Median Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['meter_reading'] = np.where(np.all(concat_sub.iloc[:,1:7] > cutoff_lo, axis=1), 1, \n                                    np.where(np.all(concat_sub.iloc[:,1:7] < cutoff_hi, axis=1),\n                                             0, concat_sub['m_median']))\nconcat_sub[['row_id', 'meter_reading']].to_csv('stack_pushout_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step14: MinMax + Mean Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['meter_reading'] = np.where(np.all(concat_sub.iloc[:,1:7] > cutoff_lo, axis=1), \n                                    concat_sub['m_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:7] < cutoff_hi, axis=1),\n                                             concat_sub['m_min'], \n                                             concat_sub['m_mean']))\nconcat_sub[['row_id', 'meter_reading']].to_csv('stack_minmax_mean.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step15: MinMax + Median Stacking"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['meter_reading'] = np.where(np.all(concat_sub.iloc[:,1:7] > cutoff_lo, axis=1), \n                                    concat_sub['m_max'], \n                                    np.where(np.all(concat_sub.iloc[:,1:7] < cutoff_hi, axis=1),\n                                             concat_sub['m_min'], \n                                             concat_sub['m_median']))\nconcat_sub[['row_id', 'meter_reading']].to_csv('stack_minmax_median.csv', \n                                        index=False, float_format='%.6f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### step16: Averaging Ranks"},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_sub['meter_reading'] = concat_sub['mol0'].rank(method ='min') + concat_sub['mol1'].rank(method ='min') + concat_sub['mol2'].rank(method ='min') \nconcat_sub['meter_reading'] = (concat_sub['meter_reading']-concat_sub['meter_reading'].min())/(concat_sub['meter_reading'].max() - concat_sub['meter_reading'].min())\nconcat_sub.describe()\nconcat_sub[['row_id', 'meter_reading']].to_csv('stack_rank.csv', index=False, float_format='%.8f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}