{"cells":[{"metadata":{},"cell_type":"markdown","source":"#### I was wondering How this state of the art works,so I quickly applied this to the competition by myself (It was really simple to implement NGBoost compared with xgboost/lgbm). Hope you enjoy this :-)"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import StratifiedKFold\nfrom tqdm import tqdm_notebook as tqdm\nimport os\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified to support timestamp type, categorical type\n# Modified to add option to use float16 or not. feather format does not support float16.\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            # skip datetime type or categorical type\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#We need to install ngboost first ;-)\n!pip install ngboost","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from ngboost.ngboost import NGBoost\nfrom ngboost.learners import default_tree_learner\nfrom ngboost.scores import MLE\nfrom sklearn.metrics import mean_squared_error\nfrom ngboost.distns import Normal","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/ashrae-feather-format-for-fast-loading/'\nfiles = os.listdir(path)\nprint(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files = ['building_metadata.feather','test.feather','weather_test.feather','weather_train.feather','train.feather','sample_submission.feather']\nbmeta = pd.read_feather(path+files[0])\ntest = pd.read_feather(path+files[1])\nwtest = pd.read_feather(path+files[2])\nwtrain = pd.read_feather(path+files[3])\ntrain = pd.read_feather(path+files[4])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"test['is_train'] = 0\ntrain['is_train'] = 1\nwtotal = pd.concat([wtrain,wtest], ignore_index=True)\ntotal = pd.concat([train,test],ignore_index=True)\np_u = bmeta['primary_use'].unique().astype(str)\np_u_dict={i :idx for idx,i in enumerate(p_u)}\nbmeta.primary_use = bmeta.primary_use.map(p_u_dict)\nbmeta.primary_use = bmeta.primary_use.astype(int)\ntotal = total.merge(bmeta[['site_id','building_id','primary_use','square_feet']], on='building_id',how='left')\ntimestamp = total.groupby(['site_id','timestamp'],as_index=False).mean()[['site_id','timestamp']]\nwtotal = timestamp.merge(wtotal,on=['site_id','timestamp'],how='left')\n#Interpolation (nearest) -> (backward fill)\nfor i in tqdm(wtotal.site_id.unique()):\n    wtotal.update(wtotal.loc[wtotal.site_id==i].interpolate('nearest',limit_direction='both'))\n    wtotal.update(wtotal.loc[wtotal.site_id==i].fillna(method='bfill'))\ntotal = total.merge(wtotal, on=['site_id','timestamp'],how='left')\ntotal['M'] = total.timestamp.dt.month\ntotal['D'] = total.timestamp.dt.dayofweek\ntotal['H'] = total.timestamp.dt.hour\ntotal['Q'] = total.timestamp.dt.quarter\ntotal['W'] = total.timestamp.dt.week\ntotal = reduce_mem_usage(total)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = total.loc[total.is_train==1]\ntest = total.loc[total.is_train==0]\ntrain['log1p_meter_reading'] = np.log1p(train.meter_reading)\ntrain = train.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training & Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Select features to use for training.\ntg = ['log1p_meter_reading']\ndo_not_use =  tg + ['meter','is_train'\n              ,'timestamp'\n              ,'meter_reading'\n              ,'cloud_coverage'\n              ,'precip_depth_1h'\n              ,'sea_level_pressure'\n                ,'precip_depth_1_hr'\n              ,'row_id'\n              ,'wind_direction']\ncols = [c for c in train.columns if c not in do_not_use]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('NULL CHECKING')\nprint('#####Train#####')\nprint(train[cols].isnull().sum())\nprint('#####Test#####')\nprint(test[cols].isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del total\ndel wtrain\ndel wtest\ndel bmeta","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def Ngboost_training(df,tdf,meter):\n    folds = 2\n    seed = 7\n    shuffle = False\n    kf = StratifiedKFold(n_splits = folds, shuffle=shuffle , random_state=seed)\n    #Down-sampling\n    df = df.loc[(df.meter==meter)&(df.H==0)]\n    tdf = tdf.loc[tdf.meter==meter]\n    prediction = np.zeros(tdf.shape[0])\n    i = 0\n    ngb = NGBoost(n_estimators=50, learning_rate=0.4,\n                  Dist=Normal,\n                  Base=default_tree_learner,\n                  natural_gradient=True,\n                  minibatch_frac=0.6,\n                  Score=MLE(),verbose=False)\n    for tr,val in tqdm(kf.split(df, df['building_id']),total=folds):\n        print(f'fold:{i+1}')\n        i+=1\n        print(f'Target : {tg[0]}// Meter : {meter}// # of features : {len(cols)}')\n        print(f'Train_size : {len(tr)} Validation_size : {len(val)}')\n        \n        ngb.fit(df[cols].iloc[tr].values, df[tg[0]].iloc[tr].values)\n        \n        Y_preds = ngb.predict(df[cols].values)\n        Y_dists = ngb.pred_dist(df[cols].values)\n        \n        MSE = mean_squared_error(Y_preds, df[tg[0]].values)\n        print('MSE : ', MSE)\n        NLL = -Y_dists.logpdf(df[tg[0]].values.flatten()).mean()\n        print('NLL(Negative Log Likelihood)', NLL)\n        \n        #Test Prediction\n        test_preds = ngb.predict(tdf[cols].values)\n        print(f'Predicted Size : {len(test_preds)}')\n        prediction += test_preds\n        gc.collect()\n    prediction = prediction/folds\n    print('End')\n    return prediction,ngb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_feather('/kaggle/input/ashrae-feather-format-for-fast-loading/sample_submission.feather')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred0,ngb0 = Ngboost_training(train,test,0)\ngc.collect()\ntest.loc[test['meter'] == 0, 'meter_reading'] = np.clip(np.expm1(pred0), a_min=0, a_max=None)\npred1,ngb1 = Ngboost_training(train,test,1)\ngc.collect()\ntest.loc[test['meter'] == 1, 'meter_reading'] = np.clip(np.expm1(pred1), a_min=0, a_max=None)\npred2,ngb2 = Ngboost_training(train,test,2)\ngc.collect()\ntest.loc[test['meter'] == 2,'meter_reading'] = np.clip(np.expm1(pred2), a_min=0, a_max=None)\npred3,ngb3 = Ngboost_training(train,test,3)\ngc.collect()\ntest.loc[test['meter'] == 3, 'meter_reading'] = np.clip(np.expm1(pred3), a_min=0, a_max=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['meter_reading'] = test['meter_reading'].values\nsub.to_csv('submission.csv', index=False, float_format='%.4f')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Result"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.describe().astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predicted Meter_Reading(Plot)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Meter 0')\ntest.loc[test.meter==0][['timestamp','meter_reading']].set_index('timestamp').resample('H').meter_reading.mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Meter 1')\ntest.loc[test.meter==1][['timestamp','meter_reading']].set_index('timestamp').resample('H').meter_reading.mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Meter 2')\ntest.loc[test.meter==2][['timestamp','meter_reading']].set_index('timestamp').resample('H').meter_reading.mean().plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Meter 3')\ntest.loc[test.meter==3][['timestamp','meter_reading']].set_index('timestamp').resample('H').meter_reading.mean().plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### I think this looks good. I will consider to use it, and It is relatively fast ;-)\n\n* Next Goal//\n    Submission :)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}