{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nfrom pathlib import Path\nimport random\nimport sys\nfrom os.path import join as pjoin\n\nfrom tqdm import tqdm_notebook as tqdm\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom IPython.core.display import display, HTML\n\n# --- plotly ---\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff\n\n# --- models ---\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import KFold, train_test_split\n# import lightgbm as lgb\n# import xgboost as xgb\nimport catboost as cb","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"RAW_DATA_DIR = '/kaggle/input/ashrae-energy-prediction/'\n\nweather_dtypes = {\n    'site_id': np.uint8,\n    'air_temperature': np.float32,\n    'cloud_coverage': np.float32,\n    'dew_temperature': np.float32,\n    'precip_depth_1_hr': np.float32,\n    'sea_level_pressure': np.float32,\n    'wind_direction': np.float32,\n    'wind_speed': np.float32,\n}\n\nweather_train = pd.read_csv(pjoin(RAW_DATA_DIR, 'weather_train.csv'),dtype=weather_dtypes,\n    parse_dates=['timestamp'])\nweather_test = pd.read_csv(pjoin(RAW_DATA_DIR, 'weather_test.csv'),dtype=weather_dtypes,\n    parse_dates=['timestamp'])\n\nweather = pd.concat([weather_train,weather_test],ignore_index=True)\ndel weather_train, weather_test\nweather_key = ['site_id', 'timestamp']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key).sort_values(by=weather_key).copy()\ndata_to_plot = temp_skeleton.copy()\ndata_to_plot[\"hour\"] = data_to_plot[\"timestamp\"].dt.hour\ncount = 1\nplt.figure(figsize=(25, 15))\nfor site_id, data_by_site in data_to_plot.groupby('site_id'):\n    by_site_by_hour = data_by_site.groupby('hour').mean()\n    ax = plt.subplot(4, 4, count)\n    plt.plot(by_site_by_hour.index,by_site_by_hour['air_temperature'],'xb-')\n    ax.set_title('site: '+str(site_id))\n    count += 1\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# calculate ranks of hourly temperatures within date/site_id chunks\ntemp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')\n\n# create a dataframe of site_ids (0-16) x mean hour rank of temperature within day (0-23)\ndf_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)\n\n# Subtract the columnID of temperature peak by 14, getting the timestamp alignment gap.\nsite_ids_offsets = pd.Series(df_2d.values.argmax(axis=1) - 14)\nsite_ids_offsets.index.name = 'site_id'\n\ndef timestamp_align(df):\n    df['offset'] = df.site_id.map(site_ids_offsets)\n    df['timestamp_aligned'] = (df.timestamp - pd.to_timedelta(df.offset, unit='H'))\n    df['timestamp'] = df['timestamp_aligned']\n    del df['timestamp_aligned']\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified to support timestamp type, categorical type\n# Modified to add option to use float16 or not. feather format does not support float16.\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            # skip datetime type or categorical type\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nroot = Path('../input/ashrae-feather-format-for-fast-loading')\n\ntrain_df = pd.read_feather(root/'train.feather')\nweather_train_df = pd.read_feather(root/'weather_train.feather')\nbuilding_meta_df = pd.read_feather(root/'building_metadata.feather')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess(df):\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"weekend\"] = df[\"timestamp\"].dt.weekday\n    df[\"month\"] = df[\"timestamp\"].dt.month\n    df[\"dayofweek\"] = df[\"timestamp\"].dt.dayofweek\n    \ndef add_lag_feature(weather_df, window=3):\n    group_df = weather_df.groupby('site_id')\n    cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction', 'wind_speed']\n    rolled = group_df[cols].rolling(window=window, min_periods=0)\n    lag_mean = rolled.mean().reset_index().astype(np.float16)\n    lag_max = rolled.max().reset_index().astype(np.float16)\n    lag_min = rolled.min().reset_index().astype(np.float16)\n    lag_std = rolled.std().reset_index().astype(np.float16)\n    for col in cols:\n        weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]\n        weather_df[f'{col}_max_lag{window}'] = lag_max[col]\n        weather_df[f'{col}_min_lag{window}'] = lag_min[col]\n        weather_df[f'{col}_std_lag{window}'] = lag_std[col]\n        \ntrain_df['date'] = train_df['timestamp'].dt.date\ntrain_df['meter_reading_log1p'] = np.log1p(train_df['meter_reading'])\nbuilding_meta_df[building_meta_df.site_id == 0]\ntrain_df = train_df.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')\ndebug = False    \npreprocess(train_df)\n\n# https://www.kaggle.com/ryches/simple-lgbm-solution\ndf_group = train_df.groupby('building_id')['meter_reading_log1p']\nbuilding_mean = df_group.mean().astype(np.float16)\nbuilding_median = df_group.median().astype(np.float16)\nbuilding_min = df_group.min().astype(np.float16)\nbuilding_max = df_group.max().astype(np.float16)\nbuilding_std = df_group.std().astype(np.float16)\n\ntrain_df['building_mean'] = train_df['building_id'].map(building_mean)\ntrain_df['building_median'] = train_df['building_id'].map(building_median)\ntrain_df['building_min'] = train_df['building_id'].map(building_min)\ntrain_df['building_max'] = train_df['building_id'].map(building_max)\ntrain_df['building_std'] = train_df['building_id'].map(building_std)\n\nweather_train_df = timestamp_align(weather_train_df)\nweather_train_df = weather_train_df.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))\n\nadd_lag_feature(weather_train_df, window=3)\nadd_lag_feature(weather_train_df, window=72)\n\nprimary_use_list = building_meta_df['primary_use'].unique()\nprimary_use_dict = {key: value for value, key in enumerate(primary_use_list)} \nprint('primary_use_dict: ', primary_use_dict)\nbuilding_meta_df['primary_use'] = building_meta_df['primary_use'].map(primary_use_dict)\n\ngc.collect()\n\nreduce_mem_usage(train_df, use_float16=True)\nreduce_mem_usage(building_meta_df, use_float16=True)\nreduce_mem_usage(weather_train_df, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"category_cols = ['building_id', 'site_id', 'primary_use']  # , 'meter'\nfeature_cols = ['square_feet', 'year_built'] + [\n    'hour', 'weekend', # 'month' , 'dayofweek'\n    'building_median'] + [\n    'air_temperature', 'cloud_coverage',\n    'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n    'wind_direction', 'wind_speed', 'air_temperature_mean_lag72',\n    'air_temperature_max_lag72', 'air_temperature_min_lag72',\n    'air_temperature_std_lag72', 'cloud_coverage_mean_lag72',\n    'dew_temperature_mean_lag72', 'precip_depth_1_hr_mean_lag72',\n    'sea_level_pressure_mean_lag72', 'wind_direction_mean_lag72',\n    'wind_speed_mean_lag72', 'air_temperature_mean_lag3',\n    'air_temperature_max_lag3',\n    'air_temperature_min_lag3', 'cloud_coverage_mean_lag3',\n    'dew_temperature_mean_lag3',\n    'precip_depth_1_hr_mean_lag3', 'sea_level_pressure_mean_lag3',\n    'wind_direction_mean_lag3', 'wind_speed_mean_lag3']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_X_y(train_df, target_meter):\n    target_train_df = train_df[train_df['meter'] == target_meter]\n    target_train_df = target_train_df.merge(building_meta_df, on='building_id', how='left')\n    target_train_df = target_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')\n    X_train = target_train_df[feature_cols + category_cols]\n    y_train = target_train_df['meter_reading_log1p'].values\n\n    del target_train_df\n    return X_train, y_train\n\n\nmetric = 'l2'\nearly_stop = 5\n# verbose_eval = 50\n# iteration=5000\nlr=0.1\ntest_size=[0.1, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]\n\nprint('training CatBoostRegressor:')\nmodel = cb.CatBoostRegressor( n_estimators=50,learning_rate=lr,  # iterations=iteration \n                            task_type='GPU', devices='0:1',\n                           depth=16, loss_function='MAE',\n                           random_seed=42, eval_metric='MAE',\n                           boosting_type='Plain')\n\n# for loading the model\n# model = model.load_model('')\nprint(category_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_meter = 0\nX_train, y_train = create_X_y(train_df, target_meter=target_meter)\nprint(X_train.shape, y_train.shape)\ngc.collect()\n\ncat_features = [X_train.columns.get_loc(cat_col) for cat_col in category_cols]\nprint('cat_features', cat_features)\nfor i in range(10):\n    x_tr, x_ts, y_tr, y_ts = train_test_split(X_train, y_train, test_size=test_size[i], random_state=42, shuffle=True)\n    print(x_tr.shape, y_tr.shape, x_ts.shape, y_ts.shape)\n    model = model.fit(X=x_tr, y=y_tr, cat_features=category_cols, # previously category_cols\n                    verbose=True, plot=True,eval_set=(x_ts, y_ts),\n                    early_stopping_rounds=early_stop,\n                    init_model=None)\n    # predictions\n    y_pred_valid = model.predict(x_ts)\n\n    print('best_score', model.best_score_)\n    log = {'train/mae': model.best_score_['training']['l2'],\n           'valid/mae': model.best_score_['valid_1']['l2']}\n    \n    os.chdir('/kaggle/working')\n    model.save_model('ASHRAE_tmeter-{}-{}.cbm'.format(target_meter,model.best_score_))\n    print('==============model saved: ASHRAE_tmeter-{}-{}.cbm'.format(target_meter,model.best_score_))\n    gc.collect()\n\ndel X_train, y_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import FileLink\nimport glob, os\n\nos.chdir('/kaggle/working')\nfiles = [file for f in glob.glob(\"*.txt\")]\nprint(files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink('ASHRAE_LGBM-00-0.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_meter = 1\n\nX_train, y_train = create_X_y(train_df, target_meter=target_meter)\n# y_valid_pred_total = np.zeros(X_train.shape[0])\ngc.collect()\nprint('target_meter', target_meter, X_train.shape)\n\ncat_features = [X_train.columns.get_loc(cat_col) for cat_col in category_cols]\nprint('cat_features', cat_features)\n\nmodels0 = []\nx_tr, x_ts, y_tr, y_ts = train_test_split(X_train, y_train, test_size=0.3, random_state=87, shuffle=True) \ntrain_data = [x_tr, y_tr ]\nvalid_data = [x_ts, y_ts]\nmodel, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols, \n                                    num_rounds=5000, lr=0.01, bf=0.7)\nos.chdir('/kaggle/working')\nmodel.save_model('ASHRAE_LGBM-01-{}.txt'.format(target_meter))\nprint('===========================================model saved: ASHRAE_LGBM-01-{}.txt'.format(target_meter))\ngc.collect()\n\n\nsns.distplot(y_train)\ndel X_train, y_train\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink('ASHRAE_LGBM-00-0.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_meter = 2\n\nX_train, y_train = create_X_y(train_df, target_meter=target_meter)\n# y_valid_pred_total = np.zeros(X_train.shape[0])\ngc.collect()\nprint('target_meter', target_meter, X_train.shape)\n\ncat_features = [X_train.columns.get_loc(cat_col) for cat_col in category_cols]\nprint('cat_features', cat_features)\n\nmodels0 = []\nx_tr, x_ts, y_tr, y_ts = train_test_split(X_train, y_train, test_size=0.25, random_state=59, shuffle=True) \ntrain_data = [x_tr, y_tr ]\nvalid_data = [x_ts, y_ts]\nmodel, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,\n                                    num_rounds=5000, lr=0.03, bf=0.8)\nos.chdir('/kaggle/working')\nmodel.save_model('ASHRAE_LGBM-01-{}.txt'.format(target_meter))\nprint('===========================================model saved: ASHRAE_LGBM-01-{}.txt'.format(target_meter))\ngc.collect()\n\n\nsns.distplot(y_train)\ndel X_train, y_train\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink('ASHRAE_LGBM-00-0.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target_meter = 3\n\nX_train, y_train = create_X_y(train_df, target_meter=target_meter)\n# y_valid_pred_total = np.zeros(X_train.shape[0])\ngc.collect()\nprint('target_meter', target_meter, X_train.shape)\n\ncat_features = [X_train.columns.get_loc(cat_col) for cat_col in category_cols]\nprint('cat_features', cat_features)\n\nmodels0 = []\nx_tr, x_ts, y_tr, y_ts = train_test_split(X_train, y_train, test_size=0.15, random_state=199, shuffle=True) \ntrain_data = [x_tr, y_tr ]\nvalid_data = [x_ts, y_ts]\nmodel, y_pred_valid, log = fit_lgbm(train_data, valid_data, cat_features=category_cols,\n                                    num_rounds=5000, lr=0.05, bf=0.9)\nos.chdir('/kaggle/working')\nmodel.save_model('ASHRAE_LGBM-01-{}.txt'.format(target_meter))\nprint('===========================================model saved: ASHRAE_LGBM-01-{}.txt'.format(target_meter))\ngc.collect()\n\n\nsns.distplot(y_train)\ndel X_train, y_train\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FileLink('ASHRAE_LGBM-00-0.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('loading...')\ntest_df = pd.read_feather(root/'test.feather')\nweather_test_df = pd.read_feather(root/'weather_test.feather')\n\nprint('preprocessing building...')\ntest_df['date'] = test_df['timestamp'].dt.date\npreprocess(test_df)\ntest_df['building_mean'] = test_df['building_id'].map(building_mean)\ntest_df['building_median'] = test_df['building_id'].map(building_median)\ntest_df['building_min'] = test_df['building_id'].map(building_min)\ntest_df['building_max'] = test_df['building_id'].map(building_max)\ntest_df['building_std'] = test_df['building_id'].map(building_std)\n\nprint('preprocessing weather...')\nweather_test_df = timestamp_align(weather_test_df)\nweather_test_df = weather_test_df.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))\nweather_test_df.groupby('site_id').apply(lambda group: group.isna().sum())\n\nadd_lag_feature(weather_test_df, window=3)\nadd_lag_feature(weather_test_df, window=72)\n\nprint('reduce mem usage...')\nreduce_mem_usage(test_df, use_float16=True)\nreduce_mem_usage(weather_test_df, use_float16=True)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_feather(os.path.join(root, 'sample_submission.feather'))\nreduce_mem_usage(sample_submission)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_X(test_df, target_meter):\n    target_test_df = test_df[test_df['meter'] == target_meter]\n    target_test_df = target_test_df.merge(building_meta_df, on='building_id', how='left')\n    target_test_df = target_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')\n    X_test = target_test_df[feature_cols + category_cols]\n    return X_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pred(X_test, models, batch_size=1000000):\n    iterations = (X_test.shape[0] + batch_size -1) // batch_size\n    print('iterations', iterations)\n\n    y_test_pred_total = np.zeros(X_test.shape[0])\n    for i, model in enumerate(models):\n        print(f'predicting {i}-th model')\n        for k in tqdm(range(iterations)):\n            y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size])\n            y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test\n\n    y_test_pred_total /= len(models)\n    return y_test_pred_total","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**create models0, modesl1, modesl2, modesl3**"},{"metadata":{"trusted":true},"cell_type":"code","source":"models=[models0, modesl1, models2, models3]\ny_test = []\nfor i, model in enumerate(models):\n    %%time\n    X_test = create_X(test_df, target_meter=i)\n    gc.collect()\n    y_test.append(pred(X_test, model))\n    sns.distplot(y_test0)\n    \ndel X_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.loc[test_df['meter'] == 0, 'meter_reading'] = np.expm1(y_test[0])\nsample_submission.loc[test_df['meter'] == 1, 'meter_reading'] = np.expm1(y_test[1])\nsample_submission.loc[test_df['meter'] == 2, 'meter_reading'] = np.expm1(y_test[2])\nsample_submission.loc[test_df['meter'] == 3, 'meter_reading'] = np.expm1(y_test[3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.to_csv('submission.csv', index=False, float_format='%.4f')\nsample_submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}