{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Introduction"},{"metadata":{},"cell_type":"markdown","source":"You can find some commented code so people can recompile the train.csv, test.csv and their own models using the competition files. I compiled train, test and the model itself so it will produce less memory overhead without the merges or training.\n\nThis **competition** is about:\n\n* *Predicting energy consumption to improve building efficiencies to reduce costs and emissions*"},{"metadata":{},"cell_type":"markdown","source":"![building_picture](https://www.tetratech.com/en/images/ne18-051-sustainable-design-1-650.jpg)"},{"metadata":{},"cell_type":"markdown","source":"### Imports"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc, math, pickle, datetime, os, random\nimport numpy as np \nimport pandas as pd \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.metrics import mean_squared_log_error\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom pandas.plotting import register_matplotlib_converters\nregister_matplotlib_converters()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    \"\"\"\n    :param df: Dataframe with columns unprocessed so they use more memory than needed\n    \n    :returns:\n        df -> Dataframe with lower memory use\n    \"\"\"\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\ndef clean_timestamps(df):\n    \"\"\"\n    :param df: Dataframe containing a \"timestamp\" field which will be broken down in hour, year, day,...\n    \"\"\"\n    df['timestamp'] = pd.to_datetime(df['timestamp'])\n    df[\"year\"] = df[\"timestamp\"].dt.year.astype(np.uint16)\n    df[\"month\"] = df[\"timestamp\"].dt.month.astype(np.uint8)\n    df[\"day\"] = df[\"timestamp\"].dt.day.astype(np.uint8)\n    df[\"hour\"] = df[\"timestamp\"].dt.hour.astype(np.uint8)\n    df[\"weekend\"] = df[\"timestamp\"].dt.weekday.astype(np.uint8)\n    \ndef drop_cols(df):\n    \"\"\"\n    :param df: Dataframe with unnecessary cols\n    \n    :returns:\n            df -> dataframe containing only the desired columns\n    \"\"\"\n    #drop_cols = ['timestamp','primary_use', 'site_id', 'floor_count',\"precip_depth_1_hr\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\", \"building_id\"]\n    drop_cols = ['timestamp']\n    df = df.drop(drop_cols, axis = 1)\n    return df\n\n# Predictions lower than zero are turned zero\ndef fix_predictions(y):\n    \"\"\"\n    :param y: Column with predictions\n    \"\"\"\n    y[y < 0] = 0\n\n# Predictibility in our predictions\ndef seed_everything(seed=0):\n    \"\"\"\n    :param seed: Value for seeding random functions\n    \"\"\"\n    random.seed(seed)\n    np.random.seed(seed)\n\n# Fill given categories with their average values\ndef fill_averages(df):\n    \"\"\"\n    :param df: Dataframe containing normal and nan values\n    \"\"\"\n    data_ratios = df.count()/len(df)\n    cols = data_ratios[data_ratios < 1.0].index\n    for col in cols:\n        df[col] = df[col].fillna(-1)\n        df[col] = df[col].astype(np.int8)\n        more_zero = df[col] >= 0\n        less_zero = df[col] < 0\n        mean = df[more_zero][col].mean()\n        df.loc[less_zero, col] = mean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SEED = 5\nseed_everything(SEED)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#PATH = '../input/ashrae-energy-prediction/'\n#train_df = reduce_mem_usage(pd.read_csv(PATH + 'train.csv'))\n#building = reduce_mem_usage(pd.read_csv(PATH + 'building_metadata.csv'))\n#weather_train = reduce_mem_usage(pd.read_csv(PATH + 'weather_train.csv'))\n\n#train_merged = train_df.merge(building, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\n#train = train_merged.merge(weather_train, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"])","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"PATH = '/kaggle/input/'\ntrain = reduce_mem_usage(pd.read_csv(PATH + 'lgb-train-test/train.csv'))\ntest = reduce_mem_usage(pd.read_csv(PATH + 'lgb-train-test/test.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Basic preprocessing"},{"metadata":{"trusted":false},"cell_type":"code","source":"clean_timestamps(train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data visualization"},{"metadata":{"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nsns.countplot(x=\"floor_count\",data=train, order = train['floor_count'].value_counts().index)\nplt.title('Floor count feature column')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(10, 8))\nax = sns.countplot(x=\"primary_use\",data=train, order = train['primary_use'].value_counts().index)\nplt.title('Buiding type count (primary_use) feature column')\nax.set_xticklabels(ax.get_xticklabels(), rotation=40, ha=\"right\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Añadir leyenda con diccionario\nbuilding_id = 213\nplt.figure(figsize=(14, 8))\nax = sns.lineplot(x=\"timestamp\", y=\"meter_reading\", hue=\"meter\", data=train[train['building_id'] == building_id])\nplt.title('Meter readings from building_id {}'.format(building_id))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train = drop_cols(train)\n\ntrain_y = np.log1p(train['meter_reading'])\n\ntrain_x = train.drop('meter_reading', axis=1)\ntrain_x['primary_use'] = LabelEncoder().fit_transform(train_x['primary_use'])\n\ntrain_x.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del train\n#del train_x\n#del train_y \n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LightGBM model"},{"metadata":{"trusted":false},"cell_type":"code","source":"# Train and Validation splits\ntest_size = 0.20\nX_train, X_val, y_train, y_val = train_test_split(train_x, train_y, test_size=test_size, random_state=SEED)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":" # lgb_params = {\n #                   'objective':'regression',\n #                   'boosting_type':'gbdt',\n #                   'metric':'rmse',\n #                   'n_jobs':-1,\n #                   'learning_rate':0.07,\n #                   'num_leaves': 2**8,\n #                   'max_depth':-1,\n #                   'tree_learner':'serial',\n #                   'colsample_bytree': 0.7,\n #                   'subsample_freq':1,\n #                   'subsample':0.5,\n #                   'n_estimators':8500,\n #                   'max_bin':255,\n #                   'verbose':1,\n #                   'seed': SEED,\n #                   'early_stopping_rounds':3500, \n #               } ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del train_x\ndel train_y\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# load model\nprint('Loading model')\ngbm = lgb.Booster(model_file= PATH + 'lbg-model/lgb_classifier_20-10-2019_0.20834363390357943.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#lgb_train = lgb.Dataset(X_train, y_train)\n#lgb_eval = lgb.Dataset(X_val, y_val)\n#gbm = lgb.train(\n#            lgb_params,\n#            lgb_train,\n#            num_boost_round=5000,\n#            valid_sets=(lgb_train, lgb_eval),\n#            verbose_eval = 50\n#            )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_pred = gbm.predict(X_val, num_iteration=gbm.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It does not make sense to have a prediction of energy consumption lower than 0.0 KWh"},{"metadata":{"trusted":false},"cell_type":"code","source":"fix_predictions(y_pred)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"RMSLE error calculation using sklearn library"},{"metadata":{"trusted":false},"cell_type":"code","source":"rmsle = np.sqrt(mean_squared_log_error(y_pred, (y_val)))\nprint('RMSLE: ', rmsle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del y_val\ndel y_train\ndel X_val\ndel X_train\n#del y_pred\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Saving the model"},{"metadata":{"trusted":false},"cell_type":"code","source":"# save model\n\n#gbm.save_model('lgb_classifier_{}_{}.txt'.format(datetime.datetime.now().strftime(\"%d-%m-%Y\"), rmsle), num_iteration=gbm.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature importance"},{"metadata":{"trusted":false},"cell_type":"code","source":"feature_imp = pd.DataFrame(sorted(zip(gbm.feature_importance(), gbm.feature_name()),reverse = True), columns=['Value','Feature'])\nplt.figure(figsize=(10, 5))\nsns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False))\nplt.title('LightGBM Features')\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test preparation"},{"metadata":{"trusted":false},"cell_type":"code","source":"#building = reduce_mem_usage(pd.read_csv(PATH + 'building_metadata.csv'))\n#test = reduce_mem_usage(pd.read_csv(PATH + \"test.csv\"))\n#weather_test = reduce_mem_usage(pd.read_csv(PATH + \"weather_test.csv\"))\n\n#test = test.merge(building, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\n#test = test.merge(weather_test, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"], how=\"left\")\n\n#del weather_test\n#del building\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"clean_timestamps(test)\ntest = drop_cols(test)\ntest = test.drop('row_id', axis = 1)\ntest['primary_use'] = LabelEncoder().fit_transform(test['primary_use'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission = pd.read_csv(PATH+'ashrae-energy-prediction/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_1 = test[:len(test)//3]\ny_pred_1 = gbm.predict(test_1, num_iteration=gbm.best_iteration)\n\ndel test_1\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_2 = test[len(test)//3:(len(test)*2)//3]\ny_pred_2 = gbm.predict(test_2, num_iteration=gbm.best_iteration)\n\ndel test_2\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_3 = test[(len(test)*2)//3:]\ny_pred_3 = gbm.predict(test_3, num_iteration=gbm.best_iteration)\n\ndel test_3\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_pred_test = np.concatenate([y_pred_1, y_pred_2, y_pred_3], axis=0)\n\ndel y_pred_1\ndel y_pred_2\ndel y_pred_3\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"y_pred_test = np.expm1(y_pred_test)\nfix_predictions(y_pred_test)\nsubmission['meter_reading'] = y_pred_test\nsubmission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.log1p(submission['meter_reading']).hist(bins=30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(y_pred, color=\"blue\", label=\"train prediction\")\nsns.distplot(np.log1p(y_pred_test), color=\"green\", label=\"test prediction\")\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}