{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Imports"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datetime\nimport gc\nimport pickle\nfrom sklearn.preprocessing import LabelEncoder\nimport sys\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_absolute_error, mean_squared_error \nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import KFold    \nimport matplotlib.pyplot as plt\nimport os\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Function to reduce the DF size\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Original imputing code for weather from https://www.kaggle.com/aitude/ashrae-missing-weather-data-handling by @aitude\ndef fill_weather_dataset(weather_df):\n    \n    # Find Missing Dates\n    time_format = \"%Y-%m-%d %H:%M:%S\"\n    start_date = datetime.datetime.strptime(weather_df['timestamp'].min(),time_format)\n    end_date = datetime.datetime.strptime(weather_df['timestamp'].max(),time_format)\n    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\n    hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format) for x in range(total_hours)]\n\n    missing_hours = []\n    for site_id in range(16):\n        site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])\n        new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])\n        new_rows['site_id'] = site_id\n        weather_df = pd.concat([weather_df,new_rows])\n\n        weather_df = weather_df.reset_index(drop=True)           \n\n    # Add new Features\n    weather_df[\"datetime\"] = pd.to_datetime(weather_df[\"timestamp\"])\n    weather_df[\"day\"] = weather_df[\"datetime\"].dt.day\n    weather_df[\"week\"] = weather_df[\"datetime\"].dt.week\n    weather_df[\"month\"] = weather_df[\"datetime\"].dt.month\n    \n    # Reset Index for Fast Update\n    weather_df = weather_df.set_index(['site_id','day','month'])\n\n    air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean(),columns=[\"air_temperature\"])\n    weather_df.update(air_temperature_filler,overwrite=False)\n\n    # Step 1\n    cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()\n    # Step 2\n    cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=[\"cloud_coverage\"])\n\n    weather_df.update(cloud_coverage_filler,overwrite=False)\n\n    due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean(),columns=[\"dew_temperature\"])\n    weather_df.update(due_temperature_filler,overwrite=False)\n\n    # Step 1\n    sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()\n    # Step 2\n    sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])\n\n    weather_df.update(sea_level_filler,overwrite=False)\n\n    wind_direction_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean(),columns=['wind_direction'])\n    weather_df.update(wind_direction_filler,overwrite=False)\n\n    wind_speed_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean(),columns=['wind_speed'])\n    weather_df.update(wind_speed_filler,overwrite=False)\n\n    # Step 1\n    precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()\n    # Step 2\n    precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])\n\n    weather_df.update(precip_depth_filler,overwrite=False)\n\n    weather_df = weather_df.reset_index()\n    weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)\n        \n    return weather_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ETL and preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocessing(nsample = 0, type = 'train'):\n    \n    building_metadata = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\n\n    if (type == 'train'):\n        data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')\n        weather = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv')           \n    elif (type == 'test'):  \n        data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')\n        weather = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')     \n\n    weather = fill_weather_dataset(weather)\n\n    data = reduce_mem_usage(data)\n    building_metadata = reduce_mem_usage(building_metadata)\n    weather = reduce_mem_usage(weather)\n\n    # joining by building_id\n    data = (building_metadata.set_index(\"building_id\").join(data.set_index(\"building_id\"))).reset_index()\n\n    if (type == 'train'):\n        # Correct units for site 0 to kwh    \n        data.loc[(data['site_id'] == 0) & (data['meter'] == 0), 'meter_reading'] = data[(data['site_id'] == 0) & (data['meter'] == 0)]['meter_reading'] * 0.2931    \n    \n    # joining by site_id and timestamp using multi indexes\n    data = data.set_index(['site_id','timestamp']).join(weather.set_index(['site_id','timestamp'])).reset_index()\n    del building_metadata, weather\n    gc.collect()\n    \n    if nsample > 0 :\n        data = data.sample(n=nsample)\n        gc.collect()\n    \n    # Convert timestamp string to datetime\n    data.loc[:, 'timestamp'] = pd.to_datetime(data.timestamp)\n    data['month'] = pd.DatetimeIndex(data.timestamp).month\n    data['weekday'] = pd.DatetimeIndex(data.timestamp).dayofweek\n    data['hour'] = pd.DatetimeIndex(data.timestamp).hour\n    data['day'] = pd.DatetimeIndex(data.timestamp).day\n\n    if (type == 'train'):    \n        # Remove outliers\n        Meter1_Outliers = data.loc[(data.meter == 1) & (data.meter_reading > 20000)].building_id.unique()\n        data = data[~data['building_id'].isin(Meter1_Outliers)] \n        Meter2_Outliers = data.loc[(data.meter == 2) & (data.meter_reading > 20000)].building_id.unique()\n        data = data[~data['building_id'].isin(Meter2_Outliers)] \n        Meter3_Outliers = data.loc[(data.meter == 3) & (data.meter_reading > 5000)].building_id.unique()\n        data = data[~data['building_id'].isin(Meter3_Outliers)] \n\n        # Remove all rows where the meter reading is 0\n        data = data.drop(data.loc[data.meter_reading == 0].index, axis = 0)\n        # Split target and take the log (because the evaluation metric is RMSLE \n        # and we'll use the RMSE metric for training)\n        y = np.log1p(data.meter_reading)\n        data = data.drop('meter_reading', axis = 1)           \n    elif (type == 'test'):    \n        y = data.row_id\n        data = data.drop('row_id', axis = 1)\n    \n    # Dropping useless\n    useless = ['timestamp', \"sea_level_pressure\", \"wind_direction\", \"wind_speed\",\"year_built\",\"floor_count\"]\n    data = data.drop(useless, axis = 1)\n    gc.collect()\n    \n    pkl_file = open('/kaggle/input/ashrae-preprocessing-train/LabelEncoder.pkl', 'rb')\n    le = pickle.load(pkl_file)\n    data[\"primary_use\"] = le.transform(data[\"primary_use\"])\n\n    data = reduce_mem_usage(data)   \n    print(data.memory_usage().sum() / 1024**2, 'Mb')    \n        \n    return data, y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"preprocessingtask = 'load'               # 'load' saved data from preprocessing notebooks\n                                        #  or 'compute' fresh data (much longer)\nif (preprocessingtask == 'load') :\n    pkl_file = open('/kaggle/input/ashrae-preprocessing-train/data_train.pkl', 'rb')\n    data_train = pickle.load(pkl_file)\n    pkl_file.close()\n    pkl_file = open('/kaggle/input/ashrae-preprocessing-train/y.pkl', 'rb')\n    y = np.log1p(pickle.load(pkl_file))\n    pkl_file.close()\n    pkl_file = open('/kaggle/input/ashrae-preprocessing-test/data_test.pkl', 'rb')\n    data_test = pickle.load(pkl_file)\n    pkl_file.close()\n    pkl_file = open('/kaggle/input/ashrae-preprocessing-test/row_id.pkl', 'rb')\n    row_id = pickle.load(pkl_file)\n    pkl_file.close()\nelif (preprocessingtask == 'compute') :\n    data_train, y = preprocessing(nsample = 0, type = 'train')    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model creation and training"},{"metadata":{"trusted":true},"cell_type":"code","source":"numerical_cols = ['square_feet', 'air_temperature','dew_temperature', 'precip_depth_1_hr']\ncategorical_cols = ['site_id', 'building_id','primary_use', 'meter', 'cloud_coverage','month', 'weekday', 'hour', 'day']\n\nfeatures_cols = numerical_cols + categorical_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train, X_valid, y, y_valid = train_test_split(data_train, y, train_size=0.8, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sample grid search for hyperparameter optimisation \n# (inactive by default since it has already been done)\n\nGridSearch = False\n\nif GridSearch :\n    search_parameters = {'learning_rate' : [0.25, 0.3, 0.35],\n                         'num_leaves' : [900, 1000, 1250],\n                         'feature_fraction' : [0.85],   \n                         'reg_lambda': [0.7, 0.8, 1],}\n    clf = lgb.LGBMRegressor()\n    gs = GridSearchCV(\n        estimator=clf, \n        param_grid=search_parameters, \n        cv=3,\n        refit=True,\n        verbose=True)\n\n    search = gs.fit(X_train, y_train)\n    print(search.best_params_)\n    Lgb_predictions = Lgb_predictions = gs.predict(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if ~GridSearch :  # Main model definition and training      \n    params = {  'boosting_type': 'gbdt',\n                'objective': 'regression',\n                'metric': 'rmse',\n                'categorical_features' : categorical_cols,\n                'feature_fraction': 0.85, \n                'learning_rate': 0.35, \n                'num_leaves': 1250, \n                'reg_lambda': 0.8,\n            }\n\n    # Using 3-fold training \n    kf = KFold(n_splits=3)\n    models = []\n    \n    y = y.reset_index()\n    y = y.drop('index', axis = 1)\n    data_train = data_train.reset_index()\n    data_train = data_train.drop('index', axis = 1)\n\n    for train_index,test_index in kf.split(data_train):       \n        train_features = data_train.loc[train_index]\n        train_target = y.loc[train_index]\n        test_features = data_train.loc[test_index]    \n        test_target = y.loc[test_index]\n\n        d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_cols, free_raw_data=False)\n        d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_cols, free_raw_data=False)\n    \n        model = lgb.train(params, train_set=d_training, num_boost_round = 1000, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)\n        models.append(model)\n        del train_features, train_target, test_features, test_target, d_training, d_test\n        gc.collect()        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model evaluation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Final validation on the hold-out set\nLgb_predictions = []  \nfor model in models:\n    if  Lgb_predictions == []:\n        # taking the exp to revert the earlier log  \n        Lgb_predictions = np.expm1(model.predict(X_valid, num_iteration=model.best_iteration)) / len(models)\n    else:\n        Lgb_predictions += np.expm1(model.predict(X_valid, num_iteration=model.best_iteration)) / len(models)  \n        \n# Reverting unit for scoring in site 0 : * 3.4118\nX_valid['row_id'] = np.arange(len(X_valid))\nsite0_idx = X_valid.loc[(X_valid.site_id == 0) & (X_valid.meter == 0)].index\nto_correct = X_valid.loc[site0_idx].row_id\nLgb_predictions[to_correct] = Lgb_predictions[to_correct] * 3.4118\nX_valid = X_valid.drop('row_id', axis = 1)\n    \nprint(mean_absolute_error(y_valid, Lgb_predictions))\nprint(mean_squared_error(y_valid, Lgb_predictions))\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Feature importance by model\nimport matplotlib.pyplot as plt\n\nfor model in models:\n    lgb.plot_importance(model)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Saving the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"output = open('models.pkl', 'wb')  \npickle.dump(models, output)\noutput.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}