{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport datetime\nimport matplotlib.pyplot as plt\nimport gc\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold\nimport lightgbm as lgb\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"bm_data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\nbm_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w_train_data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv')\nw_train_data.head()\nw_train_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')\n\n# Removing weird data on site_id = 0 before '2016-05-20'\ntrain_data = train_data [ train_data['building_id'] != 1099 ]\ntrain_data = train_data.query('not(building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w_test_data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')\nw_test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')\ntest_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for viewing missing values\ndef missing_values(df):\n    mv = pd.DataFrame(df.isnull().sum()).reset_index()\n    mv.columns = ['Column Name', 'Missing Values']\n    mv['Total Rows'] = df.shape[0]\n    mv['Missing Percentage'] = round((mv['Missing Values']/mv['Total Rows'])*100, 2)\n    return mv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fill_weather_data(weather_df):\n    # Handling Missing Hours. The data has 16 sites in 2016. So it should have (16*24*366 = 140544) records instead of 139773 records. Hence, 771 records of hours are missing\n    time_format = '%Y-%m-%d %H:%M:%S'\n    start_date = datetime.datetime.strptime(weather_df['timestamp'].min(), time_format)\n    end_date = datetime.datetime.strptime(weather_df['timestamp'].max(), time_format)\n    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\n    hours_list = [(end_date - datetime.timedelta(hours = x)).strftime(time_format) for x in range(total_hours)]\n\n    missing_hours = []\n    for site_id in range(16):\n        site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])\n        new_rows = pd.DataFrame(np.setdiff1d(hours_list, site_hours), columns = ['timestamp'])\n        new_rows['site_id'] = site_id\n        weather_df = pd.concat([weather_df, new_rows])\n\n        weather_df = weather_df.reset_index(drop = True)\n    \n    # Adding Day, Week, Month columns\n    weather_df['datetime'] = pd.to_datetime(weather_df['timestamp'])\n    weather_df['day'] = weather_df['datetime'].dt.day\n    weather_df['week'] = weather_df['datetime'].dt.week\n    weather_df['month'] = weather_df['datetime'].dt.month\n    weather_df = weather_df.set_index(['site_id', 'day', 'month'])\n    \n    # Filling missing air_temperature values by mean of each month\n    fill_air_temperature = pd.DataFrame(weather_df.groupby(['site_id', 'day', 'month'])['air_temperature'].mean(), columns=['air_temperature'])\n    weather_df.update(fill_air_temperature, overwrite = False)\n    \n    # Fill missing cloud_coverage values by first calculating mean of each month then propogating missing last observation to missing values\n    fill_cloud_coverage = weather_df.groupby(['site_id', 'day', 'month'])['cloud_coverage'].mean()\n    fill_cloud_coverage = pd.DataFrame(fill_cloud_coverage.fillna(method='ffill'), columns=['cloud_coverage'])\n    weather_df.update(fill_cloud_coverage, overwrite = False)\n    \n    # Fill missing dew_temperature values with monthly mean\n    fill_dew_temperature = pd.DataFrame(weather_df.groupby(['site_id', 'day', 'month'])['dew_temperature'].mean(), columns =['dew_temperature'])\n    weather_df.update(fill_dew_temperature, overwrite = False)\n\n    # Fill missing precip_depth_1_hr values by propagating monthly average to missing values\n    fill_precip_depth = weather_df.groupby(['site_id', 'day', 'month'])['precip_depth_1_hr'].mean()\n    fill_precip_depth = pd.DataFrame(fill_precip_depth.fillna(method = 'ffill'), columns = ['precip_depth_1_hr'])\n    weather_df.update(fill_precip_depth, overwrite = False)\n    \n    # Fill missing sea level values by propagating monthly mean to missing values\n    fill_sea_level_pressure = weather_df.groupby(['site_id', 'day', 'month'])['sea_level_pressure'].mean()\n    fill_sea_level_pressure = pd.DataFrame(fill_sea_level_pressure.fillna(method = 'ffill'), columns = ['sea_level_pressure'])\n    weather_df.update(fill_sea_level_pressure, overwrite = False)\n\n    # Fill missing wind_speed values with monthly average\n    fill_wind_speed = pd.DataFrame(weather_df.groupby(['site_id', 'day', 'month'])['wind_speed'].mean(), columns = ['wind_speed'])\n    weather_df.update(fill_wind_speed, overwrite = False)\n    \n    # Drop unneccassary columns\n    weather_df = weather_df.reset_index() \n    weather_df = weather_df.drop(['wind_direction', 'datetime', 'day', 'week', 'month'], axis = 1)\n    \n    return weather_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w_train_data = fill_weather_data(w_train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = reduce_mem_usage(train_data, use_float16 = True)\nbm_data = reduce_mem_usage(bm_data, use_float16 = True)\nw_train_data = reduce_mem_usage(w_train_data, use_float16 = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merging train data with weather data\ntrain_data = train_data.merge(bm_data, left_on = 'building_id', right_on = 'building_id', how = 'left') \ntrain_data = train_data.merge(w_train_data, left_on = ['site_id', 'timestamp'], right_on = ['site_id', 'timestamp'], how = 'left')\ndel w_train_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.sort_values('timestamp')\ntrain_data.reset_index(drop = True)\n\ntrain_data['timestamp'] = pd.to_datetime(train_data['timestamp'], format = '%Y-%m-%d %H:%M:%S')\ntrain_data['hour'] = train_data['timestamp'].dt.hour\ntrain_data['weekend'] = train_data['timestamp'].dt.weekday\ntrain_data['square_feet'] = np.log1p(train_data['square_feet'])\ntrain_data = train_data.drop(['timestamp', 'sea_level_pressure', 'wind_speed', 'year_built', 'floor_count'], axis = 1)\ngc.collect()\nle = LabelEncoder()\ntrain_data['primary_use'] = le.fit_transform(train_data['primary_use'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = np.log1p(train_data['meter_reading'])\nfeatures = train_data.drop(['meter_reading'], axis = 1)\ndel train_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_features = [\"building_id\", \"site_id\", \"meter\", \"primary_use\", \"weekend\"]\nparams = {\n    \"objective\": \"regression\",\n    \"boosting\": \"gbdt\",\n    \"num_leaves\": 1280,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.85,\n    \"reg_lambda\": 2,\n    \"metric\": \"rmse\",\n}\n\nkf = KFold(n_splits=3)\nmodels = []\nfor train_index,test_index in kf.split(features):\n    train_features = features.loc[train_index]\n    train_target = target.loc[train_index]\n    \n    test_features = features.loc[test_index]\n    test_target = target.loc[test_index]\n    \n    d_training = lgb.Dataset(train_features, label=train_target,categorical_feature=categorical_features, free_raw_data=False)\n    d_test = lgb.Dataset(test_features, label=test_target,categorical_feature=categorical_features, free_raw_data=False)\n    \n    model = lgb.train(params, train_set=d_training, num_boost_round=300, valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)\n    models.append(model)\n    del train_features, train_target, test_features, test_target, d_training, d_test\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del features, target\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model in models:\n    lgb.plot_importance(model)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"row_ids = test_data['row_id']\ntest_data.drop(['row_id'], axis = 1)\ntest_data = reduce_mem_usage(test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = test_data.merge(bm_data, left_on = 'building_id', right_on='building_id',\n                           how = 'left')\ndel bm_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"w_test_data = fill_weather_data(w_test_data)\nw_test_data = reduce_mem_usage(w_test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = test_data.merge(w_test_data, how = 'left', on = ['timestamp','site_id'])\ndel w_test_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.sort_values('timestamp')\ntest_data.reset_index(drop = True)\n\ntest_data['timestamp'] = pd.to_datetime(test_data['timestamp'], format = '%Y-%m-%d %H:%M:%S')\ntest_data['hour'] = test_data['timestamp'].dt.hour\ntest_data['weekend'] = test_data['timestamp'].dt.weekday\ntest_data['square_feet'] = np.log1p(test_data['square_feet'])\ntest_data = test_data.drop(['row_id', 'timestamp', 'sea_level_pressure', 'wind_speed', 'year_built', 'floor_count'], axis = 1)\ngc.collect()\nle = LabelEncoder()\ntest_data['primary_use'] = le.fit_transform(test_data['primary_use'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction\nresults = []\nfor model in models:\n    if results == []:\n        results = np.expm1(model.predict(test_data, num_iteration=model.best_iteration)) / len(models)\n    else:\n        results += np.expm1(model.predict(test_data, num_iteration = model.best_iteration)) / len(models)\n    del model\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_data, models\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results_data = pd.DataFrame({'row_id': row_ids, 'meter_reading': np.clip(results, 0, a_max=None)})\ndel row_ids, results\ngc.collect()\nresults_data.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}