{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Some more practice"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import gc\ngc.collect()\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nfrom dask import dataframe as dd\n\nimport lightgbm as lgb\nimport xgboost as xgb\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\n\nsns.set()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Some functions to fill data, add features and reduce memory usage\n\ndef fill_weather_dataset(weather_df):\n    \n    # Find Missing Dates\n    time_format = \"%Y-%m-%d %H:%M:%S\"\n    start_date = datetime.datetime.strptime(weather_df['timestamp'].min(),time_format)\n    end_date = datetime.datetime.strptime(weather_df['timestamp'].max(),time_format)\n    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\n    hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format) for x in range(total_hours)]\n\n    missing_hours = []\n    for site_id in range(16):\n        site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])\n        new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])\n        new_rows['site_id'] = site_id\n        weather_df = pd.concat([weather_df,new_rows])\n\n        weather_df = weather_df.reset_index(drop=True)           \n\n    # Add new Features\n    weather_df[\"datetime\"] = pd.to_datetime(weather_df[\"timestamp\"])\n    weather_df[\"day\"] = weather_df[\"datetime\"].dt.day\n    weather_df[\"week\"] = weather_df[\"datetime\"].dt.week\n    weather_df[\"month\"] = weather_df[\"datetime\"].dt.month\n    \n    # Reset Index for Fast Update\n    weather_df = weather_df.set_index(['site_id','day','month'])\n\n    air_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['air_temperature'].mean(),columns=[\"air_temperature\"])\n    weather_df.update(air_temperature_filler,overwrite=False)\n\n    # Step 1\n    cloud_coverage_filler = weather_df.groupby(['site_id','day','month'])['cloud_coverage'].mean()\n    # Step 2\n    cloud_coverage_filler = pd.DataFrame(cloud_coverage_filler.fillna(method='ffill'),columns=[\"cloud_coverage\"])\n\n    weather_df.update(cloud_coverage_filler,overwrite=False)\n\n    due_temperature_filler = pd.DataFrame(weather_df.groupby(['site_id','day','month'])['dew_temperature'].mean(),columns=[\"dew_temperature\"])\n    weather_df.update(due_temperature_filler,overwrite=False)\n\n    # Step 1\n    sea_level_filler = weather_df.groupby(['site_id','day','month'])['sea_level_pressure'].mean()\n    # Step 2\n    sea_level_filler = pd.DataFrame(sea_level_filler.fillna(method='ffill'),columns=['sea_level_pressure'])\n\n    weather_df.update(sea_level_filler,overwrite=False)\n\n    wind_direction_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_direction'].mean(),columns=['wind_direction'])\n    weather_df.update(wind_direction_filler,overwrite=False)\n\n    wind_speed_filler =  pd.DataFrame(weather_df.groupby(['site_id','day','month'])['wind_speed'].mean(),columns=['wind_speed'])\n    weather_df.update(wind_speed_filler,overwrite=False)\n\n    # Step 1\n    precip_depth_filler = weather_df.groupby(['site_id','day','month'])['precip_depth_1_hr'].mean()\n    # Step 2\n    precip_depth_filler = pd.DataFrame(precip_depth_filler.fillna(method='ffill'),columns=['precip_depth_1_hr'])\n\n    weather_df.update(precip_depth_filler,overwrite=False)\n\n    weather_df = weather_df.reset_index()\n    weather_df = weather_df.drop(['datetime','day','week','month'],axis=1)\n        \n    return weather_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if isinstance(df[col], datetime.datetime) or pd.api.types.is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_lag_feature(weather_df, window=3):\n    group_df = weather_df.groupby('site_id')\n    cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr', \n            'sea_level_pressure']\n    rolled = group_df[cols].rolling(window=window, min_periods=0)\n    lag_mean = rolled.mean().reset_index().astype(np.float16)\n    lag_max = rolled.max().reset_index().astype(np.float16)\n    lag_min = rolled.min().reset_index().astype(np.float16)\n    lag_std = rolled.std().reset_index().astype(np.float16)\n    for col in cols:\n        weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]\n        weather_df[f'{col}_max_lag{window}'] = lag_max[col]\n        weather_df[f'{col}_min_lag{window}'] = lag_min[col]\n        weather_df[f'{col}_std_lag{window}'] = lag_std[col]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def addFeatures(df):\n    \n    # Sort by timestamp\n    df.sort_values(\"timestamp\")\n    df.reset_index(drop=True)\n    \n    # Add more features\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"],format=\"%Y-%m-%d %H:%M:%S\")\n    df[\"hour\"] = df[\"timestamp\"].dt.hour\n    df[\"weekend\"] = df[\"timestamp\"].dt.weekday\n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n                    \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n                    \"2017-01-02\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n                    \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n                    \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n                    \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n                    \"2019-01-01\"]\n    df[\"is_holiday\"] = (df.timestamp.isin(holidays)).astype(int)\n    #df['square_feet'] =  np.log1p(df['square_feet'])\n    \n    building_mean = df_group.mean().astype(np.float16)\n    building_median = df_group.median().astype(np.float16)\n    building_min = df_group.min().astype(np.float16)\n    building_max = df_group.max().astype(np.float16)\n    building_std = df_group.std().astype(np.float16)\n\n    df['building_mean'] = df['building_id'].map(building_mean)\n    df['building_median'] = df['building_id'].map(building_median)\n    df['building_min'] = df['building_id'].map(building_min)\n    df['building_max'] = df['building_id'].map(building_max)\n    df['building_std'] = df['building_id'].map(building_std)\n    \n    # Remove Unused Columns\n    drop = [#\"timestamp\",\"sea_level_pressure\", \"wind_direction\", \"wind_speed\",\n            \"year_built\",\"floor_count\",'timestamp']\n    df = df.drop(drop, axis=1)\n    gc.collect()\n    \n    # Encode Categorical Data\n    #le = LabelEncoder()\n    #df[\"primary_use\"] = le.fit_transform(df[\"primary_use\"])\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_metadata = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nbuilding_metadata['primary_use'] = building_metadata['primary_use'].astype('category')\nle = LabelEncoder()\nbuilding_metadata[\"primary_use\"] = le.fit_transform(building_metadata[\"primary_use\"])\nbuilding_metadata['square_feet'] =  np.log1p(building_metadata['square_feet'])\nbuilding_metadata = reduce_mem_usage(building_metadata,use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train data:')\ntrain = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\n# Remove outliers\n#train = train[ train['building_id'] != 1099 ]\ntrain = train[~((train['meter'] == 2) & (train['building_id'] == 1099))]\ntrain = train.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')\ntrain['meter_reading_log1p'] = np.log1p(train['meter_reading'])\ndf_group = train.groupby('building_id')['meter_reading_log1p']\n\n#Get part of data with full set of timestamps\ncount_full = train.groupby('building_id')['timestamp'].nunique()\n#Remember count_full is a Series object\ncount_full = count_full[count_full==count_full.max()]\n#ids with whole length\nprint(count_full.index)\ntrain = train[train['building_id'].isin(count_full.index)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Fill Weather Information\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\nweather_train = fill_weather_dataset(weather_train)\n#weather_train = weather_train.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))\nadd_lag_feature(weather_train, window=3)\nadd_lag_feature(weather_train, window=72)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Memory reduction\ntrain = reduce_mem_usage(train,use_float16=True)\nweather_train = reduce_mem_usage(weather_train,use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Merge data\ntrain = train.merge(building_metadata, on='building_id', how='left')\ntrain = train.merge(weather_train, on=['site_id', 'timestamp'], how='left')\ndel weather_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Add features\ntrain = addFeatures(train)\ntrain.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Get features and target variables\ndef get_train_data(df, site_id):\n    df_ = df[df['meter']==mtype]\n    target = df_[\"meter_reading_log1p\"]\n    features = df_.drop(['meter_reading','meter_reading_log1p'], axis = 1)\n    return features, target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_features = [\"building_id\", \"site_id\", \"meter\", \"is_holiday\", \"weekend\", 'primary_use']\n\nparams = {\n    \"objective\": \"regression\",\n    \"boosting\": \"gbdt\",\n    \"num_leaves\": 31,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.85,\n    \"reg_lambda\": 2,\n    \"metric\": \"rmse\",\n}\n\nkf = KFold(n_splits=3)\nmodels = []\nfor mtype in [0,1,2,3]:\n    print(f'training meter: {mtype}')\n    features, target = get_train_data(train, mtype); tmp = []\n    for train_index,test_index in kf.split(features, target):\n        train_features = features.iloc[train_index]\n        train_target = target.iloc[train_index]\n    \n        test_features = features.iloc[test_index]\n        test_target = target.iloc[test_index]\n    \n        d_training = lgb.Dataset(train_features, label=train_target,\n                                 categorical_feature=categorical_features, free_raw_data=False)\n        d_test = lgb.Dataset(test_features, label=test_target,\n                             categorical_feature=categorical_features, free_raw_data=False)\n    \n        model = lgb.train(params, train_set=d_training, num_boost_round=1000, \n                          valid_sets=[d_training,d_test], verbose_eval=25, early_stopping_rounds=50)\n        tmp.append(model)\n    models.append(tmp)\n\ndel train_features, train_target, test_features, test_target, d_training, d_test, features, target, train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Load test data\ntest = pd.read_csv('../input/ashrae-energy-prediction/test.csv')\nrow_ids = test[\"row_id\"]\n#ref = test[['row_id','meter']]\ntest = test.drop(\"row_id\", axis=1)\ntest = reduce_mem_usage(test)\n#td = dd.from_pandas(test, npartitions=20)\n#del test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')\nweather_test = fill_weather_dataset(weather_test)\n#weather_test = weather_test.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))\nweather_test = reduce_mem_usage(weather_test)\nadd_lag_feature(weather_test, window=3)\nadd_lag_feature(weather_test, window=72)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/ashrae-energy-prediction/sample_submission.csv', \n                  dtype={'row_id':np.uint16, 'meter_reading':np.float32})\nsub['row_id'] = row_ids\nsub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pred(df, models):\n    yp_total = np.zeros(df.shape[0])\n    for i, model in enumerate(models):\n        print(f'predicting model-{i}')\n        yp = model.predict(df, num_iteration=model.best_iteration)\n        yp_total += yp\n\n    yp_total /= len(models)\n    return yp_total","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = 1000000\nfor mtype in [0,1,2,3]:\n    tst = test.loc[test['meter']==mtype]\n    tst = tst.merge(building_metadata, on='building_id', how='left')\n    tst = tst.merge(weather_test, on=['site_id', 'timestamp'], how='left')\n    tst = addFeatures(tst)\n    print(f'meter-{mtype} dataframe shape is {tst.shape}')\n    #print(tst.columns)\n    gen = (tst[i:i+n] for i in range(0,tst.shape[0],n))\n    p_full = []\n    for x in gen:\n        p = pred(x, models[mtype])\n        p_full.append(p)\n    p_full = np.concatenate(p_full)\n    print(f'predicted array has shape {p_full.shape}')\n    sub.loc[test['meter']==mtype, 'meter_reading'] = np.expm1(p_full)\n    del tst\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index=False, float_format='%.5f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}