{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport gc\nimport psutil     \nimport random\nimport datetime\nimport warnings\nwarnings.filterwarnings('ignore')\n        \n# Any results you write to the current directory are saved as output.\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom sklearn.experimental import enable_iterative_imputer  # noqa\nfrom sklearn.impute import IterativeImputer","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"building_metadata = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\nsample_submission = pd.read_csv(\"../input/ashrae-energy-prediction/sample_submission.csv\")\ntest = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\ntrain = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\nweather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\nweather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\n\ntrain.timestamp = pd.to_datetime(train.timestamp)\ntest.timestamp = pd.to_datetime(test.timestamp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Function to reduce the DF size\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\n\ndef display_missing(df):\n    total = df.isnull().sum().sort_values(ascending=False)\n    percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)\n    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    \n    f, ax = plt.subplots(figsize=(15, 6))\n    plt.xticks(rotation='90')\n    sns.barplot(x=missing_data.index, y=missing_data['Percent'])\n    plt.xlabel('Features', fontsize=15)\n    plt.ylabel('Percent', fontsize=15)\n    plt.title('Percent of missing values by feature', fontsize=15)\n    \n    missing_data.head()\n    return missing_data\n\ndef randomize_na(df):\n    for col in df.columns:\n        data = df[col]\n        mask = data.isnull()\n        samples = random.choices( data[~mask].values , k = mask.sum() )\n        data[mask] = samples\n    return df\n\n\n## handling missing hours from\n## https://www.kaggle.com/aitude/ashrae-missing-weather-data-handling\ndef populate_missing_hours(weather_df):\n    time_format = \"%Y-%m-%d %H:%M:%S\"\n    start_date = datetime.datetime.strptime(weather_df['timestamp'].min(),time_format)\n    end_date = datetime.datetime.strptime(weather_df['timestamp'].max(),time_format)\n    total_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\n    hours_list = [(end_date - datetime.timedelta(hours=x)).strftime(time_format) for x in range(total_hours)]\n\n    missing_hours = []\n    for site_id in range(16):\n        site_hours = np.array(weather_df[weather_df['site_id'] == site_id]['timestamp'])\n        new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns=['timestamp'])\n        new_rows['site_id'] = site_id\n        weather_df = pd.concat([weather_df,new_rows])\n\n    weather_df = weather_df.reset_index(drop=True)    \n    return weather_df\n\n## refactored original in https://www.kaggle.com/aitude/ashrae-missing-weather-data-handling\ndef process_weather_data(df, update_columns, use_fillna):\n    df = populate_missing_hours(df)\n    df.timestamp = pd.to_datetime(df.timestamp)\n    df['dayofweek'] = df.timestamp.dt.dayofweek\n    df['month'] = df.timestamp.dt.month \n    df['week'] = df.timestamp.dt.week \n    df['day'] = df.timestamp.dt.day\n    \n    df = df.set_index(['site_id','day','month']) ## to speed things up\n\n    for column, fillna in zip(update_columns, use_fillna):\n        #print(f'column: {column}, fillna: {fillna}')\n        updated = pd.DataFrame(df.groupby(['site_id','day','month'])[column].mean(), columns=[column])\n        #if (fillna == True):\n        #    updated = updated.fillna(method='ffill')\n        df.update(updated, overwrite=False)\n    \n    df = df.reset_index()\n    df.drop(['day','week','month'], axis=1, inplace=True)\n    return df \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = reduce_mem_usage(train)\ntest = reduce_mem_usage(test)\nweather_train = reduce_mem_usage(weather_train)\nweather_test = reduce_mem_usage(weather_test)\nbuilding_metadata = reduce_mem_usage(building_metadata)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## building_metadata"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_missing(building_metadata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing\n\nbuilding_metadata = randomize_na(building_metadata) \nbuilding_metadata['floor_count'] = building_metadata['floor_count'].astype(np.int16)\nbuilding_metadata['year_built'] = building_metadata['year_built'].astype(np.int16)\nbuilding_metadata['year_built'] = building_metadata['year_built'] - 1900\n\n## remove outliers in SF\nbuilding_metadata['square_feet'] = building_metadata['square_feet'].apply(lambda x: x if x <= 600000 else 600000)\nbuilding_metadata['square_feet'] = building_metadata['square_feet'].apply(lambda x: np.log1p(x))\n\nprimary_use_encoder = preprocessing.LabelEncoder()\nbuilding_metadata['primary_use'] = primary_use_encoder.fit_transform(building_metadata.primary_use)\n\n## remove outliers in floor count\nbuilding_metadata['floor_count'] = building_metadata['floor_count'].apply(lambda x: 20 if x > 20 else x)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## weather_train"},{"metadata":{"trusted":true},"cell_type":"code","source":"display_missing(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"update_lolumns = ['air_temperature','cloud_coverage', 'dew_temperature', 'sea_level_pressure', 'wind_direction', 'wind_speed', 'precip_depth_1_hr']   \nuse_fillna = [False, True, False, True, False, False, True]\n    \nweather_train_df = process_weather_data(weather_train, update_lolumns, use_fillna)\nweather_test_df = process_weather_data(weather_test, update_lolumns, use_fillna)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## reduce memory a little"},{"metadata":{"trusted":true},"cell_type":"code","source":"to_drop = ['sea_level_pressure','wind_direction','wind_speed']\n\ndel weather_train_df[to_drop]\ndel weather_test_df[to_drop]\n           \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## merge datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_datasets(building, weather, data, is_test=False):\n    df = data.merge(building, left_on='building_id',right_on='building_id',how='left')\n    df = df.merge(weather, how='left', left_on=['site_id','timestamp'],right_on=['site_id','timestamp'])\n    \n    if is_test == True:\n        row_id = df['row_id']\n        df.drop(['row_id', 'timestamp'], axis=1, inplace=True)    \n        return df, row_id\n\n    ## this is train\n    target = np.log1p(data['meter_reading'])  \n    df.drop(['meter_reading','timestamp'], axis=1, inplace=True)    \n    return df, target\n\ntrain_df, target = merge_datasets(building_metadata, weather_train_df, train)\ntest_df, row_id = merge_datasets(building_metadata, weather_test_df, test, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train, test\ndel building_metadata, weather_test, weather_train\ndel weather_train_df, weather_test_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## hyperparms from: https://www.kaggle.com/aitude/ashrae-hyperparameter-tuning"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport lightgbm as lgb\n\nparams = {\n     'num_iterations':200,\n     'boosting_type': 'gbdt',\n     'objective': 'regression',\n     'metric': 'rmse',\n     'num_leaves' : 1000,\n     'learning_rate': 0.07,\n     'feature_fraction': 0.89,\n     'bagging_fraction': 0.97,\n     'lambda_l1' : 3,\n     'lambda_l2' : 5,\n     'max_depth' : 11    \n}\n\ncategorical_features = [\"building_id\", \"site_id\", \"meter\", \"primary_use\", \"dayofweek\"]\n\nevals_results = []  # to record eval results for plotting\nmodels = []\n\nkf = KFold(n_splits=3)\nfor train_index,test_index in kf.split(train_df):\n    X_train, y_train = train_df.loc[train_index], target.loc[train_index]\n    X_test, y_test = train_df.loc[test_index], target.loc[test_index]\n    \n    d_training = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_features, free_raw_data=False)\n    d_test = lgb.Dataset(X_test, label=y_test,categorical_feature=categorical_features, free_raw_data=False)\n    \n    evals_result = {}\n    model = lgb.train(params, train_set=d_training, valid_sets=[d_training,d_test], \n                      verbose_eval=25, early_stopping_rounds=50, evals_result = evals_result)\n    models.append(model)\n    evals_results.append(evals_result)\n    \n    del X_train, y_train, X_test, y_test, d_training, d_test\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for model, evals_result in zip(models, evals_results):\n    f, (ax1, ax2) = plt.subplots(nrows = 1, ncols = 2, figsize=(15, 6))\n    lgb.plot_importance(model, ax=ax1)\n    lgb.plot_metric(evals_result, metric='rmse', ax=ax2)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df, target, evals_results\ngc.collect()\n\nresults = []\nfor model in models:\n    if  results == []:\n        results = np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)\n    else:\n        results += np.expm1(model.predict(test_df, num_iteration=model.best_iteration)) / len(models)\n    del model\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_df, models\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\"row_id\": row_id, \"meter_reading\": np.clip(results, 0, a_max=None)})\ndel row_id,results\ngc.collect()\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}