{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport random\n\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom pathlib import Path\n\nfrom matplotlib import pyplot as plt\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom IPython.display import FileLink\n\npath_data = \"/kaggle/input/ashrae-energy-prediction/\"\npath_train = path_data + \"train.csv\"\npath_test = path_data + \"test.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"\npath_weather_test = path_data + \"weather_test.csv\"\n\nplt.style.use(\"seaborn\")\nsns.set(font_scale=1)\n\nmyfavouritenumber = 0\nseed = myfavouritenumber\nrandom.seed(seed)\npd.set_option('display.max_columns', 50)\npd.set_option('display.max_rows', 150)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%%time\nroot = Path('../input/ashrae-feather-format-for-fast-loading')\n\ntrain_df = pd.read_feather(root/'train.feather')\nweather_train_df = pd.read_feather(root/'weather_train.feather')\nbuilding_meta_df = pd.read_feather(root/'building_metadata.feather')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024 ** 2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n\n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024 ** 2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport datetime, math\n\n\ndef preprocess(train, test=False):\n\n    def primary_use(x):\n        p_use = {\"Religious worship\":3.91,\"Retail\":26.61,\"Other\":35.10,\"Warehouse/storage\":70.93,\"Technology/science\":227.89,\n                 \"Food sales and service\":304.76,\"Lodging/residential\":307.98,\"Entertainment/public assembly\":320.39,\"Parking\":321.10,\n                 \"Public services\":373.27,\"Utility\":538.77,\"Manufacturing/industrial\":549.41,\"Office\":752.07,\"Healthcare\":820.03,\n                 \"Education\":2456.70,\"Services\":10026.19}\n        for key in p_use.keys():\n            if x == key: return p_use.get(key)\n    train['primary_use']=train['primary_use'].apply(lambda x: primary_use(x))\n    # floor_count\n    train['floor_count_ifnan'] = train.floor_count.isnull().astype('int')\n\n    # dew_temperature\n    train['dew_temperature'] = train['dew_temperature'].fillna(23)\n    train['dew_temperature_k'] = train['dew_temperature'].apply(lambda x: x + 273.15)\n\n    # air_temperature\n    train[\"air_temperature\"] = train[\"air_temperature\"].fillna(35)\n    train['air_temperature_k'] = train['air_temperature'].apply(lambda x: x + 273.15)\n\n    # precip_depth_1_hr\n    train['precip_depth_1_hr_ifnan'] = train.precip_depth_1_hr.isnull().astype(\"int\")\n    train['precip_depth_1_hr'] = train['precip_depth_1_hr'].fillna(300)\n\n    # sea_level_pressure\n    train['sea_level_pressure'] = train['sea_level_pressure'].fillna(980)\n    train['sea_level_pressure_atm'] = train['sea_level_pressure'].apply(lambda x: x / 1013.25)\n\n    # wind_direction\n    train['wind_direction'] = train['wind_direction'].fillna(0)\n\n    # wind_speed\n    train['wind_speed'] = train['wind_speed'].fillna(15)\n\n    # timestamp\n    train.timestamp = pd.to_datetime(train.timestamp, format=\"%Y-%m-%d %H:%M:%S\")\n    train.square_feet = np.log1p(train.square_feet)\n\n    if not test:\n        train.sort_values(\"timestamp\", inplace=True)\n        train.reset_index(drop=True, inplace=True)\n\n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n                \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n                \"2017-01-01\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n                \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n                \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n                \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n                \"2019-01-01\"]\n\n    # year_build\n    train['year_built_ifnan'] = train.year_built.isnull().astype('int')\n    train['year_built'] = train['year_built'].fillna(2015)\n\n    train[\"hour\"] = train.timestamp.dt.hour\n    train['year'] = train['timestamp'].dt.year\n    train['month'] = train['timestamp'].dt.month\n    train['day'] = train['timestamp'].dt.day\n    train[\"weekday\"] = train.timestamp.dt.weekday\n    train['age'] = (train['year'] - train['year_built'])\n    train[\"is_holiday\"] = (train.timestamp.dt.date.astype(\"str\").isin(holidays)).astype(int)\n\n    # relative_humidity\n    train['relative_humidity'] = 100 - 5 * (train['air_temperature_k'] - train['dew_temperature_k'])\n\n    # air_density\n    train['air_density'] = (train['sea_level_pressure_atm'] * 14.67) / (train['air_temperature_k'] * 0.0821)\n\n    # saturated_vapour_density\n    vapour_density = 7.2785\n    train['saturated_vapour_density'] = train['relative_humidity'] * vapour_density\n\n\n\n    # Building surface area\n    def build_sur_area(x,h):\n        side = x**(0.5)\n        return 2*(side*side + side*h + side*h)\n\n    train['building_height'] = train['floor_count_ifnan'] * 3  # each floor 3m high\n    train['building_vol'] = train['building_height'] * train['square_feet']\n\n    train['build_sur_area'] = build_sur_area(train['square_feet'], train['building_height'] )\n\n    # evaporation rate\n    def evaporated_water(x0, x1, x2, x3, x4, x5, time):\n        xs = 0.622*(((x0*x1*0.026325) / x3)-1)  # maxm humidity ratio of saturated air\n        x = 0.622 * (((x0 * x2 * 0.026325) / x3) - 1)  # humidity ratio of dry air\n        if time=='hour': return x4 * x5 *(xs-x)\n        if time=='minute': return (x4 * x5 *(xs-x))/60\n        if time=='sec': return (x4 * x5 *(xs-x))/3600\n\n    train['evap_coeff'] = 25+19*train['wind_speed']\n\n    train['evap_per_hour'] = evaporated_water(train['relative_humidity'], train['dew_temperature_k'], train['air_temperature_k'],\n                                                     train['sea_level_pressure'], train['evap_coeff'], train['build_sur_area'], time='hour')\n    train['evap_per_minute'] = evaporated_water(train['relative_humidity'], train['dew_temperature_k'], train['air_temperature_k'],\n                                                     train['sea_level_pressure'], train['evap_coeff'], train['build_sur_area'], time='minute')\n    train['evap_per_sec'] = evaporated_water(train['relative_humidity'], train['dew_temperature_k'], train['air_temperature_k'],\n                                                     train['sea_level_pressure'], train['evap_coeff'], train['build_sur_area'], time='sec')\n    train['condense_per_hour'] = train['evap_per_hour']*train['relative_humidity']*100\n    train['condense_per_minute'] = train['evap_per_minute']*train['relative_humidity']*100\n    train['condense_per_sec'] = train['evap_per_sec']*train['relative_humidity']*100\n\n\n    # heat required by building\n    def required_by_build(x0, x1, x2):\n        air_layer = (x0**(1/3)+1)**3 - x0\n        air_mass = air_layer*x1\n        L = 2256  # latent heat of vaporization of water = 2256 kj/kg\n        return  (L*air_mass) * x2\n\n    train['heat_lost_by_build'] = required_by_build(train['building_vol'], train['air_density'], train['evap_per_sec'])\n\n    # dropping some features\n    drop_features = [\"timestamp\"]\n\n    train.drop(drop_features, axis=1, inplace=True)\n    # train.dropna(axis=0, how='any')\n    train['meter_reading'] = train['meter_reading'].apply(lambda x:np.log1p(x))\n    train['meter_reading'] = train['meter_reading'].apply(lambda x:round(x, 2))\n\n    return train\n    # if test:\n    #     row_ids = train.row_id\n    #     train.drop(\"row_id\", axis=1, inplace=True)\n    #     return train, row_ids\n    # else:\n    #     # train.drop(\"meter_reading\", axis=1, inplace=True)\n    #     return train, y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# reducing memory usage\ntrain = reduce_mem_usage(train_df, use_float16=True)\nb_meta = reduce_mem_usage(building_meta_df, use_float16=True)\nw_train = reduce_mem_usage(weather_train_df, use_float16=True)\n\n# merging b_meta and w_train to train\ntrain = train.merge(b_meta, on=\"building_id\", how=\"left\")\ntrain = train.merge(w_train, on=[\"site_id\", \"timestamp\"], how=\"left\")\n\nk_slice=2   # upto the just four slice\n\ndef get_train(tr_X, k_slice=1):\n    part = len(tr_X)//8\n    if k_slice==1: return tr_X[:part]\n    elif k_slice==2: return tr_X[part:part*2]\n    elif k_slice==3: return tr_X[part*2:part*3]\n    elif k_slice==4: return tr_X[part*3:part*4]\n    elif k_slice==5: return tr_X[part*4:part*5]\n    elif k_slice==6: return tr_X[part*5:part*6]\n    elif k_slice==7: return tr_X[part*6:part*7]\n    else:return tr_X[part*7:part*8]\ntrain = get_train(train, k_slice=1)\ntrain = preprocess(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df, weather_train_df, building_meta_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import h2o\nfrom h2o.automl import H2OAutoML\n\nh2o.init()\n\nx_train = h2o.H2OFrame(train)\n\nx = x_train.columns\ny = \"meter_reading\"\nx.remove(y)\n\n# For binary classification, response should be a factor\n# train[y] = x_train[y].asfactor()\n# test[y] = test[y].asfactor()\n# Run AutoML for 20 base models (limited to 1 hour max runtime by default)\naml = H2OAutoML(max_models=40, seed=42, include_algos=['GBM'],\n               max_runtime_secs=25200)\naml.train(x=x, y=y, training_frame=x_train)\nlb = aml.leaderboard\nlb.head(rows=lb.nrows)\nl_model = aml.leader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/working/model_1.zip'\nl_model.download_mojo(path)\nFileLink('model_1.zip')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# imported_model = h2o.import_mojo(path)\n# # new_observations = h2o.import_file(path='new_observations.csv')\n# predictions = imported_model.predict(test)\n# predictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"df_test = pd.read_csv(path_test)\nweather_test = pd.read_feather(root/'weather_test.feather')\nbuilding_meta_df = pd.read_feather(root/'building_metadata.feather')\n\n# reducing memory usage\ntest = reduce_mem_usage(df_test, use_float16=True)\nb_meta = reduce_mem_usage(building_meta_df, use_float16=True)\nw_train = reduce_mem_usage(weather_test, use_float16=True)\n\n# merging b_meta and w_train to train\ntest = test.merge(b_meta, on=\"building_id\", how=\"left\")\ntest = test.merge(w_train, on=[\"site_id\", \"timestamp\"], how=\"left\")\n\nX_test = preprocess(test)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"del df_test, building, weather_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"pred = np.expm1(int(l_model.predict(X_test)))//8\n\npred.to_csv(\"pred.csv\", index=False)# del model_half_1\nFileLink('pred.csv')\n# gc.collect()\n\n# pred += np.expm1(model_half_2.predict(X_test, num_iteration=model_half_2.best_iteration)) / 2\n    \n# del model_half_2\n# gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame({\"row_id\": row_ids, \"meter_reading\": np.clip(pred['predict'], 0, a_max=None)})\nsubmission.to_csv(\"submission.csv\", index=False)\nFileLink('submission.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}