{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":9994,"databundleVersionId":752467,"sourceType":"competition"}],"dockerImageVersionId":29840,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Half and Half\nThis notebook is the Python implementation of this awesomely simple R code: https://www.kaggle.com/kailex/ac-dc by [kxx](https://www.kaggle.com/kailex)\n\nIt demonstrates splitting the data in half and using each half to build a model which performs very well on the public LB with minimal feature engineering. The discussion on the same: https://www.kaggle.com/c/ashrae-energy-prediction/discussion/115851","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport random\n\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\n\npath_data = \"/kaggle/input/ashrae-energy-prediction/\"\npath_train = path_data + \"train.csv\"\npath_test = path_data + \"test.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"\npath_weather_test = path_data + \"weather_test.csv\"\n\nplt.style.use(\"seaborn\")\nsns.set(font_scale=1)\n\nmyfavouritenumber = 0\nseed = myfavouritenumber\nrandom.seed(seed)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-11-23T05:03:10.831176Z","iopub.execute_input":"2023-11-23T05:03:10.831789Z","iopub.status.idle":"2023-11-23T05:03:12.726238Z","shell.execute_reply.started":"2023-11-23T05:03:10.831731Z","shell.execute_reply":"2023-11-23T05:03:12.724625Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reading train data\nReading train data along with building and weather metadata.","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(path_train)\n\nbuilding = pd.read_csv(path_building)\nle = LabelEncoder()\nbuilding.primary_use = le.fit_transform(building.primary_use)\n\nweather_train = pd.read_csv(path_weather_train)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2023-11-23T05:03:12.729274Z","iopub.execute_input":"2023-11-23T05:03:12.729730Z","iopub.status.idle":"2023-11-23T05:03:34.985563Z","shell.execute_reply.started":"2023-11-23T05:03:12.729649Z","shell.execute_reply":"2023-11-23T05:03:34.984634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Memory optimization\n\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified to support timestamp type, categorical type\n# Modified to add option to use float16\n\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-23T05:03:34.987330Z","iopub.execute_input":"2023-11-23T05:03:34.987951Z","iopub.status.idle":"2023-11-23T05:03:35.014396Z","shell.execute_reply.started":"2023-11-23T05:03:34.987854Z","shell.execute_reply":"2023-11-23T05:03:35.013451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = reduce_mem_usage(df_train, use_float16=True)\nbuilding = reduce_mem_usage(building, use_float16=True)\nweather_train = reduce_mem_usage(weather_train, use_float16=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:03:35.015736Z","iopub.execute_input":"2023-11-23T05:03:35.016241Z","iopub.status.idle":"2023-11-23T05:03:37.936756Z","shell.execute_reply.started":"2023-11-23T05:03:35.016189Z","shell.execute_reply":"2023-11-23T05:03:37.935670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing data\nThere are two files with features that need to be merged with the data. One is building metadata that has information on the buildings and the other is weather data that has information on the weather.   \n\nNote that the only features created are hour, weekday and is_holiday!","metadata":{}},{"cell_type":"code","source":"def prepare_data(X, building_data, weather_data, test=False):\n    \"\"\"\n    Preparing final dataset with all features.\n    \"\"\"\n    \n    X = X.merge(building_data, on=\"building_id\", how=\"left\")\n    X = X.merge(weather_data, on=[\"site_id\", \"timestamp\"], how=\"left\")\n    \n    X.timestamp = pd.to_datetime(X.timestamp, format=\"%Y-%m-%d %H:%M:%S\")\n    X.square_feet = np.log1p(X.square_feet)\n    \n    if not test:\n        X.sort_values(\"timestamp\", inplace=True)\n        X.reset_index(drop=True, inplace=True)\n    \n    gc.collect()\n    \n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n                \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n                \"2017-01-01\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n                \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n                \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n                \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n                \"2019-01-01\"]\n    \n    X[\"hour\"] = X.timestamp.dt.hour\n    X[\"weekday\"] = X.timestamp.dt.weekday\n    X[\"is_holiday\"] = (X.timestamp.dt.date.astype(\"str\").isin(holidays)).astype(int)\n    \n    drop_features = [\"timestamp\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\"]\n\n    X.drop(drop_features, axis=1, inplace=True)\n\n    if test:\n        row_ids = X.row_id\n        X.drop(\"row_id\", axis=1, inplace=True)\n        return X, row_ids\n    else:\n        y = np.log1p(X.meter_reading)\n        X.drop(\"meter_reading\", axis=1, inplace=True)\n        return X, y","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:03:37.941125Z","iopub.execute_input":"2023-11-23T05:03:37.941475Z","iopub.status.idle":"2023-11-23T05:03:37.959657Z","shell.execute_reply.started":"2023-11-23T05:03:37.941419Z","shell.execute_reply":"2023-11-23T05:03:37.958458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, y_train = prepare_data(df_train, building, weather_train)\n\ndel df_train, weather_train\ngc.collect()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-23T05:03:37.961617Z","iopub.execute_input":"2023-11-23T05:03:37.962086Z","iopub.status.idle":"2023-11-23T05:04:31.467130Z","shell.execute_reply.started":"2023-11-23T05:03:37.961917Z","shell.execute_reply":"2023-11-23T05:04:31.466075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Two-fold LightGBM Model split half-and-half\nThe data is split into two based on time. Each half is used as the training data for a model.\n\n**Half 1:** The first 50% rows of train data   \n**Half 2:** The last 50% rows of train data","metadata":{}},{"cell_type":"code","source":"X_half_1 = X_train[:int(X_train.shape[0] / 2)]\nX_half_2 = X_train[int(X_train.shape[0] / 2):]\n\ny_half_1 = y_train[:int(X_train.shape[0] / 2)]\ny_half_2 = y_train[int(X_train.shape[0] / 2):]\n\ncategorical_features = [\"building_id\", \"site_id\", \"meter\", \"primary_use\", \"hour\", \"weekday\"]\n\nd_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)\nd_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)\n\nwatchlist_1 = [d_half_1, d_half_2]\nwatchlist_2 = [d_half_2, d_half_1]\n\nparams = {\n    \"objective\": \"regression\",\n    \"boosting\": \"gbdt\",\n    \"num_leaves\": 40,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.85,\n    \"reg_lambda\": 2,\n    \"metric\": \"rmse\"\n}\n\nprint(\"Building model with first half and validating on second half:\")\nmodel_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)\n\nprint(\"Building model with second half and validating on first half:\")\nmodel_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:04:31.469111Z","iopub.execute_input":"2023-11-23T05:04:31.469508Z","iopub.status.idle":"2023-11-23T05:21:25.492160Z","shell.execute_reply.started":"2023-11-23T05:04:31.469437Z","shell.execute_reply":"2023-11-23T05:21:25.490803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Importance\nPlotting the feature importance from LGBM.","metadata":{}},{"cell_type":"code","source":"df_fimp_1 = pd.DataFrame()\ndf_fimp_1[\"feature\"] = X_train.columns.values\ndf_fimp_1[\"importance\"] = model_half_1.feature_importance()\ndf_fimp_1[\"half\"] = 1\n\ndf_fimp_2 = pd.DataFrame()\ndf_fimp_2[\"feature\"] = X_train.columns.values\ndf_fimp_2[\"importance\"] = model_half_2.feature_importance()\ndf_fimp_2[\"half\"] = 2\n\ndf_fimp = pd.concat([df_fimp_1, df_fimp_2], axis=0)\n\nplt.figure(figsize=(14, 7))\nsns.barplot(x=\"importance\", y=\"feature\", data=df_fimp.sort_values(by=\"importance\", ascending=False))\nplt.title(\"LightGBM Feature Importance\")\nplt.tight_layout()","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:21:25.495905Z","iopub.execute_input":"2023-11-23T05:21:25.497435Z","iopub.status.idle":"2023-11-23T05:21:26.238105Z","shell.execute_reply.started":"2023-11-23T05:21:25.497374Z","shell.execute_reply":"2023-11-23T05:21:26.237177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del X_train, y_train, X_half_1, X_half_2, y_half_1, y_half_2, d_half_1, d_half_2, watchlist_1, watchlist_2, df_fimp_1, df_fimp_2, df_fimp\ngc.collect()","metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2023-11-23T05:21:26.239879Z","iopub.execute_input":"2023-11-23T05:21:26.240369Z","iopub.status.idle":"2023-11-23T05:21:26.433966Z","shell.execute_reply.started":"2023-11-23T05:21:26.240199Z","shell.execute_reply":"2023-11-23T05:21:26.432875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preparing test data\nPreparing test data with same features as train data.","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(path_test)\nweather_test = pd.read_csv(path_weather_test)\n\ndf_test = reduce_mem_usage(df_test)\nweather_test = reduce_mem_usage(weather_test)\n\nX_test, row_ids = prepare_data(df_test, building, weather_test, test=True)","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-23T05:21:26.435455Z","iopub.execute_input":"2023-11-23T05:21:26.435763Z","iopub.status.idle":"2023-11-23T05:23:50.348743Z","shell.execute_reply.started":"2023-11-23T05:21:26.435683Z","shell.execute_reply":"2023-11-23T05:23:50.347608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df_test, building, weather_test\ngc.collect()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-23T05:23:50.350139Z","iopub.execute_input":"2023-11-23T05:23:50.350444Z","iopub.status.idle":"2023-11-23T05:23:50.455915Z","shell.execute_reply.started":"2023-11-23T05:23:50.350397Z","shell.execute_reply":"2023-11-23T05:23:50.454942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Scoring test data\nAveraging predictions from the two half train data models.","metadata":{}},{"cell_type":"code","source":"pred = np.expm1(model_half_1.predict(X_test, num_iteration=model_half_1.best_iteration)) / 2\n\ndel model_half_1\ngc.collect()\n\npred += np.expm1(model_half_2.predict(X_test, num_iteration=model_half_2.best_iteration)) / 2\n    \ndel model_half_2\ngc.collect()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-11-23T05:23:50.457541Z","iopub.execute_input":"2023-11-23T05:23:50.458128Z","iopub.status.idle":"2023-11-23T05:49:19.458457Z","shell.execute_reply.started":"2023-11-23T05:23:50.458050Z","shell.execute_reply":"2023-11-23T05:49:19.457102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission\nPreparing final file for submission.","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({\"row_id\": row_ids, \"meter_reading\": np.clip(pred, 0, a_max=None)})\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-11-23T05:49:19.460125Z","iopub.execute_input":"2023-11-23T05:49:19.460437Z","iopub.status.idle":"2023-11-23T05:53:47.580830Z","shell.execute_reply.started":"2023-11-23T05:49:19.460388Z","shell.execute_reply":"2023-11-23T05:53:47.579351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Credits:** The original R version: https://www.kaggle.com/kailex/ac-dc","metadata":{}},{"cell_type":"markdown","source":"If you're interested in trying out site specific models, check out my [Divide and Conquer notebook](https://www.kaggle.com/rohanrao/ashrae-divide-and-conquer/output).","metadata":{}}]}