{"cells":[{"metadata":{},"cell_type":"markdown","source":"**Abstract**\n\n* KFold LightGBM regression works without \"building_id\".(score; 1.10)\n* The importance of \"site_id\" cannot be high even though \"building_id\" is dropped."},{"metadata":{},"cell_type":"markdown","source":"**Data Preparation**\n\nAfter cleaning data, drop \"building_id\"."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport random\nimport csv\nimport sys\n\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold\n\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\npath_data = \"/kaggle/input/ashrae-energy-prediction/\"\nsave_grid = \"grid_search.csv\"\npath_train = path_data + \"train.csv\"\npath_test = path_data + \"test.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"\npath_weather_test = path_data + \"weather_test.csv\"\n\nplt.style.use(\"seaborn\")\nsns.set(font_scale=1)\n\ndf_train = pd.read_csv(path_train)\n\nbuilding = pd.read_csv(path_building)\nle = LabelEncoder()\nbuilding.primary_use = le.fit_transform(building.primary_use)\n\nweather_train = pd.read_csv(path_weather_train)\n\ndf_train = reduce_mem_usage(df_train)\nbuilding = reduce_mem_usage(building)\nweather_train = reduce_mem_usage(weather_train)\n\ndef prepare_data(X, building_data, weather_data, test=False):\n    \"\"\"\n    Preparing final dataset with all features.\n    \"\"\"\n    \n    X = X.merge(building_data, on=\"building_id\", how=\"left\")\n    X = X.merge(weather_data, on=[\"site_id\", \"timestamp\"], how=\"left\")\n    \n    X.timestamp = pd.to_datetime(X.timestamp, format=\"%Y-%m-%d %H:%M:%S\")\n    X.square_feet = np.log1p(X.square_feet)\n    \n    if not test:\n        X.sort_values(\"timestamp\", inplace=True)\n        X.reset_index(drop=True, inplace=True)\n    \n    gc.collect()\n    \n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n                \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n                \"2017-01-01\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n                \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n                \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n                \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n                \"2019-01-01\"]\n    \n    X[\"hour\"] = X.timestamp.dt.hour\n    X[\"weekday\"] = X.timestamp.dt.weekday\n    X[\"is_holiday\"] = (X.timestamp.dt.date.astype(\"str\").isin(holidays)).astype(int)\n    \n    drop_features = [ \"sea_level_pressure\", \"wind_direction\", \"wind_speed\",\"year_built\",\"floor_count\",\"cloud_coverage\", \"precip_depth_1_hr\"]\n\n    X.drop(drop_features, axis=1, inplace=True)\n\n    if test:\n        row_ids = X.row_id\n        X.drop(\"row_id\", axis=1, inplace=True)\n        return X, row_ids\n    else:\n        y = np.log1p(X.meter_reading)\n        X.drop(\"meter_reading\", axis=1, inplace=True)\n        return X, y\n\nX_train, y_train = prepare_data(df_train, building, weather_train)\n\ndel df_train, weather_train\ngc.collect()\n\ny_train = y_train[~((X_train.building_id <= 104) & (X_train.meter==0) & (X_train.timestamp<='2016-05-20'))]\nX_train = X_train[~((X_train.building_id <= 104) & (X_train.meter==0) & (X_train.timestamp<='2016-05-20'))]\ny_train = y_train[X_train.building_id != 1099]\nX_train = X_train[X_train.building_id != 1099]\nX_train.drop(\"timestamp\", axis=1, inplace=True)\nX_train.drop(\"building_id\", axis=1, inplace=True)\n\ndf_test = pd.read_csv(path_test)\nweather_test = pd.read_csv(path_weather_test)\ndf_test = reduce_mem_usage(df_test)\nweather_test = reduce_mem_usage(weather_test)\n\nX_test, row_ids = prepare_data(df_test, building, weather_test, test=True)\nX_test.drop(\"timestamp\", axis=1, inplace=True)\nX_test.drop(\"building_id\", axis=1, inplace=True)\n\nnsplits = 3\n\nfolds = KFold(n_splits=nsplits)\ncategorical_features = [\"site_id\", \"meter\",  \"hour\", \"weekday\",\"primary_use\"]\nparams = {\n    \"objective\": \"regression\",\n    \"boosting\": \"gbdt\",\n    \"max_depth\": -1,\n    \"num_leaves\": 1000,\n    \"learning_rate\": 0.1,\n    \"feature_fraction\": 0.95,\n    \"reg_lambda\": 2,\n    \"max_bin\": 550,\n    \"metric\": \"rmse\"\n}\n\npred = np.zeros(len(X_test))\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**KFold LightGBM and feature Importance**\n\nHere, we use KFold LightGBM(K = 3).\nThe most important feature is \"square_feet\".\nI first thought that \"site_id\" might replace \"building_id\", but the importance of \"site_id\" is still low even after removing \"building_id\".\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_fimp = pd.DataFrame()\nfor tr_idx,va_idx in folds.split(X_train,y_train):\n    X_half_1 = X_train.iloc[tr_idx]\n    y_half_1 = y_train.iloc[tr_idx]\n    X_half_2 = X_train.iloc[va_idx]\n    y_half_2 = y_train.iloc[va_idx]\n\n    d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)\n    d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)\n\n    print(\"Building model with first half and validating on second half:\")\n    model_half = lgb.train(params, train_set=d_half_1, num_boost_round=300, valid_sets=[d_half_1,d_half_2], verbose_eval=100, early_stopping_rounds=50)\n    \n    pred += np.expm1(model_half.predict(X_test, num_iteration=model_half.best_iteration)) / nsplits\n    \n    df_fimp_1 = pd.DataFrame()\n    df_fimp_1[\"feature\"] = X_train.columns.values\n    df_fimp_1[\"importance\"] = model_half.feature_importance()\n    \n    df_fimp = pd.concat([df_fimp, df_fimp_1], axis=0)\n    del model_half\n    gc.collect()\n\nplt.figure(figsize=(14, 7))\nsns.barplot(x=\"importance\", y=\"feature\", data=df_fimp.sort_values(by=\"importance\", ascending=False))\nplt.title(\"LightGBM Feature Importance\")\nplt.tight_layout()\n\npred = pred*0.90\nsubmission = pd.DataFrame({\"row_id\": row_ids, \"meter_reading\": np.clip(pred, 0, a_max=None)})\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}