{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# ASHRAE ENERGY PREDICTION III:","metadata":{"id":"PRDnm0lhzEyp"}},{"cell_type":"markdown","source":"# From Previous Notebook ...","metadata":{}},{"cell_type":"markdown","source":"## - Importing Necessary Libraries","metadata":{}},{"cell_type":"code","source":"# Importing needed libraries to be used throughout the project\n\n%matplotlib inline\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import FunctionTransformer\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn import utils\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split,KFold,GroupKFold\nimport lightgbm as lgb\nimport gc\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"id":"5h4-2ayuABWJ","execution":{"iopub.status.busy":"2022-01-13T13:19:17.201542Z","iopub.execute_input":"2022-01-13T13:19:17.202360Z","iopub.status.idle":"2022-01-13T13:19:19.784480Z","shell.execute_reply.started":"2022-01-13T13:19:17.202209Z","shell.execute_reply":"2022-01-13T13:19:19.783652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## - Loading and Reducing Memory Usage of Data","metadata":{"id":"OPc_EXwyA9b6"}},{"cell_type":"code","source":"data_path = \"/kaggle/input/ashrae-energy-prediction/\"\n\ntrain_path = data_path + \"train.csv\"\n\nbuilding_path = data_path + \"building_metadata.csv\"\n\nweather_train_path = data_path + \"weather_train.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:19:21.483730Z","iopub.execute_input":"2022-01-13T13:19:21.484787Z","iopub.status.idle":"2022-01-13T13:19:21.490105Z","shell.execute_reply.started":"2022-01-13T13:19:21.484704Z","shell.execute_reply":"2022-01-13T13:19:21.489043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(train_path)\n\nbuilding_data = pd.read_csv(building_path)\n\nweather_train_data = pd.read_csv(weather_train_path)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:19:24.004373Z","iopub.execute_input":"2022-01-13T13:19:24.005256Z","iopub.status.idle":"2022-01-13T13:19:38.179688Z","shell.execute_reply.started":"2022-01-13T13:19:24.005207Z","shell.execute_reply":"2022-01-13T13:19:38.178725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting data into feather format since some dataframes are too large and take a long time to load\n# This method is inspired from the kaggle notebook titled: ASHRAE: feather format for fast loading\n# Which is found at: https://www.kaggle.com/corochann/ashrae-feather-format-for-fast-loading\n\ntrain_data.to_feather('train_data.feather')\n\nbuilding_data.to_feather('building_data.feather')\n\nweather_train_data.to_feather('weather_train_data.feather')","metadata":{"id":"W3dUkNtVeSfP","execution":{"iopub.status.busy":"2022-01-13T13:19:38.181498Z","iopub.execute_input":"2022-01-13T13:19:38.181820Z","iopub.status.idle":"2022-01-13T13:19:42.669662Z","shell.execute_reply.started":"2022-01-13T13:19:38.181778Z","shell.execute_reply":"2022-01-13T13:19:42.668396Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_feather('train_data.feather')\n\nbuilding_data = pd.read_feather('building_data.feather')\n\nweather_train_data = pd.read_feather('weather_train_data.feather')","metadata":{"id":"K-xSJPieegWJ","execution":{"iopub.status.busy":"2022-01-13T13:19:42.671252Z","iopub.execute_input":"2022-01-13T13:19:42.671517Z","iopub.status.idle":"2022-01-13T13:19:47.527233Z","shell.execute_reply.started":"2022-01-13T13:19:42.671486Z","shell.execute_reply":"2022-01-13T13:19:47.525834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This function helps in optimizing the memory used by the dataframes by by modifying/altering thedatatype of each column.\n# This method is inspired from the kaggle notebook titled: load data (reduce memory usage)\n# Which is found at: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\n\ndef reduce_mem_usage(df, df_name):\n\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n\n    print('Memory usage of {} is reduced by {:.2f} %. Usage dropped from {:.2f} MB to {:.2f} MB.'.format(df_name, (100 * (start_mem - end_mem) / start_mem), start_mem, end_mem))\n    \n    return df\n","metadata":{"id":"v3yIjOevKKKY","execution":{"iopub.status.busy":"2022-01-13T13:19:47.535383Z","iopub.execute_input":"2022-01-13T13:19:47.537146Z","iopub.status.idle":"2022-01-13T13:19:47.588590Z","shell.execute_reply.started":"2022-01-13T13:19:47.536845Z","shell.execute_reply":"2022-01-13T13:19:47.584707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reducing Memory Usage of Data\n\ntrain_data = reduce_mem_usage(train_data, 'Train Data')\n\nbuilding_data = reduce_mem_usage(building_data, 'Building Data')\n\nweather_train_data = reduce_mem_usage(weather_train_data, 'Weather Train Data')","metadata":{"id":"XYZL4jkJeVyB","outputId":"01e04941-ed2c-47a7-d13f-80fe439f5148","execution":{"iopub.status.busy":"2022-01-13T13:19:47.592364Z","iopub.execute_input":"2022-01-13T13:19:47.593635Z","iopub.status.idle":"2022-01-13T13:19:53.918063Z","shell.execute_reply.started":"2022-01-13T13:19:47.593563Z","shell.execute_reply":"2022-01-13T13:19:53.916490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merging Data\n\ntrain = train_data.merge(building_data, on='building_id', how='left')\ntrain = train.merge(weather_train_data, on=['site_id', 'timestamp'], how='left')","metadata":{"id":"y67TusNcgV3a","execution":{"iopub.status.busy":"2022-01-13T13:19:53.920069Z","iopub.execute_input":"2022-01-13T13:19:53.920537Z","iopub.status.idle":"2022-01-13T13:20:07.085788Z","shell.execute_reply.started":"2022-01-13T13:19:53.920487Z","shell.execute_reply":"2022-01-13T13:20:07.084182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Breaking Timestamp into Hour, Day, Month, Year\n\n# This function firstly converts timestamp to date and then breaks down date into 6 new columns: hour, day, dayOfWeek, dayOfYear, month and year\n\ndef breakdown_timestamp(dataframe):\n    \n    dataframe['timestamp']= pd.to_datetime(dataframe['timestamp'])\n\n    dataframe['hour']= np.uint8(dataframe['timestamp'].dt.hour)\n    \n    dataframe['day']= np.uint16(dataframe['timestamp'].dt.day)\n    dataframe['dayofweek']= np.uint8(dataframe['timestamp'].dt.dayofweek)\n    dataframe['dayofyear']= np.uint16(dataframe['timestamp'].dt.dayofyear)\n\n    dataframe['month']= np.uint8(dataframe['timestamp'].dt.month)\n\n    dataframe['year']= np.uint16(dataframe['timestamp'].dt.year)\n    \n    return dataframe","metadata":{"id":"6A03K8Z5uydO","execution":{"iopub.status.busy":"2022-01-13T13:20:07.089242Z","iopub.execute_input":"2022-01-13T13:20:07.090019Z","iopub.status.idle":"2022-01-13T13:20:07.101258Z","shell.execute_reply.started":"2022-01-13T13:20:07.089957Z","shell.execute_reply":"2022-01-13T13:20:07.100194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = breakdown_timestamp(train)","metadata":{"id":"0H9LBPwzuylB","execution":{"iopub.status.busy":"2022-01-13T13:20:07.102920Z","iopub.execute_input":"2022-01-13T13:20:07.103177Z","iopub.status.idle":"2022-01-13T13:20:20.522198Z","shell.execute_reply.started":"2022-01-13T13:20:07.103145Z","shell.execute_reply":"2022-01-13T13:20:20.521312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## - Applying Log Transformation to 'Meter Reading and 'Square Feet'","metadata":{}},{"cell_type":"code","source":"train['meter_reading'] = np.log1p(train['meter_reading'])","metadata":{"papermill":{"duration":0.43745,"end_time":"2022-01-09T19:16:27.491494","exception":false,"start_time":"2022-01-09T19:16:27.054044","status":"completed"},"tags":[],"id":"abfa1395","execution":{"iopub.status.busy":"2022-01-13T13:20:20.523779Z","iopub.execute_input":"2022-01-13T13:20:20.524607Z","iopub.status.idle":"2022-01-13T13:20:20.973413Z","shell.execute_reply.started":"2022-01-13T13:20:20.524568Z","shell.execute_reply":"2022-01-13T13:20:20.972620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['square_feet'] = np.log1p(train['square_feet'])","metadata":{"id":"kGOBwLjJ-cv0","outputId":"4d548e6b-1db1-4372-8d9b-3c6164911a1b","execution":{"iopub.status.busy":"2022-01-13T13:20:20.975909Z","iopub.execute_input":"2022-01-13T13:20:20.976218Z","iopub.status.idle":"2022-01-13T13:20:21.492733Z","shell.execute_reply.started":"2022-01-13T13:20:20.976182Z","shell.execute_reply":"2022-01-13T13:20:21.491673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Feature Engineering","metadata":{"id":"BXBUpjO6Sydb"}},{"cell_type":"markdown","source":"**Preprocessing and Data Preperation Steps:**\n1. Removing rows with zero meter readings as they may indicate an innacurate or unavailable reading.\n2. Setting a threshold limit to drop columns with more than 50 % missing values.\n3. Filling In missing values using column median.\n4. Adding new features.\n5. Encoding categorical data.","metadata":{"id":"vTS_jBtn1eMi"}},{"cell_type":"markdown","source":"## 5.1 Removing Zero Meter Readings","metadata":{"id":"OQo3_iiZ2lfo"}},{"cell_type":"code","source":"zero_meter_readings = list(train[train['meter_reading'] == 0].index)\ntrain.drop(zero_meter_readings, axis = 0, inplace = True)","metadata":{"id":"pqlS9-yVtl3L","execution":{"iopub.status.busy":"2022-01-13T13:20:32.697033Z","iopub.execute_input":"2022-01-13T13:20:32.697579Z","iopub.status.idle":"2022-01-13T13:20:40.797993Z","shell.execute_reply.started":"2022-01-13T13:20:32.697486Z","shell.execute_reply":"2022-01-13T13:20:40.796723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('New Shape of Train Data:',train.shape)","metadata":{"id":"wg1FLIeu271x","execution":{"iopub.status.busy":"2022-01-13T13:20:40.800402Z","iopub.execute_input":"2022-01-13T13:20:40.800725Z","iopub.status.idle":"2022-01-13T13:20:40.806516Z","shell.execute_reply.started":"2022-01-13T13:20:40.800689Z","shell.execute_reply":"2022-01-13T13:20:40.805414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.2 Dropping Columns with More than 50 % Missing Values","metadata":{"id":"kor2kWsO2ugc"}},{"cell_type":"code","source":"threshold = len(train) * 0.5\ntrain.dropna(axis=1, thresh = threshold, inplace = True)","metadata":{"id":"pOB8kEnHgLUc","execution":{"iopub.status.busy":"2022-01-13T13:20:45.671579Z","iopub.execute_input":"2022-01-13T13:20:45.672574Z","iopub.status.idle":"2022-01-13T13:20:48.573602Z","shell.execute_reply.started":"2022-01-13T13:20:45.672527Z","shell.execute_reply":"2022-01-13T13:20:48.572346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('New Shape of Train Data:',train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:20:49.728680Z","iopub.execute_input":"2022-01-13T13:20:49.729397Z","iopub.status.idle":"2022-01-13T13:20:49.734199Z","shell.execute_reply.started":"2022-01-13T13:20:49.729343Z","shell.execute_reply":"2022-01-13T13:20:49.733547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.3 Filling Missing Values","metadata":{"id":"QJ_2ZHI93TE1"}},{"cell_type":"code","source":"train['cloud_coverage'].fillna(train['cloud_coverage'].median(), inplace=True)\ntrain['sea_level_pressure'].fillna(train['sea_level_pressure'].median(), inplace=True)\ntrain['precip_depth_1_hr'].fillna(train['precip_depth_1_hr'].median(), inplace=True)\ntrain['wind_direction'].fillna(train['wind_direction'].median(), inplace=True)\ntrain['wind_speed'].fillna(train['wind_speed'].median(), inplace=True)\ntrain['dew_temperature'].fillna(train['dew_temperature'].median(), inplace=True)\ntrain['air_temperature'].fillna(train['air_temperature'].median(), inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:20:55.691209Z","iopub.execute_input":"2022-01-13T13:20:55.691989Z","iopub.status.idle":"2022-01-13T13:20:58.624563Z","shell.execute_reply.started":"2022-01-13T13:20:55.691924Z","shell.execute_reply":"2022-01-13T13:20:58.623287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isnull().sum() / len(train)","metadata":{"id":"2xxuJafB3e-E","execution":{"iopub.status.busy":"2022-01-13T13:20:58.626385Z","iopub.execute_input":"2022-01-13T13:20:58.626675Z","iopub.status.idle":"2022-01-13T13:20:59.501975Z","shell.execute_reply.started":"2022-01-13T13:20:58.626642Z","shell.execute_reply":"2022-01-13T13:20:59.500901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.4 Adding New Features","metadata":{"id":"TS0ln1PI31hL"}},{"cell_type":"code","source":"# adding new feature from existing ones to get better results\n\ntrain['season'] = train['timestamp'].apply(lambda x: 'Spring' if x.month==3 or x.month==4 or x.month==5 else \n                                                  'Summer' if x.month==6 or x.month==7 or x.month==8 else \n                                                  'Autumn' if x.month==9 or x.month==10 or x.month==11 else \n                                                  'Winter')\n\ntrain['isDayTime'] = train['timestamp'].apply(lambda x: 1 if x.hour >=6 and x.hour <=18 else 0)","metadata":{"id":"iF7oK_Gqymp6","execution":{"iopub.status.busy":"2022-01-13T13:21:00.024542Z","iopub.execute_input":"2022-01-13T13:21:00.024928Z","iopub.status.idle":"2022-01-13T13:21:01.879631Z","shell.execute_reply.started":"2022-01-13T13:21:00.024894Z","shell.execute_reply":"2022-01-13T13:21:01.878927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:21:04.236526Z","iopub.execute_input":"2022-01-13T13:21:04.237457Z","iopub.status.idle":"2022-01-13T13:21:04.276025Z","shell.execute_reply.started":"2022-01-13T13:21:04.237410Z","shell.execute_reply":"2022-01-13T13:21:04.275309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 5.5 Encoding categorical data","metadata":{"id":"KkW0Ansa4DXn"}},{"cell_type":"code","source":"categorical_features = ['primary_use', 'season']","metadata":{"id":"hnobTOL7yN4z","execution":{"iopub.status.busy":"2022-01-13T13:21:06.653558Z","iopub.execute_input":"2022-01-13T13:21:06.654286Z","iopub.status.idle":"2022-01-13T13:21:06.659519Z","shell.execute_reply.started":"2022-01-13T13:21:06.654231Z","shell.execute_reply":"2022-01-13T13:21:06.658463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = preprocessing.LabelEncoder()\n\nfor i in categorical_features:\n    \n    train[i] = encoder.fit_transform(train[i])\n    \nprint (train.info())","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:21:08.255248Z","iopub.execute_input":"2022-01-13T13:21:08.255934Z","iopub.status.idle":"2022-01-13T13:21:15.131159Z","shell.execute_reply.started":"2022-01-13T13:21:08.255889Z","shell.execute_reply":"2022-01-13T13:21:15.130103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:21:17.703604Z","iopub.execute_input":"2022-01-13T13:21:17.703971Z","iopub.status.idle":"2022-01-13T13:21:17.737207Z","shell.execute_reply.started":"2022-01-13T13:21:17.703937Z","shell.execute_reply":"2022-01-13T13:21:17.735637Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. Building Transformation Pipeline","metadata":{"id":"VpZYVBGxzsGN"}},{"cell_type":"markdown","source":"Creating a pipline with all the transformation done to the data.\n\n**Note**\n- Unfortunately, when running the pipline the session always crashes, so we didn't get an opportunity to fully test it out and that's why it's commented.","metadata":{}},{"cell_type":"code","source":"#numerical_features = train.select_dtypes(include=[np.number])\n#numerical_features = list(numerical_features)","metadata":{"id":"xFtqJVDRjAPm","execution":{"iopub.status.busy":"2022-01-13T13:24:28.032695Z","iopub.execute_input":"2022-01-13T13:24:28.033092Z","iopub.status.idle":"2022-01-13T13:24:28.038009Z","shell.execute_reply.started":"2022-01-13T13:24:28.033052Z","shell.execute_reply":"2022-01-13T13:24:28.036864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#meter_reading_ix = train.columns.get_loc('meter_reading')\n\n#def drop_zero_readings(X):\n\n#        zero_meter_readings = list(X[X.iloc[:, meter_reading_ix] == 0.0].index)\n#        X.drop(zero_meter_readings, axis = 0, inplace = True)\n        \n#        return X","metadata":{"id":"gojZT8RPzN7w","execution":{"iopub.status.busy":"2022-01-13T13:24:28.420889Z","iopub.execute_input":"2022-01-13T13:24:28.421395Z","iopub.status.idle":"2022-01-13T13:24:28.426094Z","shell.execute_reply.started":"2022-01-13T13:24:28.421359Z","shell.execute_reply":"2022-01-13T13:24:28.425253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#def drop_mostly_null_features(X, missing_factor = 0.5):\n\n#        threshold = len(X) * missing_factor\n#        X.dropna(axis=1, thresh = threshold, inplace = True)\n        \n#        return np.c_[X]","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:24:29.061608Z","iopub.execute_input":"2022-01-13T13:24:29.062189Z","iopub.status.idle":"2022-01-13T13:24:29.067101Z","shell.execute_reply.started":"2022-01-13T13:24:29.062146Z","shell.execute_reply":"2022-01-13T13:24:29.065972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#timestamp_ix = train.columns.get_loc(\"timestamp\")\n\n#def add_extra_features(X):\n    \n#    season = X[:, timestamp_ix].apply(lambda x: 'Spring' if x.month==3 or x.month==4 or x.month==5 else \n#                                                    'Summer' if x.month==6 or x.month==7 or x.month==8 else \n#                                                    'Autumn' if x.month==9 or x.month==10 or x.month==11 else \n#                                                    'Winter')\n#    \n#    isDayTime = X[:, timestamp_ix].apply(lambda x: 1 if x.hour >=6 and x.hour <=18 else 0)\n\n#    return np.c_[X, season, isDayTime]","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:24:29.563048Z","iopub.execute_input":"2022-01-13T13:24:29.563618Z","iopub.status.idle":"2022-01-13T13:24:29.569003Z","shell.execute_reply.started":"2022-01-13T13:24:29.563566Z","shell.execute_reply":"2022-01-13T13:24:29.567591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#num_pipeline = Pipeline([\n  \n#        ('drop_zero_readings', FunctionTransformer(drop_zero_readings, validate=False)),\n#        ('drop_mostly_null_features', FunctionTransformer(drop_mostly_null_features, validate=False)),\n#        ('imputer', SimpleImputer(strategy=\"median\")),\n#        ('attribs_adder', FunctionTransformer(add_extra_features, validate=False)),\n#    ])","metadata":{"id":"ke2SJetzDkyb","execution":{"iopub.status.busy":"2022-01-13T13:24:30.371656Z","iopub.execute_input":"2022-01-13T13:24:30.371989Z","iopub.status.idle":"2022-01-13T13:24:30.376261Z","shell.execute_reply.started":"2022-01-13T13:24:30.371956Z","shell.execute_reply":"2022-01-13T13:24:30.375076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#categorical_features = ['primary_use', 'season']","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:24:31.386962Z","iopub.execute_input":"2022-01-13T13:24:31.387407Z","iopub.status.idle":"2022-01-13T13:24:31.392000Z","shell.execute_reply.started":"2022-01-13T13:24:31.387373Z","shell.execute_reply":"2022-01-13T13:24:31.390703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#full_pipeline = ColumnTransformer([\n#        (\"num\", num_pipeline, numerical_features),\n#        (\"cat\", OneHotEncoder(handle_unknown='ignore', sparse=False), categorical_features),\n#   ])","metadata":{"id":"JDRxiosVDkyk","execution":{"iopub.status.busy":"2022-01-13T13:24:32.452452Z","iopub.execute_input":"2022-01-13T13:24:32.453331Z","iopub.status.idle":"2022-01-13T13:24:32.457587Z","shell.execute_reply.started":"2022-01-13T13:24:32.453275Z","shell.execute_reply":"2022-01-13T13:24:32.456558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#prepared_train_data = full_pipeline.fit_transform(train)","metadata":{"id":"zHkgtpZcvEOU","execution":{"iopub.status.busy":"2022-01-13T13:24:33.439866Z","iopub.execute_input":"2022-01-13T13:24:33.440158Z","iopub.status.idle":"2022-01-13T13:24:33.444226Z","shell.execute_reply.started":"2022-01-13T13:24:33.440128Z","shell.execute_reply":"2022-01-13T13:24:33.443035Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 7. Features Selection\n","metadata":{"id":"QPxtfJDbUuC1"}},{"cell_type":"markdown","source":"**Using LGBM to Select the Best Features that Actually Affect the Model Results**","metadata":{}},{"cell_type":"code","source":"train = train.drop(['timestamp'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:24:56.725066Z","iopub.execute_input":"2022-01-13T13:24:56.725628Z","iopub.status.idle":"2022-01-13T13:24:59.089763Z","shell.execute_reply.started":"2022-01-13T13:24:56.725568Z","shell.execute_reply":"2022-01-13T13:24:59.088775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reducing the Memory Usage Again**","metadata":{}},{"cell_type":"code","source":"reduced_train_data = reduce_mem_usage(train, 'Train Data')","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:25:00.178497Z","iopub.execute_input":"2022-01-13T13:25:00.178832Z","iopub.status.idle":"2022-01-13T13:25:04.835087Z","shell.execute_reply.started":"2022-01-13T13:25:00.178797Z","shell.execute_reply":"2022-01-13T13:25:04.833849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating Training and Testing Data for the Model to Work With**","metadata":{}},{"cell_type":"code","source":"x = reduced_train_data.drop(['meter_reading'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-01-13T13:25:04.837130Z","iopub.execute_input":"2022-01-13T13:25:04.837991Z","iopub.status.idle":"2022-01-13T13:25:06.827346Z","shell.execute_reply.started":"2022-01-13T13:25:04.837927Z","shell.execute_reply":"2022-01-13T13:25:06.826527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = reduced_train_data['meter_reading']","metadata":{"id":"Wp45qMaixpza","execution":{"iopub.status.busy":"2022-01-13T13:25:06.828867Z","iopub.execute_input":"2022-01-13T13:25:06.829556Z","iopub.status.idle":"2022-01-13T13:25:06.835152Z","shell.execute_reply.started":"2022-01-13T13:25:06.829422Z","shell.execute_reply":"2022-01-13T13:25:06.834115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Running The Feature Selection Model**","metadata":{}},{"cell_type":"code","source":"parameters = {\n    \n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': {'rmse'},\n    'subsample': 0.2,\n    'learning_rate': 0.9,  # 0.3 #0.5 #0.6 \n    'feature_fraction': 0.9, #0.5 #0.6 #0.8\n    'bagging_fraction': 0.9, #0.8\n    'alpha': 0.1,\n    'lambda': 0.1,\n    'num_leaves': 110, #110 #100 #150 large, but over-fitting\n    'max_bin': 66,  #60 #50 # large,but slower,over-fitting\n    'max_depth': 10, # deal with over-fitting\n    'min_data_in_leaf': 30, # deal with over-fitting\n\n}\n\n\nkf = KFold(n_splits = 5, shuffle = True, random_state = 42)\n\nmodels = []\n\nfor train_index, val_index in kf.split(x):\n    train_x = x.iloc[train_index]\n    val_x = x.iloc[val_index]\n    train_y = y.iloc[train_index]\n    val_y = y.iloc[val_index]\n    lgb_train = lgb.Dataset(train_x, train_y)\n    lgb_eval = lgb.Dataset(val_x, val_y)\n    gbm = lgb.train(parameters,\n                    lgb_train,\n                    num_boost_round = 10, \n                    valid_sets = (lgb_train, lgb_eval),\n                    early_stopping_rounds = 10,\n                    verbose_eval = 10) \n    \n    models.append(gbm)\n    \ngc.collect()","metadata":{"id":"CgPhjasCxzIL","execution":{"iopub.status.busy":"2022-01-13T13:25:06.837182Z","iopub.execute_input":"2022-01-13T13:25:06.837534Z","iopub.status.idle":"2022-01-13T13:27:08.011730Z","shell.execute_reply.started":"2022-01-13T13:25:06.837499Z","shell.execute_reply":"2022-01-13T13:27:08.010699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualizing the Feature Importance Highlighted by the Model**","metadata":{}},{"cell_type":"code","source":"feature_imp = pd.DataFrame(sorted(zip(gbm.feature_importance(), gbm.feature_name()),reverse = True), columns=['Value','Feature'])\n\nplt.figure(figsize=(16, 8))\n\nsns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False))\n\nplt.title('LightGBM Features Importance Plot')\n\nplt.tight_layout()\n\nplt.show()","metadata":{"id":"E6QRuOskyEPP","execution":{"iopub.status.busy":"2022-01-13T13:29:00.073971Z","iopub.execute_input":"2022-01-13T13:29:00.074779Z","iopub.status.idle":"2022-01-13T13:29:00.579425Z","shell.execute_reply.started":"2022-01-13T13:29:00.074708Z","shell.execute_reply":"2022-01-13T13:29:00.578508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Removing the Least Important features as Indicated by the Feature Selection Method**","metadata":{}},{"cell_type":"code","source":"new_data = reduced_train_data[['building_id','square_feet','primary_use','meter','site_id','air_temperature','dayofyear','hour','isDayTime','dew_temperature','dayofweek', 'meter_reading']]","metadata":{"id":"zn8fH0M72yG6","execution":{"iopub.status.busy":"2022-01-13T13:29:41.873078Z","iopub.execute_input":"2022-01-13T13:29:41.873899Z","iopub.status.idle":"2022-01-13T13:29:42.804641Z","shell.execute_reply.started":"2022-01-13T13:29:41.873843Z","shell.execute_reply":"2022-01-13T13:29:42.803694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating a Heatmap to See the Correlated Features in the New Data**","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(20,10)) \n\nsns.heatmap(new_data.corr(), annot=True, vmin = -1,vmax = 1,linewidths = 1,annot_kws={\"size\": 5},fmt=\"g\",ax=ax,cmap=\"YlGnBu\").set_title('Correlation Matrix')","metadata":{"id":"tQ2N8irc26Yb","execution":{"iopub.status.busy":"2022-01-13T13:29:46.750149Z","iopub.execute_input":"2022-01-13T13:29:46.750471Z","iopub.status.idle":"2022-01-13T13:30:05.857043Z","shell.execute_reply.started":"2022-01-13T13:29:46.750430Z","shell.execute_reply":"2022-01-13T13:30:05.856271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Removing Highly Correlated Features**","metadata":{"id":"23r6ypbLxpex"}},{"cell_type":"code","source":"new_data.drop(['site_id','dew_temperature'],axis = 1,inplace=True)","metadata":{"id":"oLudnHxa3B9P","execution":{"iopub.status.busy":"2022-01-13T13:30:43.260102Z","iopub.execute_input":"2022-01-13T13:30:43.260993Z","iopub.status.idle":"2022-01-13T13:30:44.552038Z","shell.execute_reply.started":"2022-01-13T13:30:43.260909Z","shell.execute_reply":"2022-01-13T13:30:44.551150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating another Heatmap to Verify that there are no Further Improvements to be Done**","metadata":{}},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(20,10)) \n\nsns.heatmap(new_data.corr(), annot=True, vmin = -1,vmax = 1,linewidths=1,annot_kws={\"size\": 5},fmt=\"g\",ax=ax,cmap=\"YlGnBu\").set_title('Correlation Matrix')","metadata":{"id":"ji1toMXA3PpN","execution":{"iopub.status.busy":"2022-01-13T13:30:48.582719Z","iopub.execute_input":"2022-01-13T13:30:48.583289Z","iopub.status.idle":"2022-01-13T13:31:02.346763Z","shell.execute_reply.started":"2022-01-13T13:30:48.583249Z","shell.execute_reply":"2022-01-13T13:31:02.346043Z"},"trusted":true},"execution_count":null,"outputs":[]}]}