{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":9994,"databundleVersionId":752467,"sourceType":"competition"}],"dockerImageVersionId":30626,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Importing the libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\n\n# Import libraries for plotting\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.patches as patches\nfrom plotly import tools, subplots\nimport plotly.offline as py\n\n# Configure the libraries\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\npd.set_option('display.max_columns', 150)\n\npy.init_notebook_mode(connected=True)\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nimport os,random, math, psutil, pickle    ","metadata":{"_kg_hide-output":false,"execution":{"iopub.status.busy":"2024-01-26T20:41:50.532960Z","iopub.execute_input":"2024-01-26T20:41:50.533351Z","iopub.status.idle":"2024-01-26T20:41:53.662031Z","shell.execute_reply.started":"2024-01-26T20:41:50.533320Z","shell.execute_reply":"2024-01-26T20:41:53.660495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Loading the data","metadata":{}},{"cell_type":"code","source":"# Input data files are available in the read-only \"../input/\" directory\n# Running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output\n# when you create a version using \"Save & Run All\" ","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:41:53.664350Z","iopub.execute_input":"2024-01-26T20:41:53.665144Z","iopub.status.idle":"2024-01-26T20:41:53.674823Z","shell.execute_reply.started":"2024-01-26T20:41:53.665101Z","shell.execute_reply":"2024-01-26T20:41:53.673624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root = '../input/ashrae-energy-prediction/'\ntrain = pd.read_csv(root + 'train.csv')\n\nweather_train = pd.read_csv(root + 'weather_train.csv')\nweather_test_df = pd.read_csv(root + 'weather_test.csv')\nbuilding_meta_df = pd.read_csv(root + 'building_metadata.csv')\n\ntest_df = pd.read_csv(root + 'test.csv')\nsample_submission = pd.read_csv(root + 'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:41:53.676593Z","iopub.execute_input":"2024-01-26T20:41:53.677461Z","iopub.status.idle":"2024-01-26T20:42:59.610294Z","shell.execute_reply.started":"2024-01-26T20:41:53.677421Z","shell.execute_reply":"2024-01-26T20:42:59.609158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Size of train data', train.shape)\nprint('Size of weather_train data', weather_train.shape)\nprint('Size of weather_test_df data', weather_test_df.shape)\nprint('Size of building_meta_df data', building_meta_df.shape)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:42:59.613547Z","iopub.execute_input":"2024-01-26T20:42:59.614807Z","iopub.status.idle":"2024-01-26T20:42:59.621741Z","shell.execute_reply.started":"2024-01-26T20:42:59.614765Z","shell.execute_reply":"2024-01-26T20:42:59.620677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge the dataframes on the key fields to get the full dataset\ntrain_full = train.merge(building_meta_df, on='building_id', how='left')\ntest_full = test_df.merge(building_meta_df, on='building_id', how='left')\n\ntrain_full = train_full.merge(weather_train, on=['site_id', 'timestamp'], how='left')\ntest_full = test_full.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:42:59.622792Z","iopub.execute_input":"2024-01-26T20:42:59.623155Z","iopub.status.idle":"2024-01-26T20:43:32.998771Z","shell.execute_reply.started":"2024-01-26T20:42:59.623116Z","shell.execute_reply":"2024-01-26T20:43:32.997617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Truncate all floats to four decimal places to save memory","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                # Choose appropriate reduced int type for integer columns\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else: \n                # Choose appropriate reduced float type for floating point columns\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64) \n                    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-26T20:43:33.000137Z","iopub.execute_input":"2024-01-26T20:43:33.000454Z","iopub.status.idle":"2024-01-26T20:43:33.014765Z","shell.execute_reply.started":"2024-01-26T20:43:33.000426Z","shell.execute_reply":"2024-01-26T20:43:33.013738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reduce memory usage\ntrain = reduce_mem_usage(train)\ntest_df = reduce_mem_usage(test_df)\n\ntrain_full = reduce_mem_usage(train_full)\ntest_full = reduce_mem_usage(test_full)\n\nweather_train = reduce_mem_usage(weather_train)\nweather_test_df = reduce_mem_usage(weather_test_df)\nbuilding_meta_df = reduce_mem_usage(building_meta_df)","metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-26T20:43:33.016351Z","iopub.execute_input":"2024-01-26T20:43:33.017168Z","iopub.status.idle":"2024-01-26T20:43:45.674236Z","shell.execute_reply.started":"2024-01-26T20:43:33.017136Z","shell.execute_reply":"2024-01-26T20:43:45.672931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Converting from string to datetime columns\ntrain[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"], format='%Y-%m-%d %H:%M:%S')\ntrain_full[\"timestamp\"] = pd.to_datetime(train_full[\"timestamp\"])\ntest_full[\"timestamp\"] = pd.to_datetime(test_full[\"timestamp\"])","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:43:45.675797Z","iopub.execute_input":"2024-01-26T20:43:45.676116Z","iopub.status.idle":"2024-01-26T20:44:03.032489Z","shell.execute_reply.started":"2024-01-26T20:43:45.676086Z","shell.execute_reply":"2024-01-26T20:44:03.031298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Taking a glimpse at the data","metadata":{}},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:03.034446Z","iopub.execute_input":"2024-01-26T20:44:03.034920Z","iopub.status.idle":"2024-01-26T20:44:03.056299Z","shell.execute_reply.started":"2024-01-26T20:44:03.034880Z","shell.execute_reply":"2024-01-26T20:44:03.055501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.timestamp.is_monotonic_increasing ","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:03.060285Z","iopub.execute_input":"2024-01-26T20:44:03.060904Z","iopub.status.idle":"2024-01-26T20:44:03.101236Z","shell.execute_reply.started":"2024-01-26T20:44:03.060861Z","shell.execute_reply":"2024-01-26T20:44:03.100118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"So we can see that the **training dataset spans the year 2016**, with the data of almost 1450 buildings.","metadata":{}},{"cell_type":"code","source":"weather_train","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:03.102529Z","iopub.execute_input":"2024-01-26T20:44:03.103104Z","iopub.status.idle":"2024-01-26T20:44:03.130668Z","shell.execute_reply.started":"2024-01-26T20:44:03.103073Z","shell.execute_reply":"2024-01-26T20:44:03.129366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Weather data time span: ', weather_train.timestamp.min(), '—', weather_train.timestamp.max())\nprint('Unique sites: ', weather_train.site_id.unique())","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:03.132284Z","iopub.execute_input":"2024-01-26T20:44:03.132957Z","iopub.status.idle":"2024-01-26T20:44:03.187907Z","shell.execute_reply.started":"2024-01-26T20:44:03.132926Z","shell.execute_reply":"2024-01-26T20:44:03.186831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As expected, the weather training data covers the same period. **There are 16 different sites** of the weather stations that provided this data (so that each building is associated/located near one of the sites, which we can see below).","metadata":{"execution":{"iopub.status.busy":"2023-12-23T12:05:08.758485Z","iopub.execute_input":"2023-12-23T12:05:08.758990Z","iopub.status.idle":"2023-12-23T12:05:08.780374Z","shell.execute_reply.started":"2023-12-23T12:05:08.758953Z","shell.execute_reply":"2023-12-23T12:05:08.779121Z"}}},{"cell_type":"code","source":"building_meta_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:03.189627Z","iopub.execute_input":"2024-01-26T20:44:03.190085Z","iopub.status.idle":"2024-01-26T20:44:03.210986Z","shell.execute_reply.started":"2024-01-26T20:44:03.190040Z","shell.execute_reply":"2024-01-26T20:44:03.209936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:03.212770Z","iopub.execute_input":"2024-01-26T20:44:03.213887Z","iopub.status.idle":"2024-01-26T20:44:03.226350Z","shell.execute_reply.started":"2024-01-26T20:44:03.213852Z","shell.execute_reply":"2024-01-26T20:44:03.225221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The training dataset consists of the same buildings, but from a different time period:","metadata":{}},{"cell_type":"code","source":"test_df.timestamp.min(), test_df.timestamp.max()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:03.228108Z","iopub.execute_input":"2024-01-26T20:44:03.228516Z","iopub.status.idle":"2024-01-26T20:44:16.005761Z","shell.execute_reply.started":"2024-01-26T20:44:03.228485Z","shell.execute_reply":"2024-01-26T20:44:16.004686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"markdown","source":"Let's look at the number of missing (NaN) values for each column:","metadata":{}},{"cell_type":"code","source":"# Count how many available values (in %) there are for each column, except the target column\ntrain_data = (train_full.count() / len(train_full)).drop('meter_reading').sort_values().values\nind = np.arange(len(train_data))\nwidth = 0.35\n\nfig, axes = plt.subplots(1,1,figsize=(14, 6), dpi=100)\ntr = axes.bar(ind, train_data, width, color='royalblue')\n\ntest_data = (test_full.count() / len(test_full)).drop('row_id').sort_values().values\ntt = axes.bar(ind+width, test_data, width, color='seagreen')\n\naxes.set_ylabel('Amount of data available');\naxes.set_xticks(ind + width / 2)\naxes.set_xticklabels((train_full.count() / len(train_full)).drop('meter_reading').sort_values().index, rotation=40)\naxes.legend([tr, tt], ['Train', 'Test']);","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-26T20:44:16.007161Z","iopub.execute_input":"2024-01-26T20:44:16.007514Z","iopub.status.idle":"2024-01-26T20:44:28.357936Z","shell.execute_reply.started":"2024-01-26T20:44:16.007483Z","shell.execute_reply":"2024-01-26T20:44:28.356734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that there are little data available on the floor counts and the years of the buildings.","metadata":{}},{"cell_type":"markdown","source":"Now let's include meter readings, plotting hour-of-observation vs. building_id, using different colors to indicate what kind of data was recorded at each hour (zero, non-zero, NaN):","metadata":{}},{"cell_type":"code","source":"train = train.set_index(['timestamp'])\nbuildings_count = len(train.building_id.unique())\n\nmeters_count = len(train.meter.unique())\nmeters = ['electricity', 'chilledwater', 'steam', 'hotwater']\n\nPLOT_MISSING_VALUES = True\n\n# Plot missing values per building/meter\nif PLOT_MISSING_VALUES:\n    f,a=plt.subplots(1,4,figsize=(20,30))\n    \n    for meter in np.arange(meters_count):\n        # group by the consumed energy type (meter)\n        df = train[train.meter==meter].copy().reset_index()\n        \n        # Convert from seconds to hours elapsed since the starting point of the dataset\n        df['timestamp'] = df['timestamp'] - df['timestamp'].min()\n        df['timestamp'] = df['timestamp'].dt.total_seconds() // 3600\n        \n        # Generate a matrix of missing and zero values\n        missmap = np.empty((buildings_count, int(df.timestamp.max())+1))\n        missmap.fill(np.nan)\n        \n        for l in df.values:\n            if l[2]!=meter:\n                continue\n            # l[1] is a building id, l[0] is an hour, l[3] is a meter reading\n            missmap[int(l[1]), int(l[0])] = 0 if l[3]==0 else 1\n            \n        a[meter].set_title(f'{meters[meter]}')\n        sns.heatmap(missmap, cmap='Paired', ax=a[meter], cbar=False)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-26T20:44:28.359761Z","iopub.execute_input":"2024-01-26T20:44:28.360538Z","iopub.status.idle":"2024-01-26T20:44:28.703090Z","shell.execute_reply.started":"2024-01-26T20:44:28.360496Z","shell.execute_reply":"2024-01-26T20:44:28.701654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Legend:**\n\n* X axis: hours elapsed since Jan 1st 2016, for each of the 4 meter types\n* Y axis: building_id\n* Brown: meter reading available with non-zero value\n* Light blue: meter reading available with zero value\n* White: missing meter reading","metadata":{}},{"cell_type":"markdown","source":"**What we can conclude:**\n1. Not every building has all meter types.\n2. There are a bunch of NaN values which often go in contiguous time intervals for multiple buidings (white rectangles in the graph above).\n3. Vertical blue lines indicate that for a number of buildings, the meter readings were zero at specific time intervals.","metadata":{}},{"cell_type":"markdown","source":"**Number of observations by day:**","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(14, 6))\n# Floor the dates to the nearest day, then count repetitions (i.e. dates of the same day)\ntrain_full['timestamp'].dt.floor('d').value_counts().sort_index().plot(ax=axes).set_xlabel('Date', fontsize=14);\ntest_full['timestamp'].dt.floor('d').value_counts().sort_index().plot(ax=axes).set_ylabel('Number of observations', fontsize=14);\naxes.set_title('Number of observations by day', fontsize=16);\naxes.legend(['Train', 'Test']);","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-26T20:44:28.704612Z","iopub.execute_input":"2024-01-26T20:44:28.705040Z","iopub.status.idle":"2024-01-26T20:44:31.267104Z","shell.execute_reply.started":"2024-01-26T20:44:28.705001Z","shell.execute_reply":"2024-01-26T20:44:31.266026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This is something interesting. We can see that number of observations by day in testing dataset is always the same. But not in the training set.","metadata":{}},{"cell_type":"code","source":"train_data = train_full.copy()\ntest_data = test_full.copy()\n\ntrain_data['hour'] = train_data['timestamp'].dt.hour\ntest_data['hour'] = test_data['timestamp'].dt.hour\n\n# Calculate the number of values that have the same hour of the day in the dataset\nplot_data = train_data['hour'].value_counts(dropna=False, normalize=True).sort_index().values\nind = np.arange(len(plot_data))\nwidth = 0.35\n\nfig, axes = plt.subplots(1,1,figsize=(14, 6), dpi=100)\ntr = axes.bar(ind, plot_data, width, color='royalblue')\n\nplot_data = test_data['hour'].value_counts(dropna=False, normalize=True).sort_index().values\ntt = axes.bar(ind+width, plot_data, width, color='seagreen')\n\n# Plot the number of observations of the same hour \naxes.set_ylabel('Normalized number of observations');\naxes.set_xlabel('Hour');\naxes.set_xticks(ind + width / 2)\naxes.set_xticklabels(train_data['hour'].value_counts().sort_index().index, rotation=0)\naxes2 = axes.twinx()\n\n# Plot the mean number of the meter reading in each hour\nmr = axes2.plot(ind, train_data[['hour', 'meter_reading']].groupby('hour')['meter_reading'].mean().sort_index().values, 'D-', color='tab:orange', label='Mean meter reading');\naxes2.grid(False);\naxes2.tick_params(axis='y', labelcolor='tab:orange');\naxes2.set_ylabel('Mean meter reading by hour', color='tab:orange');\naxes.legend([tr, tt], ['Train', 'Test'], facecolor='white');\naxes2.legend(loc=2, facecolor='white');","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-26T20:44:31.268477Z","iopub.execute_input":"2024-01-26T20:44:31.268848Z","iopub.status.idle":"2024-01-26T20:44:41.366155Z","shell.execute_reply.started":"2024-01-26T20:44:31.268817Z","shell.execute_reply":"2024-01-26T20:44:41.364881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The train and test datasets appear to be pretty well balanced. Number of observations per hour is almost the same.\n\nAlso we can see that energy consumption raises up during the day and lowers during the night hours.","metadata":{}},{"cell_type":"markdown","source":"# Dealing with NaN values","metadata":{}},{"cell_type":"markdown","source":"As we saw earlier, most of the available data comes from the 'meter 0' - electricity, so we'll focus on predicting only electricity from now on.","metadata":{}},{"cell_type":"code","source":"train = train_full[train_full.meter == 0]\ndel train['meter']\ntrain","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:41.368520Z","iopub.execute_input":"2024-01-26T20:44:41.369169Z","iopub.status.idle":"2024-01-26T20:44:43.208821Z","shell.execute_reply.started":"2024-01-26T20:44:41.369126Z","shell.execute_reply":"2024-01-26T20:44:43.207689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Since the number of observations per day is not uniform as we saw earlier, we need to resample the dataset to a 1-hour frequency. However, some buildings have too little data to perform interpolation for resampling, so we'll need to discard them first.","metadata":{}},{"cell_type":"code","source":"unique_buildings = train.building_id.unique()\n\n# Extract time series data for each building from the dataset as a separate dataframe\nbuildings_slices = [train[train['building_id'] == id].set_index('timestamp') \n                    for id in unique_buildings]\nprint('All buildings with electricity meter: ', len(buildings_slices))","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:44:43.210035Z","iopub.execute_input":"2024-01-26T20:44:43.210844Z","iopub.status.idle":"2024-01-26T20:45:03.936937Z","shell.execute_reply.started":"2024-01-26T20:44:43.210813Z","shell.execute_reply":"2024-01-26T20:45:03.935609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a desired time range with uniform frequency of 1 hour\ndesired_index = pd.date_range(start=train.timestamp.min(), \\\n                              end=train.timestamp.max(), freq='1H')","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:45:03.938618Z","iopub.execute_input":"2024-01-26T20:45:03.939444Z","iopub.status.idle":"2024-01-26T20:45:04.002078Z","shell.execute_reply.started":"2024-01-26T20:45:03.939400Z","shell.execute_reply":"2024-01-26T20:45:04.001065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"buildings_slices_resampled = []\nweather_train.set_index('timestamp', inplace=True) \n\nfor slice_df in buildings_slices:\n    \"\"\"For each building time series:\n        1. Reindex based on the desired time range\n        2. Perform resampling (this will introduce NaNs for missing time periods)\n        3. Reindex back to the desired time range\n    \"\"\" \n    slice_resampled = slice_df.reindex(slice_df.index.union(desired_index)) \\\n                              .asfreq('1h') \\\n                              .reindex(desired_index)  \n    \n    # 4. Fill in the NaN values introduced by resampling\n    new_rows_mask = slice_resampled.isna().all(axis=1) # newly inserted rows have all values NaN\n    \n    # Pick a sample non-empty row that was originally in the dataset\n    original_sample_row = slice_df.iloc[0] \n        \n    # Fill in all the building_metadata for the new rows\n    slice_resampled.loc[new_rows_mask, 'building_id': 'floor_count'] = \\\n        original_sample_row.loc['building_id': 'floor_count'].values\n    \n    # Fill in all the weather data for the new rows\n    slice_resampled.update(weather_train[weather_train['site_id']==original_sample_row['site_id']])\n    \n    # Finish with assigning a NaN value to the meter_reading, in order to interpolate it later\n    slice_resampled.loc[new_rows_mask, 'meter_reading'] = np.nan\n    \n    buildings_slices_resampled.append(slice_resampled)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:45:04.003384Z","iopub.execute_input":"2024-01-26T20:45:04.003746Z","iopub.status.idle":"2024-01-26T20:45:56.336985Z","shell.execute_reply.started":"2024-01-26T20:45:04.003715Z","shell.execute_reply":"2024-01-26T20:45:56.336015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_r = pd.concat(buildings_slices_resampled)\ntrain_r","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2024-01-26T20:45:56.338327Z","iopub.execute_input":"2024-01-26T20:45:56.338819Z","iopub.status.idle":"2024-01-26T20:45:57.396517Z","shell.execute_reply.started":"2024-01-26T20:45:56.338788Z","shell.execute_reply":"2024-01-26T20:45:57.395444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now verify that the sampling rate is uniform:","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(14, 6))\n\ntrain_r.index.floor('d').value_counts().sort_index()\\\n.plot(ax=axes).set_xlabel('Date', fontsize=14)\n\naxes.set_title('Number of observations by day', fontsize=16);","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2024-01-26T20:45:57.397925Z","iopub.execute_input":"2024-01-26T20:45:57.398625Z","iopub.status.idle":"2024-01-26T20:45:58.314114Z","shell.execute_reply.started":"2024-01-26T20:45:57.398594Z","shell.execute_reply":"2024-01-26T20:45:58.312950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot a histogram of the missing values rate after resampling\n\nn = len(desired_index)\nbuilding_nans = train_r.groupby('building_id')['meter_reading'].apply(lambda x: x.isna().sum() / n)\nbuilding_nans.plot(kind='hist', weights=np.ones_like(building_nans) / len(building_nans))","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:45:58.315475Z","iopub.execute_input":"2024-01-26T20:45:58.315828Z","iopub.status.idle":"2024-01-26T20:45:59.555539Z","shell.execute_reply.started":"2024-01-26T20:45:58.315799Z","shell.execute_reply":"2024-01-26T20:45:59.554415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Discard all the buildings, who have more than **NANS_THRESHOLD** % of NaN values:","metadata":{}},{"cell_type":"code","source":"NANS_THRESHOLD = 0\n\nbuildings_to_discard = (building_nans[building_nans > NANS_THRESHOLD]).index\n\n# First resampled, now reduced\ntrain_rr = train_r[~train_r['building_id'].isin(buildings_to_discard)]\ntrain_rr.isna().sum()\n# Plot the number of NaNs in each column","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:45:59.557100Z","iopub.execute_input":"2024-01-26T20:45:59.558058Z","iopub.status.idle":"2024-01-26T20:46:00.754383Z","shell.execute_reply.started":"2024-01-26T20:45:59.558023Z","shell.execute_reply":"2024-01-26T20:46:00.753401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fill_metanans_strategy = 'DROP' # 'VIRTUAL', 'MEDIAN'\n\"\"\"Fill in the stationary metabuilding NaN columns according to 3 strategies:\n    1. MEDIAN: impute the NaN values with median values of the year_built and floor_count columns\n    2. VIRTUAL: method based on the idea that the absence of the metabuilding info can actually reveal\n    something about the building (e.g. that the building is old). In this case, we impute all the NaN\n    values with 'virtual' stationary negative value, like -103 (the choice of a value doesn't matter here)\n    3. DROP: drop the year_built and floor_count columns\n\"\"\"\n\nif fill_metanans_strategy=='VIRTUAL':\n    train_rr['floor_count'] = train_rr['floor_count'].fillna(-103)\n    train_rr['year_built'] = train_rr['year_built'].fillna(-103)\nelif fill_metanans_strategy == 'MEDIAN':\n    nans_mask = train_rr.isna()\n    \n    floor_nans = nans_mask['floor_count']\n    year_nans = nans_mask['year_built']\n    \n    train_rr.loc[floor_nans, 'floor_count'] = train_rr['floor_count'].median(skipna=True)\n    train_rr.loc[year_nans, 'year_built'] = train_rr['year_built'].median(skipna=True)\nelif fill_metanans_strategy == 'DROP':\n    train_rr = train_rr.drop(columns=['year_built', 'floor_count'])\nelse:\n    print('Unsupported strategy.')\n    \ntrain_rr.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:47:11.905286Z","iopub.execute_input":"2024-01-26T20:47:11.905713Z","iopub.status.idle":"2024-01-26T20:47:12.649243Z","shell.execute_reply.started":"2024-01-26T20:47:11.905681Z","shell.execute_reply":"2024-01-26T20:47:12.648117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Interpolating resampled time series","metadata":{}},{"cell_type":"markdown","source":"Finally, let's use interpolation to fill in the remaining NaNs:","metadata":{}},{"cell_type":"code","source":"interp_method = 'linear'","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:47:14.828396Z","iopub.execute_input":"2024-01-26T20:47:14.828826Z","iopub.status.idle":"2024-01-26T20:47:14.833361Z","shell.execute_reply.started":"2024-01-26T20:47:14.828791Z","shell.execute_reply":"2024-01-26T20:47:14.832430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_buildings = train_rr.building_id.unique()\nfeatures_to_fill = [ 'meter_reading',\n                     'air_temperature',       \n                     'cloud_coverage',       \n                     'dew_temperature',       \n                     'precip_depth_1_hr',    \n                     'sea_level_pressure',   \n                     'wind_direction',        \n                     'wind_speed',  ] \n\nbuilding_slices = [train_rr[train_rr['building_id'] == id]\n                    for id in unique_buildings]\n\ntrain_rri = pd.concat([slice.interpolate(method=interp_method, axis='index', extrapolate=True)\n                       for slice in building_slices])\n\n# After interpolation, there are still some NaNs left outside the interpolation bounds.\n# We'll fill them with the median values\n\nnans_mask = train_rri.isna()\nfor f in features_to_fill:\n    feature_nans = nans_mask[f]\n\n    # Check if there are NaN values before filling\n    if feature_nans.any():\n        # Calculate and fill the median for the feature\n        median_value = train_rri[f].median(skipna=True)\n        train_rri.loc[feature_nans, f] = median_value\n\n# Check if there are still NaN values\ntrain_rri.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:47:15.060701Z","iopub.execute_input":"2024-01-26T20:47:15.061325Z","iopub.status.idle":"2024-01-26T20:47:21.329794Z","shell.execute_reply.started":"2024-01-26T20:47:15.061284Z","shell.execute_reply":"2024-01-26T20:47:21.328604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from numpy.random import choice, seed\nseed(42)\n\n\"\"\" Optionally drop a portion of the buildings left (to save computational time later)\n    If BUILDINGS_DROP_VALUE is less than 1, it specifies a fraction of the buildings to drop (e.g. 0.5)\n    If BUILDINGS_DROP_VALUE is bigger (or equal) than 1, it specifices the absolute value of the buildings to leave in the dataset (e.g. 1, 10, ...)\"\"\"\n\nBUILDINGS_DROP_VALUE = 0.9\n\nnew_buildings = train_rri.building_id.unique()\nnew_buildings_count = len(new_buildings)\n\nif BUILDINGS_DROP_VALUE < 1: # drop a fraction of the buildings left\n    buildings_drop_count = int(np.ceil(new_buildings_count * BUILDINGS_DROP_VALUE))\n    \n    buildings_to_drop = choice(new_buildings, buildings_drop_count, replace = False)\nelse: # leave only BUILDINGS_DROP_VALUE buildings\n    assert BUILDINGS_DROP_VALUE < new_buildings_count\n    \n    buildings_drop_count = new_buildings_count - BUILDINGS_DROP_VALUE\n    buildings_to_drop = choice(new_buildings, buildings_drop_count, replace = False)\n\ntrain_rri = train_rri[~train_rri['building_id'].isin(buildings_to_drop)]\nprint(f'{len(train_rri.building_id.unique())} buildings left in the dataset.')","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:48:21.442966Z","iopub.execute_input":"2024-01-26T20:48:21.445064Z","iopub.status.idle":"2024-01-26T20:48:21.563261Z","shell.execute_reply.started":"2024-01-26T20:48:21.445011Z","shell.execute_reply":"2024-01-26T20:48:21.562108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rri","metadata":{"execution":{"iopub.status.busy":"2024-01-26T20:48:29.490468Z","iopub.execute_input":"2024-01-26T20:48:29.491290Z","iopub.status.idle":"2024-01-26T20:48:29.525169Z","shell.execute_reply.started":"2024-01-26T20:48:29.491255Z","shell.execute_reply":"2024-01-26T20:48:29.524116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Feature engineering & columns preprocessing","metadata":{}},{"cell_type":"markdown","source":"Previously we saw that there is a clear pattern in temporal change of the mean electricity consumption during the day. So, let's extract this (and similar) information from the dataset and use it as new features.","metadata":{}},{"cell_type":"code","source":"if fill_metanans_strategy != 'DROP':\n    train_rri['age'] = train_rri['year_built'].max() - train_rri['year_built'] + 1\n    train_rri['year_built'] = train_rri['year_built']-1900\n\ntrain_rri['month_datetime'] = train_rri.index.month.astype(np.int8)\ntrain_rri['hour_datetime'] = train_rri.index.hour.astype(np.int8)  \ntrain_rri['day_week'] = train_rri.index.dayofweek.astype(np.int8)\ntrain_rri['day_month_datetime'] = train_rri.index.day.astype(np.int8)\n\ntrain_rri['square_feet'] = np.log(train_rri['square_feet'])","metadata":{"execution":{"iopub.status.busy":"2024-01-26T18:11:08.753465Z","iopub.execute_input":"2024-01-26T18:11:08.753824Z","iopub.status.idle":"2024-01-26T18:11:08.768556Z","shell.execute_reply.started":"2024-01-26T18:11:08.753793Z","shell.execute_reply":"2024-01-26T18:11:08.767416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert to more lightweight types to save some memory\nd_types = {'building_id': np.int16,\n          'site_id': np.int8,\n          'primary_use': 'category',\n          'square_feet': np.int32,\n          'air_temperature': np.float32,\n          'cloud_coverage': np.float16,\n          'dew_temperature': np.float32,\n          'precip_depth_1_hr': np.float16,\n          'sea_level_pressure': np.float32,\n          'wind_direction': np.float16,\n          'wind_speed': np.float32}\nif fill_metanans_strategy != 'DROP':\n    d_types['year_built'] = np.float16,\n    d_types['floor_count'] = np.float16,\n\nfor feature in d_types:\n    train_rri[feature] = train_rri[feature].astype(d_types[feature])","metadata":{"execution":{"iopub.status.busy":"2024-01-26T18:11:08.770239Z","iopub.execute_input":"2024-01-26T18:11:08.770824Z","iopub.status.idle":"2024-01-26T18:11:08.784104Z","shell.execute_reply.started":"2024-01-26T18:11:08.770790Z","shell.execute_reply":"2024-01-26T18:11:08.782980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\n# Convert our only categorical column to numeric\nle = LabelEncoder()\n\ntrain_rri['primary_use'] = le.fit_transform(train_rri['primary_use']).astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2024-01-26T18:11:08.785555Z","iopub.execute_input":"2024-01-26T18:11:08.786471Z","iopub.status.idle":"2024-01-26T18:11:08.798196Z","shell.execute_reply.started":"2024-01-26T18:11:08.786436Z","shell.execute_reply":"2024-01-26T18:11:08.796915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rri.describe()","metadata":{"execution":{"iopub.status.busy":"2024-01-26T18:11:08.800278Z","iopub.execute_input":"2024-01-26T18:11:08.800749Z","iopub.status.idle":"2024-01-26T18:11:08.907690Z","shell.execute_reply.started":"2024-01-26T18:11:08.800702Z","shell.execute_reply":"2024-01-26T18:11:08.906504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Splitting into Training & Testing datasets","metadata":{}},{"cell_type":"markdown","source":"Unfortunately, the testing dataset provided by its authors will not work for us, because we don't know the target (y) values for it to be able to compute our own performance metrics.\n\n**So, we'll choose the last months of spring, summer and autumn as our testing dataset (25%), leaving everything else for the training.**","metadata":{}},{"cell_type":"code","source":"testing_months = [5, 8, 11]\n\ntest_df = train_rri[train_rri['month_datetime'].isin(testing_months)]\ntrain_df = train_rri[~train_rri['month_datetime'].isin(testing_months)]\n\nprint('Training dataset length:', len(train_df))\nprint('Test dataset length:', len(test_df))","metadata":{"execution":{"iopub.status.busy":"2024-01-26T18:11:08.910711Z","iopub.execute_input":"2024-01-26T18:11:08.911411Z","iopub.status.idle":"2024-01-26T18:11:08.924398Z","shell.execute_reply.started":"2024-01-26T18:11:08.911368Z","shell.execute_reply":"2024-01-26T18:11:08.922833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save the datasets to be used later...\ntrain_df.to_pickle('train_df.pkl')\ntest_df.to_pickle('test_df.pkl')","metadata":{"execution":{"iopub.status.busy":"2024-01-26T18:11:08.926225Z","iopub.execute_input":"2024-01-26T18:11:08.926711Z","iopub.status.idle":"2024-01-26T18:11:08.934885Z","shell.execute_reply.started":"2024-01-26T18:11:08.926667Z","shell.execute_reply":"2024-01-26T18:11:08.933565Z"},"trusted":true},"execution_count":null,"outputs":[]}]}