{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Welcome to the ASHRAE - Great Energy Predictor Competition\nThis notebook is a starter code for all beginners and easy to understand. The train and test data are very large so we will work with a data generator based on the template to generate the data on the fly <br>\nhttps://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly\n\nAdditionally we follow an efficient workflow. <br>\nWe also use categorical feature encoding techniques, compare <br>\nhttps://www.kaggle.com/drcapa/categorical-feature-encoding-challenge-xgb\n\nFor the first step we will take a simple neural network based on the keras library. After that we will use a RNN.<br>\nCurrent status of the kernel: The workflow is complete.<br>\nNext steps: \n* Improve the LSTM.\n* Expand the feature engineering based on the kernel: https://www.kaggle.com/drcapa/ashrae-feature-engineering","metadata":{}},{"cell_type":"markdown","source":"# Load Libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport scipy.special\nimport matplotlib.pyplot as plt\nimport os\nimport random","metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","execution":{"iopub.status.busy":"2023-04-25T16:11:14.646381Z","iopub.execute_input":"2023-04-25T16:11:14.646734Z","iopub.status.idle":"2023-04-25T16:11:14.869461Z","shell.execute_reply.started":"2023-04-25T16:11:14.646658Z","shell.execute_reply":"2023-04-25T16:11:14.868693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.utils import Sequence\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Dropout, Flatten, LSTM, Embedding\nfrom keras.optimizers import RMSprop,Adam\nimport keras.backend as K","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:14.871468Z","iopub.execute_input":"2023-04-25T16:11:14.871921Z","iopub.status.idle":"2023-04-25T16:11:16.721533Z","shell.execute_reply.started":"2023-04-25T16:11:14.871745Z","shell.execute_reply":"2023-04-25T16:11:16.720693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import MinMaxScaler","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:16.723003Z","iopub.execute_input":"2023-04-25T16:11:16.723291Z","iopub.status.idle":"2023-04-25T16:11:17.490698Z","shell.execute_reply.started":"2023-04-25T16:11:16.723237Z","shell.execute_reply":"2023-04-25T16:11:17.489813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(pd.__version__)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:17.494302Z","iopub.execute_input":"2023-04-25T16:11:17.494572Z","iopub.status.idle":"2023-04-25T16:11:17.501542Z","shell.execute_reply.started":"2023-04-25T16:11:17.494519Z","shell.execute_reply":"2023-04-25T16:11:17.50065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:17.505558Z","iopub.execute_input":"2023-04-25T16:11:17.505984Z","iopub.status.idle":"2023-04-25T16:11:17.512821Z","shell.execute_reply.started":"2023-04-25T16:11:17.505809Z","shell.execute_reply":"2023-04-25T16:11:17.511928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:17.515766Z","iopub.execute_input":"2023-04-25T16:11:17.516285Z","iopub.status.idle":"2023-04-25T16:11:17.522768Z","shell.execute_reply.started":"2023-04-25T16:11:17.516015Z","shell.execute_reply":"2023-04-25T16:11:17.522043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"path_in = '../input/ashrae-energy-prediction/'\nprint(os.listdir(path_in))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:17.524212Z","iopub.execute_input":"2023-04-25T16:11:17.524807Z","iopub.status.idle":"2023-04-25T16:11:17.53558Z","shell.execute_reply.started":"2023-04-25T16:11:17.524455Z","shell.execute_reply":"2023-04-25T16:11:17.534804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv(path_in+'train.csv', parse_dates=['timestamp'])\ntrain_weather = pd.read_csv(path_in+'weather_train.csv', parse_dates=['timestamp'])\nbuilding_data = pd.read_csv(path_in+'building_metadata.csv')","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:17.53729Z","iopub.execute_input":"2023-04-25T16:11:17.537827Z","iopub.status.idle":"2023-04-25T16:11:35.890575Z","shell.execute_reply.started":"2023-04-25T16:11:17.53758Z","shell.execute_reply":"2023-04-25T16:11:35.889729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Help function","metadata":{}},{"cell_type":"code","source":"def plot_bar(data, name):\n    fig = plt.figure(figsize=(16, 9))\n    ax = fig.add_subplot(111)\n    data_label = data[name].value_counts()\n    dict_train = dict(zip(data_label.keys(), ((data_label.sort_index())).tolist()))\n    names = list(dict_train.keys())\n    values = list(dict_train.values())\n    plt.bar(names, values)\n    ax.set_xticklabels(names, rotation=45)\n    plt.grid()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:35.892111Z","iopub.execute_input":"2023-04-25T16:11:35.892487Z","iopub.status.idle":"2023-04-25T16:11:35.899987Z","shell.execute_reply.started":"2023-04-25T16:11:35.892407Z","shell.execute_reply":"2023-04-25T16:11:35.898135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Handle missing values of building and weather data\nThe missing data are numerical values. So for the first step we can use a simple imputer of the sklearn library.","metadata":{}},{"cell_type":"code","source":"cols_with_missing_train_weather = [col for col in train_weather.columns if train_weather[col].isnull().any()]\ncols_with_missing_building = [col for col in building_data.columns if building_data[col].isnull().any()]","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:35.901334Z","iopub.execute_input":"2023-04-25T16:11:35.901954Z","iopub.status.idle":"2023-04-25T16:11:35.916289Z","shell.execute_reply.started":"2023-04-25T16:11:35.90184Z","shell.execute_reply":"2023-04-25T16:11:35.91562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(cols_with_missing_train_weather)\nprint(cols_with_missing_building)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:35.917641Z","iopub.execute_input":"2023-04-25T16:11:35.917965Z","iopub.status.idle":"2023-04-25T16:11:35.927079Z","shell.execute_reply.started":"2023-04-25T16:11:35.917913Z","shell.execute_reply":"2023-04-25T16:11:35.9263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp_most = SimpleImputer(strategy='most_frequent')\ntrain_weather[cols_with_missing_train_weather] = imp_most.fit_transform(train_weather[cols_with_missing_train_weather])\nbuilding_data[cols_with_missing_building] = imp_most.fit_transform(building_data[cols_with_missing_building])","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:35.928313Z","iopub.execute_input":"2023-04-25T16:11:35.928794Z","iopub.status.idle":"2023-04-25T16:11:36.115333Z","shell.execute_reply.started":"2023-04-25T16:11:35.92873Z","shell.execute_reply":"2023-04-25T16:11:36.114275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scale objective label","metadata":{}},{"cell_type":"code","source":"train_data['meter_reading'] = np.log1p(train_data['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:36.116851Z","iopub.execute_input":"2023-04-25T16:11:36.117312Z","iopub.status.idle":"2023-04-25T16:11:36.425387Z","shell.execute_reply.started":"2023-04-25T16:11:36.117262Z","shell.execute_reply":"2023-04-25T16:11:36.424319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['meter_reading'].head()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:36.426958Z","iopub.execute_input":"2023-04-25T16:11:36.42729Z","iopub.status.idle":"2023-04-25T16:11:36.437109Z","shell.execute_reply.started":"2023-04-25T16:11:36.427233Z","shell.execute_reply":"2023-04-25T16:11:36.436188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create new features\n## Train data\nBased on the timestamp we create new features which are cyclic.","metadata":{}},{"cell_type":"code","source":"train_data['timestamp']","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:36.438761Z","iopub.execute_input":"2023-04-25T16:11:36.439342Z","iopub.status.idle":"2023-04-25T16:11:36.448868Z","shell.execute_reply.started":"2023-04-25T16:11:36.43929Z","shell.execute_reply":"2023-04-25T16:11:36.447938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['timestamp'].dt.month","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:36.45057Z","iopub.execute_input":"2023-04-25T16:11:36.451129Z","iopub.status.idle":"2023-04-25T16:11:38.187684Z","shell.execute_reply.started":"2023-04-25T16:11:36.450822Z","shell.execute_reply":"2023-04-25T16:11:38.186653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['month'] = train_data['timestamp'].dt.month\ntrain_data['day'] = train_data['timestamp'].dt.weekday\ntrain_data['year'] = train_data['timestamp'].dt.year\ntrain_data['hour'] = train_data['timestamp'].dt.hour","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:38.189199Z","iopub.execute_input":"2023-04-25T16:11:38.189664Z","iopub.status.idle":"2023-04-25T16:11:45.60429Z","shell.execute_reply.started":"2023-04-25T16:11:38.189466Z","shell.execute_reply":"2023-04-25T16:11:45.603236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['day'].value_counts","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:45.605643Z","iopub.execute_input":"2023-04-25T16:11:45.60597Z","iopub.status.idle":"2023-04-25T16:11:45.612788Z","shell.execute_reply.started":"2023-04-25T16:11:45.605915Z","shell.execute_reply":"2023-04-25T16:11:45.611781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Additionally we create the feature weekend: 5 = saturday and 6 = sunday.","metadata":{}},{"cell_type":"code","source":"train_data['weekend'] = np.where((train_data['day'] == 5) | (train_data['day'] == 6), 1, 0)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:45.614479Z","iopub.execute_input":"2023-04-25T16:11:45.615043Z","iopub.status.idle":"2023-04-25T16:11:46.204705Z","shell.execute_reply.started":"2023-04-25T16:11:45.614989Z","shell.execute_reply":"2023-04-25T16:11:46.203782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Weather data\nThe feature wind_direction is cyclic.","metadata":{}},{"cell_type":"code","source":"train_weather['wind_direction'+'_sin'] = np.sin((2*np.pi*train_weather['wind_direction'])/360)\ntrain_weather['wind_direction'+'_cos'] = np.cos((2*np.pi*train_weather['wind_direction'])/360)\ntrain_weather = train_weather.drop(['wind_direction'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:46.206007Z","iopub.execute_input":"2023-04-25T16:11:46.206297Z","iopub.status.idle":"2023-04-25T16:11:46.264136Z","shell.execute_reply.started":"2023-04-25T16:11:46.206246Z","shell.execute_reply":"2023-04-25T16:11:46.263346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Encoding\nThere is a greate encoding competition: https://www.kaggle.com/drcapa/categorical-feature-encoding-challenge-xgb\n## Train data\n### Feature meter\nThere are 4 types of meters: <br>\n0 = electricity, 1 = chilledwater, 2 = steam, 3 = hotwater <br>\nWe use the one hot encoding for this 4 feature.","metadata":{}},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:46.265471Z","iopub.execute_input":"2023-04-25T16:11:46.265767Z","iopub.status.idle":"2023-04-25T16:11:46.289979Z","shell.execute_reply.started":"2023-04-25T16:11:46.265718Z","shell.execute_reply":"2023-04-25T16:11:46.288992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.get_dummies(train_data, columns=['meter'])","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:46.291404Z","iopub.execute_input":"2023-04-25T16:11:46.291984Z","iopub.status.idle":"2023-04-25T16:11:51.09296Z","shell.execute_reply.started":"2023-04-25T16:11:46.291646Z","shell.execute_reply":"2023-04-25T16:11:51.092089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features month, day and hour\nWe created the features month, day and hour which are cyclic.","metadata":{}},{"cell_type":"code","source":"features_cyc = {'month' : 12, 'day' : 7, 'hour' : 24}\nfor feature in features_cyc.keys():\n    train_data[feature+'_sin'] = np.sin((2*np.pi*train_data[feature])/features_cyc[feature])\n    train_data[feature+'_cos'] = np.cos((2*np.pi*train_data[feature])/features_cyc[feature])\ntrain_data = train_data.drop(features_cyc.keys(), axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:51.094323Z","iopub.execute_input":"2023-04-25T16:11:51.094617Z","iopub.status.idle":"2023-04-25T16:11:58.790209Z","shell.execute_reply.started":"2023-04-25T16:11:51.094567Z","shell.execute_reply":"2023-04-25T16:11:58.789435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:58.791793Z","iopub.execute_input":"2023-04-25T16:11:58.792086Z","iopub.status.idle":"2023-04-25T16:11:58.818945Z","shell.execute_reply.started":"2023-04-25T16:11:58.792032Z","shell.execute_reply":"2023-04-25T16:11:58.818228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building data\nThe feature primary_use is a categorical feature with 16 categories. For the first we use a simple mapping.","metadata":{}},{"cell_type":"code","source":"plot_bar(building_data, 'primary_use')","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:58.820147Z","iopub.execute_input":"2023-04-25T16:11:58.820577Z","iopub.status.idle":"2023-04-25T16:11:59.097324Z","shell.execute_reply.started":"2023-04-25T16:11:58.820526Z","shell.execute_reply":"2023-04-25T16:11:59.096542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_use = dict(zip(building_data['primary_use'].value_counts().sort_index().keys(),\n                     range(1, len(building_data['primary_use'].value_counts())+1)))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.098674Z","iopub.execute_input":"2023-04-25T16:11:59.099194Z","iopub.status.idle":"2023-04-25T16:11:59.109631Z","shell.execute_reply.started":"2023-04-25T16:11:59.099135Z","shell.execute_reply":"2023-04-25T16:11:59.108945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"map_use","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.111604Z","iopub.execute_input":"2023-04-25T16:11:59.112118Z","iopub.status.idle":"2023-04-25T16:11:59.11907Z","shell.execute_reply.started":"2023-04-25T16:11:59.111926Z","shell.execute_reply":"2023-04-25T16:11:59.117814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_data['primary_use'] = building_data['primary_use'].replace(map_use)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.12093Z","iopub.execute_input":"2023-04-25T16:11:59.121458Z","iopub.status.idle":"2023-04-25T16:11:59.134596Z","shell.execute_reply.started":"2023-04-25T16:11:59.121191Z","shell.execute_reply":"2023-04-25T16:11:59.133652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_data['primary_use']","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.135939Z","iopub.execute_input":"2023-04-25T16:11:59.136251Z","iopub.status.idle":"2023-04-25T16:11:59.146173Z","shell.execute_reply.started":"2023-04-25T16:11:59.136181Z","shell.execute_reply":"2023-04-25T16:11:59.145421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#building_data = pd.get_dummies(building_data, columns=['primary_use'])","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.147662Z","iopub.execute_input":"2023-04-25T16:11:59.148327Z","iopub.status.idle":"2023-04-25T16:11:59.15212Z","shell.execute_reply.started":"2023-04-25T16:11:59.148088Z","shell.execute_reply":"2023-04-25T16:11:59.151155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scale building and weather data\n## Weather data","metadata":{}},{"cell_type":"code","source":"weather_scale = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'sea_level_pressure', 'wind_speed']","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.153722Z","iopub.execute_input":"2023-04-25T16:11:59.154229Z","iopub.status.idle":"2023-04-25T16:11:59.161Z","shell.execute_reply.started":"2023-04-25T16:11:59.15411Z","shell.execute_reply":"2023-04-25T16:11:59.160204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = train_weather[weather_scale].mean(axis=0)\ntrain_weather[weather_scale] = train_weather[weather_scale].astype('float32')\ntrain_weather[weather_scale] -= train_weather[weather_scale].mean(axis=0)\nstd = train_weather[weather_scale].std(axis=0)\ntrain_weather[weather_scale] /= train_weather[weather_scale].std(axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.167478Z","iopub.execute_input":"2023-04-25T16:11:59.167756Z","iopub.status.idle":"2023-04-25T16:11:59.21345Z","shell.execute_reply.started":"2023-04-25T16:11:59.167703Z","shell.execute_reply":"2023-04-25T16:11:59.212835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_weather[weather_scale]","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.214991Z","iopub.execute_input":"2023-04-25T16:11:59.215271Z","iopub.status.idle":"2023-04-25T16:11:59.231377Z","shell.execute_reply.started":"2023-04-25T16:11:59.215224Z","shell.execute_reply":"2023-04-25T16:11:59.230508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Building data","metadata":{}},{"cell_type":"code","source":"building_scale = ['square_feet', 'year_built', 'floor_count']","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.232716Z","iopub.execute_input":"2023-04-25T16:11:59.233194Z","iopub.status.idle":"2023-04-25T16:11:59.240495Z","shell.execute_reply.started":"2023-04-25T16:11:59.232993Z","shell.execute_reply":"2023-04-25T16:11:59.239615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean = building_data[building_scale].mean(axis=0)\nbuilding_data[building_scale] = building_data[building_scale].astype('float32')\nbuilding_data[building_scale] -= building_data[building_scale].mean(axis=0)\nstd = building_data[building_scale].std(axis=0)\nbuilding_data[building_scale] /= building_data[building_scale].std(axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.241837Z","iopub.execute_input":"2023-04-25T16:11:59.242307Z","iopub.status.idle":"2023-04-25T16:11:59.264436Z","shell.execute_reply.started":"2023-04-25T16:11:59.242088Z","shell.execute_reply":"2023-04-25T16:11:59.263769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_data[building_scale]","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.2658Z","iopub.execute_input":"2023-04-25T16:11:59.266237Z","iopub.status.idle":"2023-04-25T16:11:59.278516Z","shell.execute_reply.started":"2023-04-25T16:11:59.266055Z","shell.execute_reply":"2023-04-25T16:11:59.277768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge data","metadata":{}},{"cell_type":"code","source":"train_data = pd.merge(train_data, building_data, on='building_id', right_index=True)\ntrain_data = train_data.sort_values(['timestamp'])\ntrain_data = pd.merge_asof(train_data, train_weather, on='timestamp', by='site_id', right_index=True)\ndel train_weather","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:11:59.279801Z","iopub.execute_input":"2023-04-25T16:11:59.280237Z","iopub.status.idle":"2023-04-25T16:12:31.310714Z","shell.execute_reply.started":"2023-04-25T16:11:59.280187Z","shell.execute_reply":"2023-04-25T16:12:31.309807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:31.312352Z","iopub.execute_input":"2023-04-25T16:12:31.312827Z","iopub.status.idle":"2023-04-25T16:12:43.838461Z","shell.execute_reply.started":"2023-04-25T16:12:31.312634Z","shell.execute_reply":"2023-04-25T16:12:43.8377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build the data generator","metadata":{}},{"cell_type":"code","source":"class DataGenerator(Sequence):\n    \"\"\" A data generator based on the template\n        https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly\n        \"\"\"\n    \n    def __init__(self, data, list_IDs, features, batch_size, shuffle=False):\n        self.data = data.loc[list_IDs].copy()\n        self.list_IDs = list_IDs\n        self.features = features\n        self.batch_size = batch_size\n        self.shuffle = shuffle\n        self.on_epoch_end()\n    \n    \n    def __len__(self):\n        return int(np.floor(len(self.list_IDs)/self.batch_size))\n    \n    \n    def __getitem__(self, index):\n        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n        list_IDs_temp = [self.list_IDs[k] for k in indexes]\n        X, y = self.__data_generation(list_IDs_temp)\n        return X, y\n    \n    \n    def on_epoch_end(self):\n        self.indexes = np.arange(len(self.list_IDs))\n        if self.shuffle == True:\n            np.random.shuffle(self.indexes)\n\n        \n    def __data_generation(self, list_IDs_temp):        \n        X = np.empty((len(list_IDs_temp), len(self.features)), dtype=float)\n        y = np.empty((len(list_IDs_temp), 1), dtype=float)\n        X = self.data.loc[list_IDs_temp, self.features].values\n        \n        if 'meter_reading' in self.data.columns:\n            y = self.data.loc[list_IDs_temp, 'meter_reading'].values\n        # reshape\n        X = np.reshape(X, (X.shape[0], 1, X.shape[1]))\n        return X, y","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:43.839855Z","iopub.execute_input":"2023-04-25T16:12:43.840357Z","iopub.status.idle":"2023-04-25T16:12:43.852393Z","shell.execute_reply.started":"2023-04-25T16:12:43.840302Z","shell.execute_reply":"2023-04-25T16:12:43.851456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split the random input data into train and val\nSince it's a timeseries problem, we split the train and validation data by timestamp and not with a random split.","metadata":{}},{"cell_type":"code","source":"train_size = int(len(train_data.index)*0.75)\nval_size = len(train_data.index) - train_size\ntrain_list, val_list = train_data.index[0:train_size], train_data.index[train_size:train_size+val_size]\nprint(train_size, val_size)\n","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:43.853723Z","iopub.execute_input":"2023-04-25T16:12:43.854048Z","iopub.status.idle":"2023-04-25T16:12:43.867965Z","shell.execute_reply.started":"2023-04-25T16:12:43.853969Z","shell.execute_reply":"2023-04-25T16:12:43.866913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define the features","metadata":{}},{"cell_type":"code","source":"no_features = ['building_id', 'timestamp', 'meter_reading', 'year']\nfeatures = train_data.columns.difference(no_features)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:43.869324Z","iopub.execute_input":"2023-04-25T16:12:43.869655Z","iopub.status.idle":"2023-04-25T16:12:43.877095Z","shell.execute_reply.started":"2023-04-25T16:12:43.869565Z","shell.execute_reply":"2023-04-25T16:12:43.876422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:43.878592Z","iopub.execute_input":"2023-04-25T16:12:43.879132Z","iopub.status.idle":"2023-04-25T16:12:43.888561Z","shell.execute_reply.started":"2023-04-25T16:12:43.879012Z","shell.execute_reply":"2023-04-25T16:12:43.887583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define train and validation data via Data Generator","metadata":{}},{"cell_type":"code","source":"batch_size = 1024\ntrain_generator = DataGenerator(train_data, train_list, features, batch_size)\nval_generator = DataGenerator(train_data, val_list, features, batch_size)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:43.890148Z","iopub.execute_input":"2023-04-25T16:12:43.890599Z","iopub.status.idle":"2023-04-25T16:12:57.669427Z","shell.execute_reply.started":"2023-04-25T16:12:43.890404Z","shell.execute_reply":"2023-04-25T16:12:57.668528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define Recurrent Neural Network\nWe use a simple recurrent neural network for train and prediction. Later we will improve.","metadata":{}},{"cell_type":"code","source":"input_dim = len(features)\nprint(input_dim)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:57.670919Z","iopub.execute_input":"2023-04-25T16:12:57.671215Z","iopub.status.idle":"2023-04-25T16:12:57.678104Z","shell.execute_reply.started":"2023-04-25T16:12:57.671166Z","shell.execute_reply":"2023-04-25T16:12:57.677364Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Sequential()\n#model.add(Embedding(input_length=input_dim))\nmodel.add(LSTM(units=8, activation = 'relu', input_shape=(1, input_dim)))\n#model.add(LSTM(units=64, activation = 'relu'))\n#model.add(Dense(128, activation='relu', input_dim=input_dim))\n#model.add(Dense(256, activation='relu'))\n#model.add(Dense(512, activation='relu'))\nmodel.add(Dense(1, activation='relu'))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:57.679636Z","iopub.execute_input":"2023-04-25T16:12:57.680208Z","iopub.status.idle":"2023-04-25T16:12:57.945465Z","shell.execute_reply.started":"2023-04-25T16:12:57.680153Z","shell.execute_reply":"2023-04-25T16:12:57.944753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.layers","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:57.94665Z","iopub.execute_input":"2023-04-25T16:12:57.946943Z","iopub.status.idle":"2023-04-25T16:12:57.95344Z","shell.execute_reply.started":"2023-04-25T16:12:57.946894Z","shell.execute_reply":"2023-04-25T16:12:57.952417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rmse(y_true, y_pred):\n    \"\"\" root_mean_squared_error \"\"\"\n    return K.sqrt(K.mean(K.square(y_pred - y_true)))","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:57.955034Z","iopub.execute_input":"2023-04-25T16:12:57.955495Z","iopub.status.idle":"2023-04-25T16:12:57.961415Z","shell.execute_reply.started":"2023-04-25T16:12:57.955314Z","shell.execute_reply":"2023-04-25T16:12:57.960375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(optimizer = Adam(lr=1e-4),\n              loss='mse',\n              metrics=[rmse])","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:57.962539Z","iopub.execute_input":"2023-04-25T16:12:57.962786Z","iopub.status.idle":"2023-04-25T16:12:58.021145Z","shell.execute_reply.started":"2023-04-25T16:12:57.962739Z","shell.execute_reply":"2023-04-25T16:12:58.020563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:58.022511Z","iopub.execute_input":"2023-04-25T16:12:58.023095Z","iopub.status.idle":"2023-04-25T16:12:58.031308Z","shell.execute_reply.started":"2023-04-25T16:12:58.022769Z","shell.execute_reply":"2023-04-25T16:12:58.030486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"epochs = 1","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:58.032728Z","iopub.execute_input":"2023-04-25T16:12:58.033238Z","iopub.status.idle":"2023-04-25T16:12:58.037515Z","shell.execute_reply.started":"2023-04-25T16:12:58.033185Z","shell.execute_reply":"2023-04-25T16:12:58.036713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train model","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:58.038838Z","iopub.execute_input":"2023-04-25T16:12:58.039353Z","iopub.status.idle":"2023-04-25T16:12:59.201479Z","shell.execute_reply.started":"2023-04-25T16:12:58.039301Z","shell.execute_reply":"2023-04-25T16:12:59.20044Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:59.204766Z","iopub.execute_input":"2023-04-25T16:12:59.205017Z","iopub.status.idle":"2023-04-25T16:12:59.208928Z","shell.execute_reply.started":"2023-04-25T16:12:59.204976Z","shell.execute_reply":"2023-04-25T16:12:59.208127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit_generator(generator=train_generator,\n                              validation_data=val_generator,\n                              epochs = epochs)","metadata":{"execution":{"iopub.status.busy":"2023-04-25T16:12:59.210331Z","iopub.execute_input":"2023-04-25T16:12:59.211064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Analyse results\nA short analysis of the train results.","metadata":{}},{"cell_type":"code","source":"loss = history.history['loss']\nloss_val = history.history['val_loss']\nepochs = range(1, len(loss)+1)\nplt.plot(epochs, loss, 'bo', label='loss_train')\nplt.plot(epochs, loss_val, 'b', label='loss_val')\nplt.title('value of the loss function')\nplt.xlabel('epochs')\nplt.ylabel('value of the loss function')\nplt.legend()\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acc = history.history['rmse']\nacc_val = history.history['val_rmse']\nepochs = range(1, len(loss)+1)\nplt.plot(epochs, acc, 'bo', label='accuracy_train')\nplt.plot(epochs, acc_val, 'b', label='accuracy_val')\nplt.title('accuracy')\nplt.xlabel('epochs')\nplt.ylabel('value of accuracy')\nplt.legend()\nplt.grid()\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Delete train data","metadata":{}},{"cell_type":"code","source":"del train_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict test data\n* We following the steps above to prepare the data\n* Build data generator\n* Predict subdate\n* Write data in an array","metadata":{}},{"cell_type":"code","source":"nrows = 1667904\nbatch_size = 1022\nsteps = 25\ny_test = np.empty(())\ntest_weather = pd.read_csv(path_in+'weather_test.csv', parse_dates=['timestamp'])\ncols_with_missing_test_weather = [col for col in test_weather.columns if test_weather[col].isnull().any()]\ntest_weather[cols_with_missing_test_weather] = imp_most.fit_transform(test_weather[cols_with_missing_test_weather])\n\nmean = test_weather[weather_scale].mean(axis=0)\ntest_weather[weather_scale] = test_weather[weather_scale].astype('float32')\ntest_weather[weather_scale] -= test_weather[weather_scale].mean(axis=0)\nstd = test_weather[weather_scale].std(axis=0)\ntest_weather[weather_scale] /= test_weather[weather_scale].std(axis=0)\n\ntest_weather['wind_direction'+'_sin'] = np.sin((2*np.pi*test_weather['wind_direction'])/360)\ntest_weather['wind_direction'+'_cos'] = np.cos((2*np.pi*test_weather['wind_direction'])/360)\ntest_weather = test_weather.drop(['wind_direction'], axis=1)\n\nfor i in range(0, steps):\n    print('work on step ', (i+1))\n    test_data = pd.read_csv(path_in+'test.csv', skiprows=range(1,i*(nrows)+1), nrows=nrows, parse_dates=['timestamp'])\n    test_data['month'] = test_data['timestamp'].dt.month\n    test_data['day'] = test_data['timestamp'].dt.weekday\n    test_data['year'] = test_data['timestamp'].dt.year\n    test_data['hour'] = test_data['timestamp'].dt.hour\n    test_data['weekend'] = np.where((test_data['day'] == 5) | (test_data['day'] == 6), 1, 0)\n    for feature in features_cyc.keys():\n        test_data[feature+'_sin'] = np.sin((2*np.pi*test_data[feature])/features_cyc[feature])\n        test_data[feature+'_cos'] = np.cos((2*np.pi*test_data[feature])/features_cyc[feature])\n    test_data = test_data.drop(features_cyc.keys(), axis=1)\n    test_data = pd.get_dummies(test_data, columns=['meter'])\n    test_data = pd.merge(test_data, building_data, on='building_id', right_index=True)\n    test_data = test_data.sort_values(['timestamp'])\n    test_data = pd.merge_asof(test_data, test_weather, on='timestamp', by='site_id', right_index=True)\n    test_data = test_data.sort_values(['row_id'])\n    for feature in features:\n        if feature not in test_data:\n            #print('   not in:', feature)\n            test_data[feature] = 0\n    test_generator = DataGenerator(test_data, test_data.index, features, batch_size)\n    predict = model.predict_generator(test_generator, verbose=1, workers=1)\n    predict = np.expm1(predict)\n    y_test = np.vstack((y_test, predict))\n    del test_data\n    del test_generator","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test = np.delete(y_test, 0, 0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Delete data","metadata":{}},{"cell_type":"code","source":"del test_weather\ndel building_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Write output for submission","metadata":{}},{"cell_type":"code","source":"output = pd.DataFrame({'row_id': range(0, len(y_test)),\n                       'meter_reading': y_test.reshape(len(y_test))})\noutput = output[['row_id', 'meter_reading']]\noutput.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save(\"my_h5_model.h5\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}