{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport dask.dataframe as dd\nimport pandas as pd\nimport pandas_profiling\nfrom tqdm import tqdm_notebook as tqdm\nfrom statsmodels.tsa.stattools import adfuller\nfrom scipy import stats\nfrom IPython.display import display\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\nfrom sklearn.metrics import mean_squared_log_error\nimport os, re, pickle, datetime, gc, shutil\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.nn.utils.rnn import pad_sequence\nfrom torch.utils.data import Dataset, DataLoader\n\ngc.enable()\n\nbuilding_df   = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\nweather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\ntrain         = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\n\nweather_train.sort_values(['site_id','timestamp'], inplace=True)\nweather_train.fillna(method='ffill', inplace=True)\nweather_train.fillna(method='bfill', inplace=True)\nweather_train.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"time_cols = [col for col in weather_train.columns if col not in ['site_id', 'timestamp', 'cloud_coverage']]\nweather_scalers = {}\n\nfor col in tqdm(time_cols):\n    Scaler = StandardScaler()\n    weather_train[col] = Scaler.fit_transform(weather_train[col].values.reshape(-1, 1))\n    assert Scaler.n_samples_seen_ == len(weather_train[col])\n    weather_scalers[col] = Scaler\n    \npickle.dump(weather_scalers, open('weather_scalers.pkl','wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_df['square_feet'] = np.log(building_df['square_feet'])\nmean_per_site = pd.DataFrame(building_df.groupby('site_id').mean()).reset_index()\nmean_entire_col = pd.DataFrame(building_df.mean()).reset_index()\nmean_entire_col.columns = ['column','mean']\ncols_with_nulls = building_df.columns[building_df.isna().any()].tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for site_id in tqdm(pd.unique(building_df['site_id'])):\n    for col in tqdm(cols_with_nulls, leave=False):  \n        mean = mean_per_site.loc[mean_per_site['site_id']==site_id, col].values\n        if np.isnan(mean)[0]:\n            mean = mean_entire_col.loc[mean_entire_col['column']==col, 'mean'].values\n            \n        building_df[col] = building_df[col].mask((building_df['site_id']==site_id) & (np.isnan(building_df[col])), mean[0])\n        \nbuilding_df.isnull().sum().sum()\n\ndel mean_per_site, mean_entire_col, cols_with_nulls\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pickle.dump(building_df, open('building_df.pkl','wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(building_df, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\ntrain = train.merge(weather_train, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del building_df, weather_train\ngc.collect()\ntrain.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Based on this great kernel https://www.kaggle.com/arjanso/reducing-dataframe-memory-size-by-65\ndef reduce_mem_usage(df):\n    start_mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in df.columns:\n        if df[col].dtype != object:  # Exclude strings            \n            # Print current column type\n            print(\"******************************\")\n            print(\"Column: \",col)\n            print(\"dtype before: \",df[col].dtype)            \n            # make variables for Int, max and min\n            IsInt = False\n            mx = df[col].max()\n            mn = df[col].min()\n            print(\"min for this col: \",mn)\n            print(\"max for this col: \",mx)\n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(df[col]).all(): \n                NAlist.append(col)\n                df[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = df[col].fillna(0).astype(np.int64)\n            result = (df[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        df[col] = df[col].astype(np.int8)\n                    elif mx < 65535:\n                        df[col] = df[col].astype(np.int16)\n                    elif mx < 4294967295:\n                        df[col] = df[col].astype(np.int32)\n                    else:\n                        df[col] = df[col].astype(np.int64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)    \n            # Make float datatypes 32 bit\n            else:\n                df[col] = df[col].astype(np.float32)\n            \n            # Print new column type\n            print(\"dtype after: \",df[col].dtype)\n            print(\"******************************\")\n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return df, NAlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, NAlist = reduce_mem_usage(train)\nNAlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"])\ntrain[\"hour\"]      = train[\"timestamp\"].dt.hour.astype(np.int8)\ntrain[\"day\"]       = train[\"timestamp\"].dt.day.astype(np.int8)\ntrain[\"weekday\"]   = train[\"timestamp\"].dt.weekday.astype(np.int8)\ntrain[\"month\"]     = train[\"timestamp\"].dt.month.astype(np.int8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categoricals = [\"site_id\", \n                \"building_id\",\n                \"cloud_coverage\", \n                \"primary_use\", \n                \"hour\", \n                \"day\", \n                \"weekday\", \n                \"month\", \n                \"meter\"]\nnumericals = ['meter_reading',\n              'square_feet',\n              'year_built',\n              'floor_count',\n              'air_temperature',\n              'dew_temperature',\n              'precip_depth_1_hr',\n              'sea_level_pressure',\n              'wind_direction',\n              'wind_speed']\n\nfeat_cols = categoricals + numericals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.astype({cat:'category' for cat in categoricals})\n\ndef Cat2IDs(data, categoricals_cols, IX_start):\n    encode = {}\n    for col in categoricals_cols:\n        encode[col] = dict(enumerate(data[col].cat.categories, start=IX_start)) \n\n    Cat2Int = {}\n    for cat in categoricals_cols:\n        ValueKey = {}\n        for key, value in encode[cat].items():\n            ValueKey[value] = key\n        Cat2Int[cat] = ValueKey\n    return Cat2Int\n    \nCat2Int = Cat2IDs(data = train, \n                  categoricals_cols = categoricals, \n                  IX_start = 2) #index 0 will be for padding and index 1 will be for OOV \ncategorical_sizes = [train[c].nunique() + 2 for c in categoricals]\n\npickle.dump(Cat2Int, open('Cat2Int.pkl','wb'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in categoricals:\n    train[col] = train[col].map(Cat2Int[col])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Normal, Kurtosis and Skewness test:"},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in numericals:\n    print(col,':')\n    stat, p = stats.normaltest(train[col])\n    print('normal test: Statistics=%.3f, p=%.3f' % (stat, p))\n    alpha = 0.05\n    if p > alpha:\n        print('Gaussian: True')  #fail to reject H0\n    else:\n        print('Gaussian: False') #reject H0\n    \n    print( 'Kurtosis of normal distribution: {}'.format(stats.kurtosis(train[col])))\n    print( 'Skewness of normal distribution: {}'.format(stats.skew(train[col])))\n    print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train[train['meter_reading']<0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MMScaler = MinMaxScaler()\ntrain['meter_reading'] = MMScaler.fit_transform(train['meter_reading'].values.reshape(-1,1))\npickle.dump(MMScaler, open('target_Scaler.pkl', 'wb'))\ndel MMScaler\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.building_id = train.building_id.astype('int16')\ntrain.reset_index(inplace=True)\ntrain.set_index(['building_id','timestamp'], drop=False, inplace=True)\ntrain.sort_index(inplace=True)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"numericals.remove('meter_reading')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nX_train_cat = train[categoricals]\nX_train_num = train[numericals]\ny_train     = train[['meter_reading']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_train_cat.shape)\nprint(X_train_num.shape)\nprint(y_train.shape)   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_counts = X_train_cat.building_id.value_counts().to_frame('counts').sort_index()\ntrain_counts = train_counts[train_counts.counts >0]\ntrain_counts['cumsum_counts'] = train_counts.counts.cumsum()\ntrain_counts.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('average number of samples per building train:', train_counts.counts.mean())\nprint('maximum number of samples per building train:', train_counts.counts.max())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"from pandas to numpy:"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nX_train_cat = X_train_cat.values\nX_train_num = X_train_num.values\ny_train     = y_train.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_splits = train_counts.drop(train_counts.tail(1).index).cumsum_counts\n\nX_train_cat  = np.split(X_train_cat, train_splits)\nX_train_num  = np.split(X_train_num, train_splits)\ny_train      = np.split(y_train,     train_splits)\n \ndel train_splits\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_path(path_list):\n    for directory in path_list:\n        if not os.path.isdir(directory):\n            os.makedirs(directory)\n            os.makedirs(f'{directory}/categorical')\n            os.makedirs(f'{directory}/numerical')\n            os.makedirs(f'{directory}/Ys')\n        else:\n            shutil.rmtree(directory)\n            os.makedirs(directory)\n            os.makedirs(f'{directory}/categorical')\n            os.makedirs(f'{directory}/numerical')\n            os.makedirs(f'{directory}/Ys')\n            \ncreate_path(path_list=['train',\n                       'validation'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def save_data(train_size, TBPTT, Building_IDs, X_Cat, X_Num, Ys):     \n        \n    for cats_values, nums_values, y_values, building_id in tqdm(zip(X_Cat, \n                                                                    X_Num, \n                                                                    Ys, \n                                                                    Building_IDs), \n                                                                total=len(X_Cat)):           \n        \n        X_cat_loop, X_num_loop, y_loop = [], [], []\n        for i in range(TBPTT, cats_values.shape[0]+TBPTT, TBPTT):\n\n            cat = cats_values[i-TBPTT:i]\n            num = nums_values[i-TBPTT:i]\n            y_v = y_values[i-TBPTT:i]\n\n            # most of the time the number of data point in each building did not divide to pices of 100 so I need to pad them\n            # the last batch in the most of the time did not equal to TBPTT to I pad them:\n            zero_shape = cat.shape[0]\n            if zero_shape != TBPTT:\n\n                cat_pad = np.zeros((TBPTT, cat.shape[1]))\n                num_pad = np.zeros((TBPTT, num.shape[1]))\n                y_v_pad = np.zeros((TBPTT, y_v.shape[1]))\n\n                cat_pad[:zero_shape,:] = cat\n                num_pad[:zero_shape,:] = num\n                y_v_pad[:zero_shape,:] = y_v\n\n                cat = cat_pad\n                num = num_pad\n                y_v = y_v_pad\n\n            X_cat_loop.append(cat)\n            X_num_loop.append(num)\n            y_loop.append(    y_v)\n\n        X_cat_loop = np.array(X_cat_loop)\n        X_num_loop = np.array(X_num_loop)\n        y_loop     = np.array(y_loop)\n\n        train_cat = X_cat_loop[:-train_size]\n        val_cat   = X_cat_loop[-train_size:]\n        train_num = X_num_loop[:-train_size]\n        val_num   = X_num_loop[-train_size:]\n        train_Y   = y_loop[:-train_size]\n        val_Y     = y_loop[-train_size:]\n\n        np.save(f'train/categorical/'     +str(building_id)+'.npy', train_cat)\n        np.save(f'train/numerical/'       +str(building_id)+'.npy', train_num)\n        np.save(f'train/Ys/'              +str(building_id)+'.npy', train_Y)\n\n        np.save(f'validation/categorical/'+str(building_id)+'.npy', val_cat)\n        np.save(f'validation/numerical/'  +str(building_id)+'.npy', val_num)\n        np.save(f'validation/Ys/'         +str(building_id)+'.npy', val_Y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TBPTT = 48\n\nsave_data(train_size  = 4,\n          TBPTT       = TBPTT, \n          Building_IDs= list(train_counts.index), \n          X_Cat       = X_train_cat, \n          X_Num       = X_train_num, \n          Ys          = y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_train_cat, X_train_num, y_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_counts.index.name = 'building_id'\ntrain_counts.reset_index(inplace=True)\ntrain_counts.sort_values('counts',inplace=True)\ntrain_counts.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ASHRAE_Dataset(Dataset):\n    def __init__(self, \n                 data, \n                 cat_path, \n                 num_path, \n                 label_path, \n                 Reduce_Data=False, \n                 pct_reduce=1, \n                 is_val=False):\n        \n        self.data        = data        \n        self.cat_path    = cat_path\n        self.num_path    = num_path\n        self.label_path  = label_path        \n        self.len         = self.data.shape[0]\n        self.Reduce_Data = Reduce_Data\n        self.pct_reduce  = pct_reduce\n        self.is_val      = is_val\n        \n    def __getitem__(self,IXs):\n\n        B_ID = self.data.loc[IXs, 'building_id']\n\n        cats  = np.load(self.cat_path  +str(B_ID)+'.npy', allow_pickle=True)\n        nums  = np.load(self.num_path  +str(B_ID)+'.npy', allow_pickle=True)\n        label = np.load(self.label_path+str(B_ID)+'.npy', allow_pickle=True)\n\n        # if is not validation set - reduce the data\n        if self.Reduce_Data and not self.is_val:   \n            \n            seq_len     = cats.shape[0]\n            new_seq_len = int(seq_len * self.pct_reduce) \n            start_at = np.random.randint(low  = 0, \n                                         high = seq_len - new_seq_len)\n            \n            cats  =  cats[start_at : start_at + new_seq_len]\n            nums  =  nums[start_at : start_at + new_seq_len]\n            label = label[start_at : start_at + new_seq_len]\n        \n        cats = cats.astype(np.int16)\n\n        return [cats, nums, label]\n    \n    def __len__(self):\n        return self.len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyCollator(object):\n    def __init__(self):\n        pass\n    \n    def __call__(self, batch):\n        \n        cat   = [torch.tensor(item[0]) for item in batch]\n        num   = [torch.tensor(item[1]) for item in batch]\n        label = [torch.tensor(item[2]) for item in batch]\n\n        cat   = pad_sequence(cat,   batch_first=True, padding_value=0)\n        num   = pad_sequence(num,   batch_first=True, padding_value=0)\n        label = pad_sequence(label, batch_first=True, padding_value=0)\n        \n        cat   =   cat.type(torch.long)\n        num   =   num.type(torch.float)\n        label = label.type(torch.float)\n        \n        return [cat, num, label]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categorical_sizes = [c+1 for c in categorical_sizes] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ASHRAE_LSTM(nn.Module):\n\n    def __init__(self, \n                 hidden_dim, \n                 dropout_proba, \n                 categorical_sizes, \n                 LSTM_layers   = 1,\n                 bidirectional = False,\n                 Numeric_Feat  = None):\n        super(ASHRAE_LSTM, self).__init__()\n\n        self.hidden_dim    = hidden_dim\n        self.LSTM_layers   = LSTM_layers\n        self.bidirectional = bidirectional\n        \n        emb_dims           = [(c, min(50, (c+1)//2)) for c in categorical_sizes]        \n        self.emb_layers    = nn.ModuleList([nn.Embedding(x, y, padding_idx=0) \n                                            for x, y in emb_dims])        \n        total_embs_size    = sum([y for x, y in emb_dims])        \n        total_nums_size    = len(Numeric_Feat) if Numeric_Feat else 0\n        total_size         = total_embs_size + total_nums_size\n                \n        self.lstm    = nn.LSTM(input_size    = total_size, \n                               hidden_size   = hidden_dim,\n                               batch_first   = True,\n                               bidirectional = bidirectional,\n                               num_layers    = LSTM_layers)  \n    \n        self.dropout = nn.Dropout(p=dropout_proba)\n        \n        self.fc1     = nn.Linear(hidden_dim, 64)\n        self.fc2     = nn.Linear(64, 32)\n        self.fc3     = nn.Linear(32, 1)\n        \n        \n    def forward(self, cat_data, numeric_data, hidden_states, cell_states, seq_len, batch_size):\n\n        cat_embs = [emb_layer(cat_data[:,:, i]) \n                    for i, emb_layer in enumerate(self.emb_layers)]        \n        cat_embs = torch.cat(cat_embs, 2)        \n\n        x = torch.cat([cat_embs, numeric_data], 2)        \n        \n        output, (hidden_states, cell_states) = self.lstm(x, (hidden_states, cell_states))\n        \n        output = self.dropout(output)\n        output = F.relu(self.fc1(output))\n        output = self.dropout(output)\n        output = F.relu(self.fc2(output))\n        output = self.dropout(output)\n        output = self.fc3(output) \n        \n        return output, hidden_states, cell_states\n\n    def init_hidden(self, batch_size): # initialize the hidden state and the cell state to zeros\n        if self.bidirectional:\n            nb_directions = 2\n        else:\n            nb_directions = 1\n            \n        return (torch.zeros(nb_directions * self.LSTM_layers, batch_size, self.hidden_dim),\n                torch.zeros(nb_directions * self.LSTM_layers, batch_size, self.hidden_dim))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_parameters(model):\n    print('Number of parameters:')\n    print('{:0,.0f}'.format(sum(p.numel() for p in model.parameters() if p.requires_grad)))\n\nm = ASHRAE_LSTM(hidden_dim        =80,  \n                dropout_proba     =0.45, \n                categorical_sizes =categorical_sizes, \n                Numeric_Feat      = numericals)\ncount_parameters(m)\nm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del m\ngc.collect()\n\ndef RMSLE_LOSS(y_true, y_pred):\n    return np.sqrt(mean_squared_log_error(y_true, y_pred))\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\ndevice","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs        = 5\nBATCH_SIZE    = 16\nhidden_dim    = 100\ndropout_proba = 0.3\nReduce_Data   = True\npct_reduce    = 0.30\n\nTrainSet = ASHRAE_Dataset(data       =  train_counts.sort_values('counts'),\n                          cat_path   = 'train/categorical/', \n                          num_path   = 'train/numerical/', \n                          label_path = 'train/Ys/',\n                          Reduce_Data= Reduce_Data,\n                          pct_reduce = pct_reduce,\n                          is_val     = False)\n\nValSet = ASHRAE_Dataset(data        = train_counts.sort_values('counts'),\n                        cat_path    = 'validation/categorical/', \n                        num_path    = 'validation/numerical/', \n                        label_path  = 'validation/Ys/',\n                        is_val      = True)\n\ncollate = MyCollator()\n\ntrain_loader = DataLoader(TrainSet, \n                          batch_size=BATCH_SIZE, \n                          shuffle=False,\n                          collate_fn=collate)\n\nval_loader = DataLoader(ValSet, \n                        batch_size=BATCH_SIZE, \n                        shuffle=False,\n                        collate_fn=collate)\n\n\nmodel = ASHRAE_LSTM(hidden_dim        = hidden_dim, \n                    dropout_proba     = dropout_proba, \n                    categorical_sizes = categorical_sizes, \n                    Numeric_Feat      = numericals)\nmodel.to(device)\n\ncriterion = nn.MSELoss()\noptimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n\nBest_Test_loss = 10**10.0\ntotal_loss = 0\n\nfor epochs in tqdm(range(epochs)):\n    model.train()\n    \n    counter = 0\n    total_loss = 0\n    total_RMSLE_loss = 0\n    for cat, num, label in tqdm(train_loader, leave=False):\n\n        BS = cat.shape[0]\n        hidden_states, cell_states = model.init_hidden(BS)\n        hidden_states, cell_states = hidden_states.to(device), cell_states.to(device)\n        \n        for i, Batch_IX in enumerate(range(cat.shape[1]),start=1):\n            \n            cat_batch   = cat[:,Batch_IX,:,:].clone()\n            num_batch   = num[:,Batch_IX,:,:].clone()\n            label_batch = label[:,Batch_IX,:].clone()\n            \n            cat_batch, num_batch, label_batch = cat_batch.to(device), num_batch.to(device), label_batch.to(device)\n            \n            predictions, hidden_states, cell_states = model.forward(cat_batch, \n                                                                    num_batch, \n                                                                    hidden_states, \n                                                                    cell_states,\n                                                                    TBPTT,\n                                                                    BS)\n        \n            optimizer.zero_grad()\n            loss = criterion(predictions, label_batch)\n            total_loss+=loss.item()\n            counter+=1\n            loss.backward(retain_graph=True)\n            optimizer.step()\n            RMSLE_loss = RMSLE_LOSS(torch.clamp(label_batch.reshape(-1), min=0).cpu().detach().numpy(), \n                                    torch.clamp(predictions.reshape(-1), min=0).cpu().detach().numpy())\n            total_RMSLE_loss += RMSLE_loss\n            print(f'train - batch loss: {loss} avg loss: {total_loss/counter} RMSLE_loss: {RMSLE_loss} avg RMSLE loss: {total_RMSLE_loss/counter}' ,end='\\r')\n            torch.cuda.empty_cache()\n            \n            \n    print('validation time!')\n\n    y_preds = []\n    y_trues = []\n    with torch.no_grad():\n        for cat, num, label in tqdm(val_loader, leave=False):\n\n            BS=cat.shape[0]\n            hidden_states, cell_states = model.init_hidden(BS)\n            hidden_states, cell_states = hidden_states.to(device), cell_states.to(device)\n\n            for Batch_IX in tqdm(range(cat.shape[1]),leave=False):\n\n                cat_batch   = cat[:,Batch_IX,:,:].clone()\n                num_batch   = num[:,Batch_IX,:,:].clone()\n                label_batch = label[:,Batch_IX,:].clone()\n\n                cat_batch, num_batch = cat_batch.to(device), num_batch.to(device)\n\n                predictions, hidden_states, cell_states = model.forward(cat_batch, \n                                                                        num_batch, \n                                                                        hidden_states, \n                                                                        cell_states, \n                                                                        TBPTT,\n                                                                        BS)\n\n                y_preds.append(torch.clamp(predictions.reshape(-1), min=0).cpu().detach().numpy())\n                y_trues.append(torch.clamp(label_batch.reshape(-1), min=0).detach().numpy())\n                torch.cuda.empty_cache()\n                \n        y_preds = np.concatenate(y_preds)\n        y_trues = np.concatenate(y_trues)\n\n        RMSLE_loss = RMSLE_LOSS(y_trues, y_preds)\n\n        print('val RMSLE loss:', RMSLE_loss)\n\n        is_best = RMSLE_loss < Best_Test_loss\n        Best_Test_loss = min(RMSLE_loss, Best_Test_loss)\n\n        if is_best:\n            print('best val score so far!')\n            torch.save(model.state_dict(), 'Best_Model.pt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_counts\ngc.collect()\n\ntest        = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\nbuilding_df = pd.read_pickle(\"building_df.pkl\")\ntest = test.merge(building_df, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\n\ndel building_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test, NAlist = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\nweather_test.sort_values(['site_id','timestamp'], inplace=True)\nweather_test.fillna(method='ffill', inplace=True)\nweather_test.fillna(method='bfill', inplace=True)\nweather_test.isnull().sum().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"weather_scalers.pkl\", \"rb\") as ws:\n    weather_scalers = pickle.load(ws)\n\nfor col in tqdm(time_cols):\n    Scaler = weather_scalers[col]\n    weather_test[col] = Scaler.transform(weather_test[col].values.reshape(-1, 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.merge(weather_test, \n                  left_on = [\"site_id\", \"timestamp\"], \n                  right_on = [\"site_id\", \"timestamp\"], how = \"left\")\ndel weather_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test, NAlist = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"timestamp\"] = pd.to_datetime(test[\"timestamp\"])\ntest[\"hour\"]      = test[\"timestamp\"].dt.hour.astype(np.int8)\ntest[\"day\"]       = test[\"timestamp\"].dt.day.astype(np.int8)\ntest[\"weekday\"]   = test[\"timestamp\"].dt.weekday.astype(np.int8)\ntest[\"month\"]     = test[\"timestamp\"].dt.month.astype(np.int8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"Cat2Int.pkl\", \"rb\") as s:\n    Cat2Int = pickle.load(s)\n    \nfor col in categoricals:\n    if col!='building_id':\n        test[col] = test[col].map(Cat2Int[col]).fillna(1).astype(np.int8) \n    else:\n        test[col] = test[col].map(Cat2Int[col]).fillna(1).astype(np.int16)\n        \ntest.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.sort_values(['building_id','timestamp'], inplace=True)\nunorder_row_id = test['row_id'].values\n\nX_test_cat = test[categoricals]\nX_test_num = test[numericals]\n\ndel test\ngc.collect()\n\ntest_counts = X_test_cat.building_id.value_counts().to_frame('counts').sort_index()\ntest_counts = test_counts[test_counts.counts >0]\ntest_counts['cumsum_counts'] = test_counts.counts.cumsum()\ntest_counts.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('average number of samples per building train:', test_counts.counts.mean())\nprint('maximum number of samples per building train:', test_counts.counts.max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nX_test_cat = X_test_cat.values\nX_test_num = X_test_num.values\n\ntest_splits = test_counts.drop(test_counts.tail(1).index).cumsum_counts\n\nX_test_cat  = np.split(X_test_cat, test_splits)\nX_test_num  = np.split(X_test_num, test_splits)\n\ndel test_splits, test_counts\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 1\n\nmodel = ASHRAE_LSTM(hidden_dim        = hidden_dim, \n                    dropout_proba     = dropout_proba, \n                    categorical_sizes = categorical_sizes, \n                    Numeric_Feat      = numericals)\nmodel.load_state_dict(torch.load('Best_Model.pt'))\nmodel.eval()\nmodel.to(device)\n\ny_preds = []\nwith torch.no_grad():\n    for cat, num in tqdm(zip(X_test_cat, X_test_num), total=len(X_test_cat)):\n        \n        hidden_states, cell_states = model.init_hidden(BATCH_SIZE)\n        hidden_states, cell_states = hidden_states.to(device), cell_states.to(device)\n        \n        cat_batch, num_batch = (torch.tensor(cat, dtype=torch.long).to(device), \n                                torch.tensor(num, dtype=torch.float).to(device))\n\n        cat_batch, num_batch = cat_batch.unsqueeze(0), num_batch.unsqueeze(0)\n        \n        predictions, hidden_states, cell_states = model.forward(cat_batch, \n                                                                num_batch, \n                                                                hidden_states, \n                                                                cell_states, \n                                                                cat_batch.shape[1],\n                                                                BATCH_SIZE)\n\n        y_preds.append(torch.clamp(predictions.reshape(-1), min=0).cpu().detach().numpy())\n        torch.cuda.empty_cache()\n        \n    y_preds = np.concatenate(y_preds)           ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_test_cat, X_test_num\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"target_Scaler.pkl\", \"rb\") as f:\n    MMScaler = pickle.load(f)\n    \nresult = MMScaler.inverse_transform(y_preds.reshape(-1, 1))\nresult.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df = pd.DataFrame({'row_id':unorder_row_id,\n                          'meter_reading':result.reshape(-1)})\nresult_df.sort_values('row_id',inplace=True)\nresult_df.to_csv('submission.csv', index=False)\ncreate_path(path_list=['train','validation'])\nresult_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.9"}},"nbformat":4,"nbformat_minor":1}