{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"N_EPOCHS = 75\nEARLY_STOPPING = 5\nBATCH_SIZE = 512\nFOLD_NUMBER = 0\n\ndef reduce_mem_usage(df, verbose=True, downcast_float=False):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if downcast_float:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        df[col] = df[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        df[col] = df[col].astype(np.float32)\n                    else:\n                        df[col] = df[col].astype(np.float64)    \n                \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PreprocessingUnit(object):\n    def __init__(self, categoricals, conts, replace_nans, target_variable):\n        self.categoricals = categoricals\n        self.conts = conts\n        self.replace_nans = replace_nans\n        self.target_variable = target_variable\n        \n        self.scaling_stats = {}\n        self.col_label_codes = {}\n        self.predictors = None\n        \n    def train_preprocessing(self, data):\n        for col in self.replace_nans:\n            data.loc[data[col] == -1, col] = data.loc[data[col] != -1, col].median()\n        \n        y = data[self.target_variable]\n        data = data.drop(columns=self.target_variable)\n        \n        self.predictors = list(data.columns)\n        \n        for col in self.conts:\n            self.scaling_stats[col] = {'mean':data[col].mean(), 'std':data[col].std()}\n            data[col] = (data[col] - self.scaling_stats[col]['mean']) / self.scaling_stats[col]['std']\n            \n        print(data[self.conts].isna().sum().sum())\n        print('Scaling completed!')\n        \n        cat_columns = []\n        for col in self.categoricals:\n            self.col_label_codes[col] = {k:v for v, k in enumerate(data[col].unique())}\n            cat_columns.append(data[col].map(self.col_label_codes[col]).values)\n            data = data.drop(columns=col)\n            gc.collect()\n        \n        print('Labeling completed!')\n        \n        gc.collect()\n        \n        return [data.values] + cat_columns, y.values\n    \n    def test_preprocessing(self, data, is_val=False):\n        for col in self.replace_nans:\n            data.loc[data[col] == -1, col] = data.loc[data[col] != -1, col].median()\n        \n        if is_val:\n            y = data[self.target_variable]\n            data = data.drop(columns=self.target_variable)\n            \n        data = data[self.predictors]\n        gc.collect()\n        \n        for col in self.conts:\n            data[col] = (data[col] - self.scaling_stats[col]['mean']) / self.scaling_stats[col]['std']\n            \n        cat_columns = []\n        for col in self.categoricals:\n            cat_columns.append(data[col].map(self.col_label_codes[col]).values)\n            data = data.drop(columns=col)\n            gc.collect()\n        \n        gc.collect()\n        if is_val:\n            return [data.values] + cat_columns, y.values\n        else:\n            return [data.values] + cat_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\nimport keras.backend as K\n\nfrom keras.layers import Input, Dense, Dropout, Embedding, Concatenate, Lambda\nfrom keras.models import Model\nfrom keras.optimizers import Adam, Nadam, Adamax\nfrom keras import callbacks\n\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.preprocessing import OneHotEncoder\n\n\nes = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=EARLY_STOPPING, verbose=False, mode='auto', restore_best_weights=True)\nrlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=3, min_lr=1e-6, mode='auto', verbose=False)\n\ndef root_mean_squared_error(y_true, y_pred):\n        return K.sqrt(K.mean(K.square(y_pred - y_true)))\n\ndef create_model(inp_dim):\n    inps = Input(shape=(inp_dim,))\n    build_inp = Input(shape=(1,))\n    meter_inp = Input(shape=(1,))\n    site_inp = Input(shape=(1,))\n    prime_inp = Input(shape=(1,))\n    dow_inp = Input(shape=(1,))\n    hod_inp = Input(shape=(1,))\n    \n    build_embed = Embedding(1449, 38)(build_inp)\n    build_embed = Lambda(lambda x: K.squeeze(x, 1))(build_embed)\n    \n    meter_embed = Embedding(4, 2)(meter_inp)\n    meter_embed = Lambda(lambda x: K.squeeze(x, 1))(meter_embed)\n    \n    site_embed = Embedding(16, 4)(site_inp)\n    site_embed = Lambda(lambda x: K.squeeze(x, 1))(site_embed)\n    \n    prime_embed = Embedding(16, 4)(site_inp)\n    prime_embed = Lambda(lambda x: K.squeeze(x, 1))(prime_embed)\n    \n    dow_embed = Embedding(7, 3)(dow_inp)\n    dow_embed = Lambda(lambda x: K.squeeze(x, 1))(dow_embed)\n    \n    hod_embed = Embedding(24, 5)(hod_inp)\n    hod_embed = Lambda(lambda x: K.squeeze(x, 1))(hod_embed)\n        \n    x = Concatenate(axis=-1)([inps,build_embed,meter_embed,site_embed,prime_embed, dow_embed, hod_embed])\n    \n    x = Dense(256, activation='elu')(x)\n    x = Dropout(0.2)(x)\n    x = Dense(128, activation='elu')(x)\n    x = Dropout(0.1)(x)\n    x = Dense(1, activation='softplus')(x)\n    model = Model(inputs=[inps,build_inp,meter_inp,site_inp,prime_inp,dow_inp,hod_inp], outputs=x)\n    model.compile(\n        optimizer=Adamax(lr=1e-3),\n        loss=root_mean_squared_error\n    )\n    return model\n        \n        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Main"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\n\nfrom os import path\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_columns = [\"building_id\", \"meter\", \"site_id\", \"primary_use\", \"tm_day_of_week\", \"tm_hour_of_day\"]\ntarget_column = \"meter_reading\"\nfold_col = 'k_folds'\ncont_columns = [\n 'air_temperature', 'air_temperature_max_lag24', 'air_temperature_mean_lag24', 'air_temperature_median_lag24',\n 'air_temperature_min_lag24', 'cloud_coverage', 'dew_temperature', 'dew_temperature_max_lag24',\n 'dew_temperature_mean_lag24', 'dew_temperature_median_lag24', 'dew_temperature_min_lag24',\n 'floor_count', 'max_at', 'mean_dt', 'min_at', 'min_dt', 'precip_depth_1_hr', 'sea_level_pressure',\n 'square_feet', 'wind_direction', 'wind_speed', 'year_built'\n]\nrequired_columns = cat_columns + cont_columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pp = PreprocessingUnit(categoricals=cat_columns, \n                       conts=cont_columns, \n                       replace_nans=['year_built', 'floor_count', 'precip_depth_1_hr'], \n                       target_variable=target_column)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = reduce_mem_usage(pd.read_parquet('/kaggle/input/baseline-preprocessing-leaks-train-fe/X_train.parquet.gzip')[required_columns + [target_column,fold_col]])\n\nX_train, X_val = X_train[X_train['k_folds'] != FOLD_NUMBER].reset_index(drop=True), X_train[X_train['k_folds'] == FOLD_NUMBER].reset_index(drop=True)\n\nX_train = X_train.drop(columns='k_folds')\nX_val = X_val.drop(columns='k_folds')\nprint(X_train.columns)\ngc.collect()\n\nX_train, y_train = pp.train_preprocessing(X_train)\ngc.collect()\nX_val, y_val = pp.test_preprocessing(X_val, is_val=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"neural_net = create_model(X_train[0].shape[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = neural_net.fit(\n            X_train, y_train, epochs=N_EPOCHS, batch_size=BATCH_SIZE, validation_data=(X_val, y_val), verbose=True, callbacks=[es, rlr]\n        )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(history.history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data again"},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_train\ndel y_train\ndel X_val\ndel y_val\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del history\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = reduce_mem_usage(pd.read_parquet('/kaggle/input/baseline-preprocessing-leaks-train-fe/X_test.parquet.gzip')[required_columns], downcast_float=True)\nX_test = pp.test_preprocessing(X_test, is_val=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = neural_net.predict(X_test, batch_size=BATCH_SIZE, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"prediction = prediction.flatten()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('prediction.npy', prediction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}