{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dropout, Dense, Embedding, SpatialDropout1D, concatenate, BatchNormalization, Flatten\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.losses import mean_squared_error as mse_loss\n\nfrom keras import optimizers\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n\nfrom sklearn.model_selection import KFold\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"building_df = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\nweather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\ntrain = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\n\ntrain = train.merge(building_df, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\ntrain = train.merge(weather_train, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"])\ndel weather_train\n\ntrain[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"])\ntrain[\"hour\"] = train[\"timestamp\"].dt.hour\ntrain[\"day\"] = train[\"timestamp\"].dt.day\ntrain[\"weekday\"] = train[\"timestamp\"].dt.weekday\ntrain[\"month\"] = train[\"timestamp\"].dt.month\n\ntrain['day']-=1\ntrain['month']-=1\n\ndel train[\"timestamp\"]\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\ntrain[\"primary_use\"] = le.fit_transform(train[\"primary_use\"])\n\ncategoricals = [\"site_id\", \"building_id\", \"primary_use\", \"hour\", \"day\", \"weekday\", \"month\", \"meter\"]\n\ndrop_cols = [\"precip_depth_1_hr\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\"]\n\nnumericals = [\"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\n              \"dew_temperature\"]\n\nfeat_cols = categoricals + numericals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = np.log1p(train[\"meter_reading\"])\n\ndel train[\"meter_reading\"] \n\ntrain = train.drop(drop_cols + [\"floor_count\"], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Based on this great kernel https://www.kaggle.com/arjanso/reducing-dataframe-memory-size-by-65\ndef reduce_mem_usage(df):\n    start_mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in df.columns:\n        if df[col].dtype != object:  # Exclude strings            \n            # Print current column type\n            print(\"******************************\")\n            print(\"Column: \",col)\n            print(\"dtype before: \",df[col].dtype)            \n            # make variables for Int, max and min\n            IsInt = False\n            mx = df[col].max()\n            mn = df[col].min()\n            print(\"min for this col: \",mn)\n            print(\"max for this col: \",mx)\n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(df[col]).all(): \n                NAlist.append(col)\n                df[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = df[col].fillna(0).astype(np.int64)\n            result = (df[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        df[col] = df[col].astype(np.uint8)\n                    elif mx < 65535:\n                        df[col] = df[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        df[col] = df[col].astype(np.uint32)\n                    else:\n                        df[col] = df[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)    \n            # Make float datatypes 32 bit\n            else:\n                df[col] = df[col].astype(np.float32)\n            \n            # Print new column type\n            print(\"dtype after: \",df[col].dtype)\n            print(\"******************************\")\n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return df, NAlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, NAlist = reduce_mem_usage(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model(dense_dim_1=64, dense_dim_2=32, dense_dim_3=32, dense_dim_4=16, \ndropout1=0.2, dropout2=0.2, dropout3=0.1, dropout4=0.1, lr=0.0005):\n\n    #Inputs\n    site_id = Input(shape=[1], name=\"site_id\")\n    building_id = Input(shape=[1], name=\"building_id\")\n    meter = Input(shape=[1], name=\"meter\")\n    primary_use = Input(shape=[1], name=\"primary_use\")\n    square_feet = Input(shape=[1], name=\"square_feet\")\n    year_built = Input(shape=[1], name=\"year_built\")\n    air_temperature = Input(shape=[1], name=\"air_temperature\")\n    cloud_coverage = Input(shape=[1], name=\"cloud_coverage\")\n    dew_temperature = Input(shape=[1], name=\"dew_temperature\")\n    hour = Input(shape=[1], name=\"hour\")\n    day = Input(shape=[1], name=\"day\")\n    weekday = Input(shape=[1], name=\"weekday\")\n    month = Input(shape=[1], name=\"month\")\n   \n    #Embeddings layers\n    emb_site_id = Embedding(16, 2)(site_id)\n    emb_building_id = Embedding(1449, 6)(building_id)\n    emb_meter = Embedding(4, 2)(meter)\n    emb_primary_use = Embedding(16, 2)(primary_use)\n    emb_hour = Embedding(24, 3)(hour)\n    emb_day = Embedding(31, 3)(day)\n    emb_weekday = Embedding(7, 2)(weekday)\n    emb_month = Embedding(12, 2)(month)\n\n    concat_emb = concatenate([\n           Flatten() (emb_site_id)\n         , Flatten() (emb_building_id)\n         , Flatten() (emb_meter)\n         , Flatten() (emb_primary_use)\n         , Flatten() (emb_hour)\n         , Flatten() (emb_day)\n         , Flatten() (emb_weekday)\n         , Flatten() (emb_month)\n    ])\n    \n    categ = Dropout(dropout1)(Dense(dense_dim_1,activation='relu') (concat_emb))\n    categ = BatchNormalization()(categ)\n    categ = Dropout(dropout2)(Dense(dense_dim_2,activation='relu') (categ))\n    \n    #main layer\n    main_l = concatenate([\n          categ\n        , square_feet\n        , year_built\n        , air_temperature\n        , cloud_coverage\n        , dew_temperature\n    ])\n    \n    main_l = Dropout(dropout3)(Dense(dense_dim_3,activation='relu') (main_l))\n    main_l = BatchNormalization()(main_l)\n    main_l = Dropout(dropout4)(Dense(dense_dim_4,activation='relu') (main_l))\n    \n    #output\n    output = Dense(1) (main_l)\n\n    model = Model([ site_id,\n                    building_id, \n                    meter, \n                    primary_use, \n                    square_feet, \n                    year_built, \n                    air_temperature,\n                    cloud_coverage,\n                    dew_temperature, \n                    hour,\n                    day,\n                    weekday, \n                    month ], output)\n\n    model.compile(optimizer = Adam(lr=lr),\n                  loss= mse_loss,\n                  metrics=[root_mean_squared_error])\n    return model\n\ndef root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_keras_data(df, num_cols, cat_cols):\n    cols = num_cols + cat_cols\n    X = {col: np.array(df[col]) for col in cols}\n    return X\n\ndef train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold, patience=3):\n    early_stopping = EarlyStopping(patience=patience, verbose=1)\n    model_checkpoint = ModelCheckpoint(\"model_\" + str(fold) + \".hdf5\",\n                                       save_best_only=True, verbose=1, monitor='val_root_mean_squared_error', mode='min')\n\n    hist = keras_model.fit(X_t, y_train, batch_size=batch_size, epochs=epochs,\n                            validation_data=(X_v, y_valid), verbose=1,\n                            callbacks=[early_stopping, model_checkpoint])\n\n    keras_model = load_model(\"model_\" + str(fold) + \".hdf5\", custom_objects={'root_mean_squared_error': root_mean_squared_error})\n    \n    return keras_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.meter.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 1024\nepochs = 10\nfolds = 4\nseed = 666","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntrain0=train[train['meter']==0]\ntarget0=target[train['meter']==0]\ntrain0.reset_index(drop=True,inplace=True)\ntarget0.reset_index(drop=True,inplace=True)\noof = np.zeros(len(train0))\n\nmodels0 = []\n\n\n\nkf = KFold(n_splits = folds, shuffle = True, random_state = seed)\n\nfor fold_n, (train_index, valid_index) in enumerate(kf.split(train0)):\n    print('Fold:', fold_n)\n    X_train, X_valid = train0.iloc[train_index], train0.iloc[valid_index]\n    y_train, y_valid = target0.iloc[train_index], target0.iloc[valid_index]\n    X_t = get_keras_data(X_train, numericals, categoricals)\n    X_v = get_keras_data(X_valid, numericals, categoricals)\n    \n    keras_model = model(dense_dim_1=64, dense_dim_2=32, dense_dim_3=32, dense_dim_4=16, \n                        dropout1=0.2, dropout2=0.1, dropout3=0.1, dropout4=0.1, lr=0.005)\n    mod = train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold_n, patience=3)\n    models0.append(mod)\n    print('*'* 50)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel train0, target0, X_train, X_valid, y_train, y_valid, X_t, X_v, kf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1=train[train['meter']==1]\ntarget1=target[train['meter']==1]\ntrain1.reset_index(drop=True,inplace=True)\ntarget1.reset_index(drop=True,inplace=True)\noof = np.zeros(len(train1))\n\nmodels1 = []\n\n\n\nkf = KFold(n_splits = folds, shuffle = True, random_state = seed)\n\nfor fold_n, (train_index, valid_index) in enumerate(kf.split(train1)):\n    print('Fold:', fold_n)\n    X_train, X_valid = train1.iloc[train_index], train1.iloc[valid_index]\n    y_train, y_valid = target1.iloc[train_index], target1.iloc[valid_index]\n    X_t = get_keras_data(X_train, numericals, categoricals)\n    X_v = get_keras_data(X_valid, numericals, categoricals)\n    \n    keras_model = model(dense_dim_1=64, dense_dim_2=32, dense_dim_3=32, dense_dim_4=16, \n                        dropout1=0.2, dropout2=0.1, dropout3=0.1, dropout4=0.1, lr=0.005)\n    mod = train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold_n, patience=3)\n    models1.append(mod)\n    print('*'* 50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train1, target1, X_train, X_valid, y_train, y_valid, X_t, X_v, kf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train2=train[train['meter']==2]\ntarget2=target[train['meter']==2]\ntrain2.reset_index(drop=True,inplace=True)\ntarget2.reset_index(drop=True,inplace=True)\noof = np.zeros(len(train2))\n\nmodels2 = []\n\n\n\nkf = KFold(n_splits = folds, shuffle = True, random_state = seed)\n\nfor fold_n, (train_index, valid_index) in enumerate(kf.split(train2)):\n    print('Fold:', fold_n)\n    X_train, X_valid = train2.iloc[train_index], train2.iloc[valid_index]\n    y_train, y_valid = target2.iloc[train_index], target2.iloc[valid_index]\n    X_t = get_keras_data(X_train, numericals, categoricals)\n    X_v = get_keras_data(X_valid, numericals, categoricals)\n    \n    keras_model = model(dense_dim_1=64, dense_dim_2=32, dense_dim_3=32, dense_dim_4=16, \n                        dropout1=0.2, dropout2=0.1, dropout3=0.1, dropout4=0.1, lr=0.005)\n    mod = train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold_n, patience=3)\n    models2.append(mod)\n    print('*'* 50)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train2, target2, X_train, X_valid, y_train, y_valid, X_t, X_v, kf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train3=train[train['meter']==3]\ntarget3=target[train['meter']==3]\ntrain3.reset_index(drop=True,inplace=True)\ntarget3.reset_index(drop=True,inplace=True)\noof = np.zeros(len(train3))\n\nmodels3 = []\n\n\n\nkf = KFold(n_splits = folds, shuffle = True, random_state = seed)\n\nfor fold_n, (train_index, valid_index) in enumerate(kf.split(train3)):\n    print('Fold:', fold_n)\n    X_train, X_valid = train3.iloc[train_index], train3.iloc[valid_index]\n    y_train, y_valid = target3.iloc[train_index], target3.iloc[valid_index]\n    X_t = get_keras_data(X_train, numericals, categoricals)\n    X_v = get_keras_data(X_valid, numericals, categoricals)\n    \n    keras_model = model(dense_dim_1=64, dense_dim_2=32, dense_dim_3=32, dense_dim_4=16, \n                        dropout1=0.2, dropout2=0.1, dropout3=0.1, dropout4=0.1, lr=0.005)\n    mod = train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold_n, patience=3)\n    models3.append(mod)\n    print('*'* 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train3, target3, X_train, X_valid, y_train, y_valid, X_t, X_v, kf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Testing phase"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\ntest = test.merge(building_df, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\ndel building_df\ngc.collect()\ntest[\"primary_use\"] = le.transform(test[\"primary_use\"])\n\nweather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\n\ntest = test.merge(weather_test, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"], how = \"left\")\ntest = test.drop(drop_cols, axis = 1)\ndel weather_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"timestamp\"] = pd.to_datetime(test[\"timestamp\"])\ntest[\"hour\"] = test[\"timestamp\"].dt.hour\ntest[\"day\"] = test[\"timestamp\"].dt.day\ntest[\"weekday\"] = test[\"timestamp\"].dt.weekday\ntest[\"month\"] = test[\"timestamp\"].dt.month\n\ntest['day']-=1\ntest['month']-=1\n\ntest = test[feat_cols]\ntest, NAlist = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\ndef pred(X_test, models, batch_size=50000):\n    i=0\n    folds=4\n    res=[]\n    print('iterations', (X_test.shape[0] + batch_size -1) // batch_size)\n    '''iterations = (X_test.shape[0] + batch_size -1) // batch_size\n    print('iterations', iterations)\n\n    y_test_pred_total = np.zeros(X_test.shape[0])\n    for i, model in enumerate(models):\n        print(f'predicting {i}-th model')\n        for k in tqdm(range(iterations)):\n            y_pred_test = model.predict(X_test[k*batch_size:(k+1)*batch_size])#, num_iteration=model.best_iteration)\n            y_test_pred_total[k*batch_size:(k+1)*batch_size] += y_pred_test\n\n    y_test_pred_total /= len(models)\n    return y_test_pred_total'''\n    for j in tqdm(range(int(np.ceil(X_test.shape[0]/batch_size)))):\n        for_prediction = get_keras_data(X_test.iloc[i:i+batch_size], numericals, categoricals)\n        res.append(np.expm1(sum([model.predict(for_prediction) for model in models])/folds))\n        i+=batch_size\n    return np.concatenate(res)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test0=test[test['meter']==0]\ntest0.reset_index(drop=True,inplace=True)\ny_test0 = pred(test0, models0)\n\n#sns.distplot(y_test0)\n\ndel test0\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test1=test[test['meter']==1]\ntest1.reset_index(drop=True,inplace=True)\ny_test1 = pred(test1, models1)\n\n#sns.distplot(y_test1)\n\ndel test1\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test2=test[test['meter']==2]\ntest2.reset_index(drop=True,inplace=True)\ny_test2 = pred(test2, models2)\n\n#sns.distplot(y_test2)\n\ndel test2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test3=test[test['meter']==3]\ntest3.reset_index(drop=True,inplace=True)\ny_test3 = pred(test3, models3)\n\n#sns.distplot(y_test3)\n\ndel test3\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')\nsubmission.loc[test['meter'] == 0, 'meter_reading'] = np.expm1(y_test0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.loc[test['meter'] == 1, 'meter_reading'] = np.expm1(y_test1)\nsubmission.loc[test['meter'] == 2, 'meter_reading'] = np.expm1(y_test2)\nsubmission.loc[test['meter'] == 3, 'meter_reading'] = np.expm1(y_test3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.loc[submission['meter_reading']<0, 'meter_reading'] = 0\nsubmission.to_csv('submission.csv', index=False)\nsubmission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}