{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.\n\nfrom keras.models import Model, load_model\nfrom keras.layers import Input, Dropout, Dense, Embedding, SpatialDropout1D, concatenate, BatchNormalization, Flatten\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nfrom keras import backend as K\nfrom keras.models import Model\nfrom keras.losses import mean_squared_error as mse_loss\n\nfrom keras import optimizers\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def average_imputation(df, column_name):\n    imputation = df.groupby(['timestamp'])[column_name].mean()\n    \n    df.loc[df[column_name].isnull(), column_name] = df[df[column_name].isnull()][[column_name]].apply(lambda x: imputation[df['timestamp'][x.index]].values)\n    del imputation\n    return df\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"building_df = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")\nweather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")\ntrain = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\n\ntrain = train.merge(building_df, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\ntrain = train.merge(weather_train, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"])\ndel weather_train\n\ntrain.loc[(train['meter']==0) & (train['site_id']==0) & (train['timestamp']<'2016-05-21 00:00:00'), 'drop'] = True\ntrain = train[train['drop']!=True]\n\ntrain[\"timestamp\"] = pd.to_datetime(train[\"timestamp\"])\ntrain[\"hour\"] = train[\"timestamp\"].dt.hour\ntrain[\"weekday\"] = train[\"timestamp\"].dt.weekday\n\ntrain = average_imputation(train, 'wind_speed')\n\nbeaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n          (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n\nfor item in beaufort:\n    train.loc[(train['wind_speed']>=item[1]) & (train['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n\ndel train[\"timestamp\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle = LabelEncoder()\ntrain[\"primary_use\"] = le.fit_transform(train[\"primary_use\"])\n\ncategoricals = [\"site_id\", \"building_id\", \"primary_use\", \"hour\", \"weekday\",  \"meter\"]\n\ndrop_cols = [\"sea_level_pressure\", \"wind_speed\", \"wind_direction\"]\n\nnumericals = [\"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\n              \"dew_temperature\", \"precip_depth_1_hr\", \"floor_count\", 'beaufort_scale']\n\nfeat_cols = categoricals + numericals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = np.log1p(train[\"meter_reading\"])\n\ndel train[\"meter_reading\"] \n\ntrain = train.drop(drop_cols, axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Based on this great kernel https://www.kaggle.com/arjanso/reducing-dataframe-memory-size-by-65\ndef reduce_mem_usage(df):\n    start_mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in df.columns:\n        if df[col].dtype != object:  # Exclude strings            \n            # Print current column type\n            print(\"******************************\")\n            print(\"Column: \",col)\n            print(\"dtype before: \",df[col].dtype)            \n            # make variables for Int, max and min\n            IsInt = False\n            mx = df[col].max()\n            mn = df[col].min()\n            print(\"min for this col: \",mn)\n            print(\"max for this col: \",mx)\n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(df[col]).all(): \n                NAlist.append(col)\n                df[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = df[col].fillna(0).astype(np.int64)\n            result = (df[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        df[col] = df[col].astype(np.uint8)\n                    elif mx < 65535:\n                        df[col] = df[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        df[col] = df[col].astype(np.uint32)\n                    else:\n                        df[col] = df[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)    \n            # Make float datatypes 32 bit\n            else:\n                df[col] = df[col].astype(np.float32)\n            \n            # Print new column type\n            print(\"dtype after: \",df[col].dtype)\n            print(\"******************************\")\n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return df, NAlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, NAlist = reduce_mem_usage(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def model(dense_dim_1=64, dense_dim_2=32, dense_dim_3=32, dense_dim_4=16, \ndropout1=0.2, dropout2=0.1, dropout3=0.1, dropout4=0.1, lr=0.001):\n\n    #Inputs\n    site_id = Input(shape=[1], name=\"site_id\")\n    building_id = Input(shape=[1], name=\"building_id\")\n    meter = Input(shape=[1], name=\"meter\")\n    primary_use = Input(shape=[1], name=\"primary_use\")\n    square_feet = Input(shape=[1], name=\"square_feet\")\n    year_built = Input(shape=[1], name=\"year_built\")\n    air_temperature = Input(shape=[1], name=\"air_temperature\")\n    cloud_coverage = Input(shape=[1], name=\"cloud_coverage\")\n    dew_temperature = Input(shape=[1], name=\"dew_temperature\")\n    hour = Input(shape=[1], name=\"hour\")\n    precip = Input(shape=[1], name=\"precip_depth_1_hr\")\n    weekday = Input(shape=[1], name=\"weekday\")\n    beaufort_scale = Input(shape=[1], name=\"beaufort_scale\")\n   \n    #Embeddings layers\n    emb_site_id = Embedding(16, 2)(site_id)\n    emb_building_id = Embedding(1449, 6)(building_id)\n    emb_meter = Embedding(4, 2)(meter)\n    emb_primary_use = Embedding(16, 2)(primary_use)\n    emb_hour = Embedding(24, 3)(hour)\n    emb_weekday = Embedding(7, 2)(weekday)\n\n    concat_emb = concatenate([\n           Flatten() (emb_site_id)\n         , Flatten() (emb_building_id)\n         , Flatten() (emb_meter)\n         , Flatten() (emb_primary_use)\n         , Flatten() (emb_hour)\n         , Flatten() (emb_weekday)\n    ])\n    \n    categ = Dropout(dropout1)(Dense(dense_dim_1,activation='relu') (concat_emb))\n    categ = BatchNormalization()(categ)\n    categ = Dropout(dropout2)(Dense(dense_dim_2,activation='relu') (categ))\n    \n    #main layer\n    main_l = concatenate([\n          categ\n        , square_feet\n        , year_built\n        , air_temperature\n        , cloud_coverage\n        , dew_temperature\n        , precip\n        , beaufort_scale\n    ])\n    \n    main_l = Dropout(dropout3)(Dense(dense_dim_3,activation='relu') (main_l))\n    main_l = BatchNormalization()(main_l)\n    main_l = Dropout(dropout4)(Dense(dense_dim_4,activation='relu') (main_l))\n    \n    #output\n    output = Dense(1) (main_l)\n\n    model = Model([ site_id,\n                    building_id, \n                    meter, \n                    primary_use, \n                    square_feet, \n                    year_built, \n                    air_temperature,\n                    cloud_coverage,\n                    dew_temperature, \n                    hour,\n                    weekday, \n                    precip,\n                    beaufort_scale], output)\n\n    model.compile(optimizer = Adam(lr=lr),\n                  loss= mse_loss,\n                  metrics=[root_mean_squared_error])\n    return model\n\ndef root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_keras_data(df, num_cols, cat_cols):\n    cols = num_cols + cat_cols\n    X = {col: np.array(df[col]) for col in cols}\n    return X\n\ndef train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold, patience=3):\n    early_stopping = EarlyStopping(patience=patience, verbose=1)\n    model_checkpoint = ModelCheckpoint(\"model_\" + str(fold) + \".hdf5\",\n                                       save_best_only=True, verbose=1, monitor='val_root_mean_squared_error', mode='min')\n\n    hist = keras_model.fit(X_t, y_train, batch_size=batch_size, epochs=epochs,\n                            validation_data=(X_v, y_valid), verbose=1,\n                            callbacks=[early_stopping, model_checkpoint])\n\n    keras_model = load_model(\"model_\" + str(fold) + \".hdf5\", custom_objects={'root_mean_squared_error': root_mean_squared_error})\n    \n    return keras_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold, StratifiedKFold\n\noof = np.zeros(len(train))\nbatch_size = 1024\nepochs = 10\nmodels = []\n\nfolds = 4\nseed = 666\n\nkf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed)\n\nfor fold_n, (train_index, valid_index) in enumerate(kf.split(train, train['building_id'])):\n    print('Fold:', fold_n)\n    X_train, X_valid = train.iloc[train_index], train.iloc[valid_index]\n    y_train, y_valid = target.iloc[train_index], target.iloc[valid_index]\n    X_t = get_keras_data(X_train, numericals, categoricals)\n    X_v = get_keras_data(X_valid, numericals, categoricals)\n    \n    keras_model = model(dense_dim_1=64, dense_dim_2=32, dense_dim_3=32, dense_dim_4=16, \n                        dropout1=0.2, dropout2=0.1, dropout3=0.1, dropout4=0.1, lr=0.001)\n    mod = train_model(keras_model, X_t, y_train, batch_size, epochs, X_v, y_valid, fold_n, patience=3)\n    models.append(mod)\n    print('*'* 50)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel train, target, X_train, X_valid, y_train, y_valid, X_t, X_v, kf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\ntest = test.merge(building_df, left_on = \"building_id\", right_on = \"building_id\", how = \"left\")\ndel building_df\ngc.collect()\ntest[\"primary_use\"] = le.transform(test[\"primary_use\"])\n\nweather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")\n\ntest = test.merge(weather_test, left_on = [\"site_id\", \"timestamp\"], right_on = [\"site_id\", \"timestamp\"], how = \"left\")\ndel weather_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"timestamp\"] = pd.to_datetime(test[\"timestamp\"])\ntest[\"hour\"] = test[\"timestamp\"].dt.hour\ntest[\"weekday\"] = test[\"timestamp\"].dt.weekday\n\ntest = average_imputation(test, 'wind_speed')\n\nfor item in beaufort:\n    test.loc[(test['wind_speed']>=item[1]) & (test['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n\n    \ntest = test[feat_cols]\ntest, NAlist = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\ni=0\nres = np.zeros((test.shape[0]),dtype=np.float32)\nstep_size = 50000\nfor j in tqdm(range(int(np.ceil(test.shape[0]/step_size)))):\n    for_prediction = get_keras_data(test.iloc[i:i+step_size], numericals, categoricals)\n    res[i:min(i+step_size,test.shape[0])] = \\\n       np.expm1(sum([model.predict(for_prediction, batch_size=1024)[:,0] for model in models])/folds)\n    i+=step_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')\nsubmission['meter_reading'] = res\nsubmission.loc[submission['meter_reading']<0, 'meter_reading'] = 0\nsubmission.to_csv('submission.csv', index=False)\nsubmission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}