{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Neural networks with tensorflow:\n\nThe data pre-processing is based on this notebook: https://www.kaggle.com/hmendonca/starter-eda-and-feature-selection-ashrae3"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os, gc\nimport random\nimport datetime\n\nfrom tqdm import tqdm_notebook as tqdm\n\n# matplotlib and seaborn for plotting\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom pandas.plotting import register_matplotlib_converters\nregister_matplotlib_converters()\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom sklearn import preprocessing\n\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"path = '../input/ashrae-energy-prediction'\n# Input data files are available in the \"../input/\" directory.\nfor dirname, _, filenames in os.walk(path):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Function to load the data:"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# unimportant features (see importance below)\nunimportant_cols = ['wind_direction', 'wind_speed', 'sea_level_pressure']\ntarget = 'meter_reading'\n\ndef load_data(source='train', path=path):\n    ''' load and merge all tables '''\n    assert source in ['train', 'test']\n    \n    building = pd.read_csv(f'{path}/building_metadata.csv', dtype={'building_id':np.uint16, 'site_id':np.uint8})\n    weather  = pd.read_csv(f'{path}/weather_{source}.csv', parse_dates=['timestamp'],\n                                                           dtype={'site_id':np.uint8, 'air_temperature':np.float16,\n                                                                  'cloud_coverage':np.float16, 'dew_temperature':np.float16,\n                                                                  'precip_depth_1_hr':np.float16},\n                                                           usecols=lambda c: c not in unimportant_cols)\n    df = pd.read_csv(f'{path}/{source}.csv', dtype={'building_id':np.uint16, 'meter':np.uint8}, parse_dates=['timestamp'])\n    df = df.merge(building, on='building_id', how='left')\n    df = df.merge(weather, on=['site_id', 'timestamp'], how='left')\n    return df\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training data:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load and display some samples\ntrain = load_data('train')\ntrain.sample(7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test data:"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = load_data('test')\ntest.sample(7)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data preprocessing:\n\nImportant: note that the target variable is computed as the logarithm of the column \"meter_reading\"."},{"metadata":{"trusted":true},"cell_type":"code","source":"# the counts above expose the missing data (Should we drop or refill the missing data?)\nprint(\"Ratio of available data (not NAN's):\")\ndata_ratios = train.count()/len(train)\ndata_ratios","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class ASHRAE3Preprocessor(object):\n    @classmethod\n    def fit(cls, df, data_ratios=data_ratios):\n        cls.avgs = df.loc[:,data_ratios < 1.0].mean()\n        cls.pu_le = LabelEncoder()\n        cls.pu_le.fit(df[\"primary_use\"])\n\n    @classmethod\n    def transform(cls, df):\n        df = df.fillna(cls.avgs) # refill NAN with averages\n        df['primary_use'] = np.uint8(cls.pu_le.transform(df['primary_use']))  # encode labels\n\n        # expand datetime into its components\n        df['hour'] = np.uint8(df['timestamp'].dt.hour)\n        df['day'] = np.uint8(df['timestamp'].dt.day)\n        df['weekday'] = np.uint8(df['timestamp'].dt.weekday)\n        df['month'] = np.uint8(df['timestamp'].dt.month)\n        df['year'] = np.uint8(df['timestamp'].dt.year-2000)\n        \n        # parse and cast columns to a smaller type\n        df.rename(columns={\"square_feet\": \"log_square_feet\"}, inplace=True)\n        df['log_square_feet'] = np.float16(np.log(df['log_square_feet']))\n        df['year_built'] = np.uint8(df['year_built']-1900)\n        df['floor_count'] = np.uint8(df['floor_count'])\n        \n        # remove redundant columns\n        for col in df.columns:\n            if col in ['timestamp', 'row_id']:\n                del df[col]\n    \n        # extract target column\n        if 'meter_reading' in df.columns:\n            df['meter_reading'] = np.log1p(df['meter_reading']).astype(np.float32) # comp metric uses log errors\n\n        return df\n        \nASHRAE3Preprocessor.fit(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = ASHRAE3Preprocessor.transform(train)\ntrain.sample(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Feature ranked correlation:"},{"metadata":{"trusted":true,"_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"%%time\nfig, ax = plt.subplots(figsize=(16,8))\n# use a ranked correlation to catch nonlinearities\ncorr = train[[col for col in train.columns if col != 'year']].sample(100100).corr(method='spearman')\n_ = sns.heatmap(corr, annot=True,\n                xticklabels=corr.columns.values,\n                yticklabels=corr.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# force the model to use the weather data instead of dates, to avoid overfitting to the past history\nfeatures = [col for col in train.columns if col not in [target, 'year', 'month', 'day']]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train-validation partition:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Shuffle:\nn = train.shape[0]\nix = np.random.permutation(n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training data:\nntrain = 15000000\nnvalid = 5000000\ntr_x, tr_y = train[features].iloc[ix[:ntrain]], train[target][ix[:ntrain]]\nva_x, va_y = train[features].iloc[ix[ntrain:(ntrain+nvalid)]], train[target][ix[ntrain:(ntrain+nvalid)]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xtr = tr_x.values\nytr = tr_y.values\nxval = va_x.values\nyval = va_y.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(xtr.shape)\nprint(ytr.shape)\nprint(xval.shape)\nprint(yval.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Variable normalization:"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\nxtr = scaler.fit_transform(xtr)\nxval = scaler.transform(xval)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(xtr.mean())\nprint(xtr.std())\nprint(xtr.shape)\n\nprint(xval.mean())\nprint(xval.std())\nprint(xval.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Neural network model with keras:"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The next function computes the RMSE to evaluate the model. Note that, as we have taken the logarithm of the target, this value is equivalent to the RLMSE that kaggle uses for the evaluation. The function is from this notebook: https://www.kaggle.com/isaienkov/keras-nn-with-embeddings-for-cat-features-1-24 "},{"metadata":{"trusted":true},"cell_type":"code","source":"def rmse(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true), axis=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = keras.Sequential()\n\n# Linear model that obtains 1.90 in the public ranking:\n#model.add(keras.layers.Dense(1, activation=\"linear\", input_shape=(13,)))\n\n# More complex network that obtains 1.27 in the public ranking:\nmodel.add(keras.layers.Dense(1000, activation=\"relu\", input_shape=(13,)))\n#model.add(keras.layers.Dense(1000, activation=\"relu\"))\nmodel.add(keras.layers.Dense(100, activation=\"relu\"))\nmodel.add(keras.layers.Dense(1))\n\nmodel.compile(optimizer=keras.optimizers.Adam(1.e-2), loss='mse', metrics=[rmse])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nepochs = 6\nhistory = model.fit(xtr, \n                    ytr, \n                    epochs=nepochs, \n                    validation_data=(xval, yval),\n                    batch_size=10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hd = history.history\n\nepochs = range(1, nepochs+1)\n\nplt.figure(figsize=(12,6))\n\nplt.subplot(1,2,1)\nplt.plot(epochs, hd['rmse'], \"r\", label=\"train\")\nplt.plot(epochs, hd['val_rmse'], \"b\", label=\"valid\")\nplt.grid(True)\nplt.xlabel(\"epoch\")\nplt.ylabel(\"rmse\")\nplt.title(\"RMSE\")\nplt.legend()\n\nplt.subplot(1,2,2)\nplt.plot(epochs, hd['loss'], \"r\", label=\"train\")\nplt.plot(epochs, hd['val_loss'], \"b\", label=\"valid\")\nplt.grid(True)\nplt.xlabel(\"epoch\")\nplt.ylabel(\"loss\")\nplt.title(\"Loss\")\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Predictions on the test set:"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = ASHRAE3Preprocessor.transform(test)\ntest.sample(7)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"tst_x = test[features].iloc[:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xtst = tst_x.values\nprint(xtst.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xtst = scaler.transform(xtst)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(xtst.mean())\nprint(xtst.std())\nprint(xtst.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tst_preds = model.predict(xtst)\nprint(tst_preds.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Note that we have to compute the exponential of the predictions, as we used the logarithm of the target to train the network:"},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv(f'{path}/sample_submission.csv')\nsubmission['meter_reading'] = np.clip(np.expm1(tst_preds), a_min=0, a_max=None) # clip min at zero","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head(9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}