{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport time\nimport warnings\nimport matplotlib.pyplot as plt\n\nwarnings.filterwarnings(\"ignore\")\ngc.enable()\ndevice_id = 0  # cpu -> -1, gpu -> 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# import Dataset to play with it\ntrain_data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv')\nbuilding = pd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\nweather_train = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv')\ntrain_data = train_data.merge(building, on='building_id', how='left')\ntrain_data = train_data.merge(weather_train, on=['site_id', 'timestamp'], how='left')\n\ntest_data = pd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv')\nweather_test = pd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv')\ntest_data = test_data.merge(building, on='building_id', how='left')\ntest_data = test_data.merge(weather_test, on=['site_id', 'timestamp'], how='left')\n\nprint (\"Done!\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Features that are likely predictive:\n\n**Buildings**\n* primary_use\n* square_feet\n* year_built\n* floor_count (may be too sparse to use)\n\n**Weather**\n* time of day\n* holiday\n* weekend\n* cloud_coverage + lags\n* dew_temperature + lags\n* precip_depth + lags\n* sea_level_pressure + lags\n* wind_direction + lags\n* wind_speed + lags\n\n**Train**\n* max, mean, min, std of the specific building historically\n* number of meters\n* number of buildings at a siteid"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.isnull().any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Based on this great kernel https://www.kaggle.com/arjanso/reducing-dataframe-memory-size-by-65\ndef reduce_mem_usage(df):\n    start_mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in df.columns:\n        if df[col].dtype != object:  # Exclude strings            \n            # Print current column type\n            print(\"******************************\")\n            print(\"Column: \",col)\n            print(\"dtype before: \",df[col].dtype)            \n            # make variables for Int, max and min\n            IsInt = False\n            mx = df[col].max()\n            mn = df[col].min()\n            print(\"min for this col: \",mn)\n            print(\"max for this col: \",mx)\n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(df[col]).all(): \n                NAlist.append(col)\n                df[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = df[col].fillna(0).astype(np.int64)\n            result = (df[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        df[col] = df[col].astype(np.uint8)\n                    elif mx < 65535:\n                        df[col] = df[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        df[col] = df[col].astype(np.uint32)\n                    else:\n                        df[col] = df[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)    \n            # Make float datatypes 32 bit\n            else:\n                df[col] = df[col].astype(np.float32)\n            \n            # Print new column type\n            print(\"dtype after: \",df[col].dtype)\n            print(\"******************************\")\n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return df, NAlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, _ = reduce_mem_usage(train_data)\ntest, _ = reduce_mem_usage(test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del building, weather_train, weather_test\ndel train_data\ndel test_data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_columns = train.columns.tolist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Simple data check"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for c in train_columns:\n    print(train[c].value_counts())\n    print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for c in test.columns:\n    print(test[c].value_counts())\n    print()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Feature preprocessing by sklearn pipeline"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.preprocessing import MinMaxScaler, OrdinalEncoder, QuantileTransformer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DateFeatureExtractor(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        pass\n    \n    def fit(self, X, y=None):\n        return self\n    \n    def transform(self, df, y=None):\n        # df = df.copy()\n        df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n        df[\"hour\"] = df[\"timestamp\"].dt.hour\n        df[\"day\"] = df[\"timestamp\"].dt.day\n        df[\"weekday\"] = df[\"timestamp\"].dt.weekday\n        df[\"month\"] = df[\"timestamp\"].dt.month\n        return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"minmax_features = [\n    \"year_built\",\n    \"hour\",\n    \"day\",\n    \"weekday\",\n    \"month\",\n]\n\nminmax_transformer = make_pipeline(\n    MinMaxScaler(),\n)\n\nnumeric_features = [\n    \"square_feet\",\n    \"air_temperature\",\n    \"cloud_coverage\",\n    \"dew_temperature\",\n    \"floor_count\",\n]\n\nnumeric_transformer = make_pipeline(\n    QuantileTransformer(\n        n_quantiles=100,\n        output_distribution=\"normal\",\n        random_state=0,\n    ),\n)\n\ncategorical_features = [\n    \"primary_use\",\n    \"meter\",\n    \"building_id\",\n]\n\ncategorical_transformer = make_pipeline(\n    OrdinalEncoder(),\n)\n\npreprocessor = make_pipeline(\n    DateFeatureExtractor(),\n    ColumnTransformer(\n        transformers=[\n            (\"numeric\", numeric_transformer, numeric_features),\n            (\"minmax\", minmax_transformer, minmax_features),\n            (\"categorical\", categorical_transformer, categorical_features),\n        ]\n    ),\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_train = preprocessor.fit_transform(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preprocessed_train[:5, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = np.log1p(train[[\"meter_reading\"]].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target[:5, :]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Chainer regresser model"},{"metadata":{"trusted":true},"cell_type":"code","source":"import chainer\nimport chainer.functions as F\nimport chainer.links as L","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"chainer.print_runtime_info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MLP(chainer.Chain):\n\n    def __init__(self, n_units=10, n_out=10):\n        super(MLP, self).__init__()\n        with self.init_scope():\n            # embed_id\n            self.embed_primary_use = L.EmbedID(16, 2)\n            self.embed_meter = L.EmbedID(4, 2)\n            self.embed_building_id = L.EmbedID(1449, 6)\n            # the size of the inputs to each layer will be inferred\n            self.l1 = L.Linear(None, n_units)  # n_in -> n_units\n            self.l2 = L.Linear(None, n_units)  # n_units -> n_units\n            self.l3 = L.Linear(None, n_out)  # n_units -> n_out\n\n    def forward(self, numeric_x, categorical_x):\n        # embed layers\n        e1 = self.embed_primary_use(categorical_x[:, 0])\n        e2 = self.embed_meter(categorical_x[:, 1])\n        e3 = self.embed_building_id(categorical_x[:, 2])\n        \n        # concat all inputs\n        x = F.concat((numeric_x, e1, e2, e3), axis=1)\n        \n        # main layers\n        h = F.dropout(F.relu(self.l1(x)), ratio=.1)\n        h = F.dropout(F.relu(self.l2(h)), ratio=.1)\n        return self.l3(h)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_validate(\n    model,\n    optimizer,\n    train,\n    validation,\n    n_epoch,\n    batchsize,\n    device,\n):\n    # 1. If the device is gpu(>=0), send model to the gpu.\n    if device >= 0:\n        model.to_gpu(device)\n\n    # 2. Setup optimizer\n    optimizer.setup(model)\n\n    # 3. Create iterator from datast\n    train_iter = chainer.iterators.SerialIterator(train, batchsize)\n    validation_iter = chainer.iterators.SerialIterator(\n        validation, batchsize, repeat=False, shuffle=False\n    )\n\n    # 4. Create Updater/Trainer\n    updater = chainer.training.StandardUpdater(train_iter, optimizer, device=device)\n    trainer = chainer.training.Trainer(updater, (n_epoch, 'epoch'), out='out')\n\n    # 5. Extend functionalities of trainer\n    trainer.extend(chainer.training.extensions.LogReport())\n    trainer.extend(\n        chainer.training.extensions.Evaluator(\n            validation_iter, model, device=device\n        ), name='val'\n    )\n    trainer.extend(\n        chainer.training.extensions.PrintReport(\n            ['epoch', 'main/loss', 'val/main/loss', 'elapsed_time']\n        )\n    )\n    trainer.extend(\n        chainer.training.extensions.PlotReport(\n            ['main/loss', 'val/main/loss'], x_key='epoch', file_name='loss.png'\n        )\n    )\n\n    # 6. Start training\n    trainer.run()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preprocessed_train = preprocessed_train[:1000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\n\nbatchsize = 512\nn_epoch = 20\nn_splits = 3\nseed = 666\n\nkf = KFold(n_splits=n_splits, shuffle=True, random_state=seed)\n\nmodels = []\nfor fold_n, (train_index, valid_index) in enumerate(kf.split(preprocessed_train)):\n    gc.collect()\n    \n    print()\n    print('Fold:', fold_n)\n    X_train, X_valid = preprocessed_train[train_index, :], preprocessed_train[valid_index, :]\n    y_train, y_valid = target[train_index], target[valid_index]\n    \n    model = MLP(64, 1)\n    regresser = L.Classifier(model, lossfun=F.mean_squared_error, accfun=F.mean_squared_error)\n    optimizer = chainer.optimizers.Adam()\n    \n    train_and_validate(\n        regresser,\n        optimizer,\n        chainer.datasets.TupleDataset(\n            X_train[:, :len(numeric_features) + len(minmax_features)].astype(\"f\"),\n            X_train[:, len(numeric_features) + len(minmax_features):].astype(\"i\"),\n            y_train,\n        ),\n        chainer.datasets.TupleDataset(\n            X_valid[:, :len(numeric_features) + len(minmax_features)].astype(\"f\"),\n            X_valid[:, len(numeric_features) + len(minmax_features):].astype(\"i\"),\n            y_valid,\n        ),\n        n_epoch,\n        batchsize,\n        device_id,\n    )\n    \n    models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del X_train, X_valid, y_train, y_valid, preprocessed_train, target\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Inference & Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"meter_reading\"] = 0.0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\nif device_id >= 0:\n    import cupy as cp\n\nstep_size = 50000\n\ni = 0\nres = []\nfor j in tqdm(range(int(np.ceil(test.shape[0] / 50000)))):\n    gc.collect()\n    batch = test[train_columns].iloc[i : i + step_size]\n    preprocessed_batch = preprocessor.transform(batch)\n    \n    device = chainer.get_device(device_id)\n    preprocessed_batch = device.send(preprocessed_batch)\n    \n    predictions = []\n    with chainer.using_config('train', False):\n        for model in models:\n            ndarray = model(\n                preprocessed_batch[:, :len(numeric_features) + len(minmax_features)].astype(\"f\"),\n                preprocessed_batch[:, len(numeric_features) + len(minmax_features):].astype(\"i\"),\n            )\n            ndarray.to_cpu()\n            predictions.append(ndarray.array)\n        \n    res.append(np.expm1(sum(predictions) / n_splits))\n    i += step_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = np.concatenate(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/ashrae-energy-prediction/sample_submission.csv')\nsubmission['meter_reading'] = res\nsubmission.loc[submission['meter_reading'] < 0, 'meter_reading'] = 0\nsubmission.to_csv('submission.csv', index=False)\nsubmission","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}