{"cells":[{"metadata":{},"cell_type":"markdown","source":"# This kernel shows how to make a baseline submission using the fastai library using Google Colab\n# You can find this on google colab on this link\nhttps://colab.research.google.com/drive/1Y08AaPFCIwwHEAvaazN6JtNHsUWd8Ugq\n\n## NOTE: This solution yielded me a score of 1.22 on the public leaderboard"},{"metadata":{"id":"8B-AcCZVqMBL","colab_type":"text"},"cell_type":"markdown","source":"## Downloading from Kaggle with CLI"},{"metadata":{"id":"y1vih3VDqP_p","colab_type":"code","outputId":"e7520a14-5fee-4450-8c4b-4a530b2c986f","colab":{"resources":{"http://localhost:8080/nbextensions/google.colab/files.js":{"data":"Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=","ok":true,"headers":[["content-type","application/javascript"]],"status":200,"status_text":""}},"base_uri":"https://localhost:8080/","height":88},"trusted":false},"cell_type":"code","source":"# Run this cell and select the kaggle.json file downloaded\n# from the Kaggle account settings page.\nfrom google.colab import files\nfiles.upload()","execution_count":0,"outputs":[]},{"metadata":{"id":"BbCV5bBLqWZ_","colab_type":"code","outputId":"3a54471a-d464-4143-84c0-993bc588467e","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"# Let's make sure the kaggle.json file is present.\n!ls -lha kaggle.json","execution_count":0,"outputs":[]},{"metadata":{"id":"6DRUHwkdqXv9","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"# Next, install the Kaggle API client.\n!pip install -q kaggle","execution_count":0,"outputs":[]},{"metadata":{"id":"X62nRRggBIEM","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"def submit_kaggle(submission_path, competition, message=\"submission\"):\n    !kaggle competitions submit -c {competition} -f {submission_path} -m \"{message}\"","execution_count":0,"outputs":[]},{"metadata":{"id":"w-Fx2f2LqZDS","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"# The Kaggle API client expects this file to be in ~/.kaggle,\n# so move it there.\n!mkdir -p ~/.kaggle\n!cp kaggle.json ~/.kaggle/\n\n# This permissions change avoids a warning on Kaggle tool startup.\n!chmod 600 ~/.kaggle/kaggle.json","execution_count":0,"outputs":[]},{"metadata":{"id":"ZYRsT-eHqac3","colab_type":"code","outputId":"15cc0878-86d5-4413-acde-ddb0302393b4","colab":{"base_uri":"https://localhost:8080/","height":408},"trusted":false},"cell_type":"code","source":"# List available datasets.\n!kaggle datasets list","execution_count":0,"outputs":[]},{"metadata":{"id":"8rtBGzY0qboJ","colab_type":"code","outputId":"be4a7e92-cfae-4996-d782-96eb826a521b","colab":{"base_uri":"https://localhost:8080/","height":340},"trusted":false},"cell_type":"code","source":"# Copy the stackoverflow data set locally.\n#!kaggle datasets download -d stackoverflow/stack-overflow-2018-developer-survey\n!kaggle competitions download -c ashrae-energy-prediction -p /content/ashrae-energy-prediction","execution_count":0,"outputs":[]},{"metadata":{"id":"3Ek0-Dx7HikA","colab_type":"code","outputId":"7b6cc67b-4993-492c-ca6a-e129fd72fdb7","colab":{"base_uri":"https://localhost:8080/","height":51},"trusted":false},"cell_type":"code","source":"ls /content/ashrae-energy-prediction","execution_count":0,"outputs":[]},{"metadata":{"id":"Dm_YmXeDMdAe","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"DATA = \"/content/ashrae-energy-prediction/\"\n","execution_count":0,"outputs":[]},{"metadata":{"id":"Rs2BclAuqd1Q","colab_type":"code","outputId":"9dfff8f6-815f-4fb7-b794-37640f69b238","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":false},"cell_type":"code","source":"#!cd /content/ashrae-energy-prediction\n#!unzip *.zip","execution_count":0,"outputs":[]},{"metadata":{"id":"BMCnPAxNH5vP","colab_type":"code","outputId":"4e2fe8f8-559b-4336-c028-95d260d0a91a","colab":{"base_uri":"https://localhost:8080/","height":187},"trusted":false},"cell_type":"code","source":"!unzip /content/ashrae-energy-prediction/sample_submission.csv.zip -d {DATA}\n!unzip /content/ashrae-energy-prediction/test.csv.zip -d {DATA}\n!unzip /content/ashrae-energy-prediction/train.csv.zip -d {DATA}\n!unzip /content/ashrae-energy-prediction/weather_test.csv.zip -d {DATA}\n!unzip /content/ashrae-energy-prediction/weather_train.csv.zip -d {DATA}","execution_count":0,"outputs":[]},{"metadata":{"id":"wLeTr9QNs0kx","colab_type":"code","outputId":"e2aa960c-d2e5-4c5d-a0bc-7c924353f5f5","colab":{"base_uri":"https://localhost:8080/","height":85},"trusted":false},"cell_type":"code","source":"ls /content/ashrae-energy-prediction","execution_count":0,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"id":"yLzLylfFGtix","colab_type":"code","colab":{}},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport gc, math\n\nfrom pandas.tseries.holiday import USFederalHolidayCalendar as calendar\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tqdm import tqdm\nfrom sklearn.preprocessing import LabelEncoder\nfrom tqdm import tqdm\nfrom datetime import datetime","execution_count":0,"outputs":[]},{"metadata":{"id":"Fnrw8t5lGti1","colab_type":"text"},"cell_type":"markdown","source":"##### Set the size and styles of graphs"},{"metadata":{"trusted":true,"id":"akqr_tWnGti2","colab_type":"code","colab":{}},"cell_type":"code","source":"sns.set(rc={'figure.figsize':(11,8)})\nsns.set(style=\"whitegrid\")","execution_count":0,"outputs":[]},{"metadata":{"id":"by706sfIImVt","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"#DATA = \"/content/\"\n","execution_count":0,"outputs":[]},{"metadata":{"id":"34WaaL82L-Q1","colab_type":"code","outputId":"b1610f7f-1cdd-4403-c58e-24a9829804c8","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"ls","execution_count":0,"outputs":[]},{"metadata":{"id":"nVyO5YSKGti4","colab_type":"text"},"cell_type":"markdown","source":"### Read the dataset\nData is given in different CSV files which we will need to merge afterwards. \n\n`train.csv` only contains the ID of the building and meter related information including our target variable to be predicted (`meter_reading`). This `building_id` is foreign key in `building_metadata.csv`. All the information related to the buildings are given in this file. \n\n\nSame goes for `weather_train.csv` and `building_metadata.csv` files with common column (foreign key) `site_id`. So all three files are related and we will have to join these tables later"},{"metadata":{"trusted":true,"id":"ffRY5soPGti4","colab_type":"code","outputId":"c88d69d8-df29-4569-f989-c421b593be1e","colab":{"base_uri":"https://localhost:8080/","height":51}},"cell_type":"code","source":"%%time\nmetadata_df = pd.read_csv(f\"{DATA}building_metadata.csv\")\ntrain_df = pd.read_csv(f\"{DATA}train.csv\", parse_dates=['timestamp'])\ntest_df = pd.read_csv(f'{DATA}test.csv', parse_dates=['timestamp'])\nweather_train_df = pd.read_csv(f'{DATA}weather_train.csv', parse_dates=['timestamp'])\nweather_test_df = pd.read_csv(f'{DATA}weather_test.csv', parse_dates=['timestamp'])","execution_count":0,"outputs":[]},{"metadata":{"id":"UTTgAng98W4H","colab_type":"code","outputId":"5dd31728-49e2-40e2-d9bb-77a1efea5b4f","colab":{"base_uri":"https://localhost:8080/","height":282},"trusted":false},"cell_type":"code","source":"pd.Series(np.log1p(train_df['meter_reading'])).hist()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"62NshJe7Gti6","colab_type":"code","outputId":"b3b8ec87-3d1a-4102-c1bb-1ee35a1483d2","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"train_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"4SX_crrxGti8","colab_type":"code","outputId":"88843e10-8644-4dde-ce20-a16335ae168c","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"metadata_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"bhhQcVD9Gti-","colab_type":"code","outputId":"6d2c5de1-cbbd-4ac0-a9aa-4f299e937963","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"weather_train_df.shape","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"UjNIx8zIGti_","colab_type":"code","outputId":"1258d2d2-30f5-471c-98b4-c2aa7ad044b6","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"weather_train_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"JvKsvCijGtjB","colab_type":"code","outputId":"a5f2fa5a-9cc3-4974-9ca3-e22b8e079e86","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"test_df.head()","execution_count":0,"outputs":[]},{"metadata":{"id":"QRhBknk7GtjD","colab_type":"text"},"cell_type":"markdown","source":"### Align timestamps\nTimestap data is not in their local time. As energy consumptions are related to the local time, an alighment is nescessary before using timestamp. \n\nThe credit goes to [this kernel](https://www.kaggle.com/nz0722/aligned-timestamp-lgbm-by-meter-type) for the idea. Refer it for more details and explanation about below code."},{"metadata":{"trusted":true,"id":"KpWBnWunGtjD","colab_type":"code","colab":{}},"cell_type":"code","source":"weather = pd.concat([weather_train_df,weather_test_df],ignore_index=True)\nweather_key = ['site_id', 'timestamp']\n\ntemp_skeleton = weather[weather_key + ['air_temperature']].drop_duplicates(subset=weather_key).sort_values(by=weather_key).copy()\n\n# calculate ranks of hourly temperatures within date/site_id chunks\ntemp_skeleton['temp_rank'] = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.date])['air_temperature'].rank('average')\n\n# create a dataframe of site_ids (0-16) x mean hour rank of temperature within day (0-23)\ndf_2d = temp_skeleton.groupby(['site_id', temp_skeleton.timestamp.dt.hour])['temp_rank'].mean().unstack(level=1)\n\n# Subtract the columnID of temperature peak by 14, getting the timestamp alignment gap.\nsite_ids_offsets = pd.Series(df_2d.values.argmax(axis=1) - 14)\nsite_ids_offsets.index.name = 'site_id'\n\ndef timestamp_align(df):\n    df['offset'] = df.site_id.map(site_ids_offsets)\n    df['timestamp_aligned'] = (df.timestamp - pd.to_timedelta(df.offset, unit='H'))\n    df['timestamp'] = df['timestamp_aligned']\n    del df['timestamp_aligned']\n    return df","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"EohIYMBDGtjF","colab_type":"code","outputId":"f965c84d-cd46-42be-df2b-bd6a68564fc8","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"weather_train_df.tail()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"y50Vk1xDGtjH","colab_type":"code","colab":{}},"cell_type":"code","source":"weather_train_df = timestamp_align(weather_train_df)\nweather_test_df = timestamp_align(weather_test_df)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"qlfWP9nuGtjI","colab_type":"code","colab":{}},"cell_type":"code","source":"del weather \ndel df_2d\ndel temp_skeleton\ndel site_ids_offsets","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"4KFu1hCuGtjK","colab_type":"code","outputId":"47de2b33-e3c0-4a96-a60f-9063d152278d","colab":{"base_uri":"https://localhost:8080/","height":289}},"cell_type":"code","source":"weather_train_df.tail()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"KoQveNoJGtjM","colab_type":"code","colab":{}},"cell_type":"code","source":"def add_lag_feature(weather_df, window=3):\n    group_df = weather_df.groupby('site_id')\n    cols = ['air_temperature', 'cloud_coverage', 'dew_temperature', 'precip_depth_1_hr']\n    rolled = group_df[cols].rolling(window=window, min_periods=0)\n    lag_mean = rolled.mean().reset_index().astype(np.float16)\n    lag_std = rolled.std().reset_index().astype(np.float16)\n    for col in cols:\n        weather_df[f'{col}_mean_lag{window}'] = lag_mean[col]","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"6qTtoQD6GtjN","colab_type":"code","colab":{}},"cell_type":"code","source":"add_lag_feature(weather_train_df, window=72)\nadd_lag_feature(weather_test_df, window=72)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"Yncwdn7GGtjP","colab_type":"code","outputId":"40a8d98b-e922-4f41-b5d5-fc6948d1014b","colab":{"base_uri":"https://localhost:8080/","height":119}},"cell_type":"code","source":"weather_train_df.columns","execution_count":0,"outputs":[]},{"metadata":{"id":"soMOTJUyGtjR","colab_type":"text"},"cell_type":"markdown","source":"### Fill NaNs in weather data by interpolation"},{"metadata":{"trusted":true,"id":"5A5xTwR8GtjR","colab_type":"code","outputId":"435d3f5a-bc0d-4793-ec7d-9e8f71b7ee2a","colab":{"base_uri":"https://localhost:8080/","height":272}},"cell_type":"code","source":"weather_train_df.isna().sum()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"LqAEZNBLGtjU","colab_type":"code","outputId":"454e678f-7861-4aa0-b939-186d9ee3fa36","colab":{"base_uri":"https://localhost:8080/","height":272}},"cell_type":"code","source":"weather_test_df.isna().sum()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"V6lDAX8BGtjW","colab_type":"code","colab":{}},"cell_type":"code","source":"weather_train_df = weather_train_df.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"1WsmU_0QGtjZ","colab_type":"code","colab":{}},"cell_type":"code","source":"weather_test_df = weather_test_df.groupby('site_id').apply(lambda group: group.interpolate(limit_direction='both'))","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"o6QOcMztGtjd","colab_type":"code","outputId":"55922a32-deef-4ec4-bbdd-d06edf9742d1","colab":{"base_uri":"https://localhost:8080/","height":272}},"cell_type":"code","source":"weather_train_df.isna().sum()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"L2ap0zXJGtjh","colab_type":"code","outputId":"611ccd49-4cc5-46c4-dc5c-f4714f8c12e6","colab":{"base_uri":"https://localhost:8080/","height":272}},"cell_type":"code","source":"weather_test_df.isna().sum()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"kls_Nl6LGtjj","colab_type":"code","colab":{}},"cell_type":"code","source":"# Since loss metric is RMSLE\ntrain_df['meter_reading'] = np.log1p(train_df['meter_reading'])","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"ZjkKren0Gtjk","colab_type":"code","outputId":"d25adace-6805-4326-fd2e-d1f0f9d7ce4d","colab":{"base_uri":"https://localhost:8080/","height":309}},"cell_type":"code","source":"weather_train_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"Z7tlDb-OGtjn","colab_type":"code","colab":{}},"cell_type":"code","source":"## Function to reduce the memory usage\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"CFP5cXVKGtjo","colab_type":"code","colab":{}},"cell_type":"code","source":"le = LabelEncoder()\nmetadata_df['primary_use'] = le.fit_transform(metadata_df['primary_use'])","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"ZBqk3NHjGtjq","colab_type":"code","outputId":"c1003760-b3b0-40eb-f8b1-a607e6635b12","colab":{"base_uri":"https://localhost:8080/","height":102}},"cell_type":"code","source":"metadata_df = reduce_mem_usage(metadata_df)\ntrain_df = reduce_mem_usage(train_df)\ntest_df = reduce_mem_usage(test_df)\nweather_train_df = reduce_mem_usage(weather_train_df)\nweather_test_df = reduce_mem_usage(weather_test_df)","execution_count":0,"outputs":[]},{"metadata":{"id":"VwyWxxycGtjs","colab_type":"text"},"cell_type":"markdown","source":"#### Get insights of shapes and first few data rows of all the files"},{"metadata":{"trusted":true,"id":"ihiAs8fEGtjs","colab_type":"code","outputId":"6d22db60-6132-4113-9da6-b774eb6f2745","colab":{"base_uri":"https://localhost:8080/","height":102}},"cell_type":"code","source":"print (f'Training data shape: {train_df.shape}')\nprint (f'Weather training shape: {weather_train_df.shape}')\nprint (f'Weather training shape: {weather_test_df.shape}')\nprint (f'Weather testing shape: {metadata_df.shape}')\nprint (f'Test data shape: {test_df.shape}')","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"-R8mevrGGtju","colab_type":"code","outputId":"9c16ae57-dba3-4797-bdc9-d833ea530b39","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"train_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"vSmuwGQVGtjw","colab_type":"code","outputId":"76bb1a47-0aee-4f85-c908-d8822532b14f","colab":{"base_uri":"https://localhost:8080/","height":309}},"cell_type":"code","source":"weather_train_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"3_WtL8BIGtjx","colab_type":"code","outputId":"d3ef48bd-b3a1-470d-e043-ea7f7cd36da7","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"metadata_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"CpWsrhRkGtjz","colab_type":"code","outputId":"8d2ea8f8-0e1f-4d3c-ab8e-530adba659d6","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"test_df.head()","execution_count":0,"outputs":[]},{"metadata":{"id":"s0nuzcKKGtj1","colab_type":"text"},"cell_type":"markdown","source":"#### Merge necessary files\nAs mentioned previously, to get a single dataframe for training and a single data frame for testing with all the feature included, we need to join the tables/files which are related by foreign keys. Let's first merge/join training data."},{"metadata":{"trusted":true,"id":"jejQXairGtj1","colab_type":"code","outputId":"3e1aff4a-d4c6-442b-9ecc-1c5d33d35a48","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"train_df.head()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"w8iGKGVzGtj3","colab_type":"code","outputId":"97b49947-8ea0-4dbe-b4ed-f48c3a781401","colab":{"base_uri":"https://localhost:8080/","height":51}},"cell_type":"code","source":"%%time\nfull_train_df = train_df.merge(metadata_df, on='building_id', how='left')\nfull_train_df = full_train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')","execution_count":0,"outputs":[]},{"metadata":{"id":"Oo6S7WkbGtj4","colab_type":"text"},"cell_type":"markdown","source":"Drop all NaN rows which are generated by timestamp alignment"},{"metadata":{"trusted":true,"id":"KbVYGbUEGtj4","colab_type":"code","outputId":"449d199d-9447-4000-901a-b28823b5ad9b","colab":{"base_uri":"https://localhost:8080/","height":460}},"cell_type":"code","source":"full_train_df = full_train_df.loc[~(full_train_df['air_temperature'].isnull() & full_train_df['cloud_coverage'].isnull() & full_train_df['dew_temperature'].isnull() & full_train_df['precip_depth_1_hr'].isnull() & full_train_df['sea_level_pressure'].isnull() & full_train_df['wind_direction'].isnull() & full_train_df['wind_speed'].isnull() & full_train_df['offset'].isnull())]\n#full_train_df.loc[(full_train_df['air_temperature'].isnull() & full_train_df['cloud_coverage'].isnull() & full_train_df['dew_temperature'].isnull() & full_train_df['precip_depth_1_hr'].isnull() & full_train_df['sea_level_pressure'].isnull() & full_train_df['wind_direction'].isnull() & full_train_df['wind_speed'].isnull() & full_train_df['offset'].isnull())] = -1","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"ue1iXvnfGtj7","colab_type":"code","outputId":"4b210e8c-11fb-4644-e0a6-0b7b6ab9b16d","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"full_train_df.shape","execution_count":0,"outputs":[]},{"metadata":{"id":"KEzX-o71GtkA","colab_type":"text"},"cell_type":"markdown","source":"#### Reducing the memory usage\nLet's delete unnecessary dataframes from memory to lower the memory usage"},{"metadata":{"trusted":true,"id":"E0GCwwA8GtkA","colab_type":"code","outputId":"21fce5e0-77fa-4acd-ca79-abae7fb6d8b7","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"# Delete unnecessary dataframes to decrease memory usage\ndel train_df\ndel weather_train_df\ngc.collect()","execution_count":0,"outputs":[]},{"metadata":{"id":"y01k6J_8GtkC","colab_type":"text"},"cell_type":"markdown","source":"Now let's change the data types of necessary feature columns based on the range of the data values. This will lower the data usage. But **how**? Let's see. For example the datatype of feature `building_id` is `int64` but based on the range of this feature, it can be accomodated in lower range i.e. `int16`. So this will decrease the memory usage."},{"metadata":{"id":"mAWJrVUKGtkC","colab_type":"text"},"cell_type":"markdown","source":"#### Let's do the same for test data"},{"metadata":{"trusted":true,"id":"7mkQyza5GtkC","colab_type":"code","outputId":"36f128c5-fb4f-44d7-a0e0-63de3cc9fe4c","colab":{"base_uri":"https://localhost:8080/","height":51}},"cell_type":"code","source":"%%time\nfull_test_df = test_df.merge(metadata_df, on='building_id', how='left')\nfull_test_df = full_test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"AwWaifDvGtkE","colab_type":"code","outputId":"9811f4a7-7a39-4baf-85d2-edbd590240dd","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"full_test_df.shape","execution_count":0,"outputs":[]},{"metadata":{"id":"Xf9DfghxGtkF","colab_type":"text"},"cell_type":"markdown","source":"#### Reducing the memory usage\nLet's delete unnecessary dataframes from memory to lower the memory usage"},{"metadata":{"trusted":true,"id":"GZH1887iGtkF","colab_type":"code","outputId":"d2289fa4-3751-44ff-e8bb-e367cc92889a","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"# Delete unnecessary dataframes to decrease memory usage\ndel metadata_df\ndel weather_test_df\ndel test_df\ngc.collect()","execution_count":0,"outputs":[]},{"metadata":{"id":"nhHal3VAGtkH","colab_type":"text"},"cell_type":"markdown","source":"Now let's change the data types of necessary feature columns based on the range of the data values. This will lower the data usage. But **how**? Let's see. For example the datatype of feature `building_id` is `int64` but based on the range of this feature, it can be accomodated in lower range i.e. `int16`. So this will decrease the memory usage."},{"metadata":{"id":"nlWafWiWGtkH","colab_type":"text"},"cell_type":"markdown","source":"## Exploratory Data Analysis\nExploratory Data Analysis (EDA) is an open-ended process where we calculate statistics and make figures to find trends, anomalies, patterns, or relationships within the data."},{"metadata":{"id":"uJr4Kle6GtkI","colab_type":"text"},"cell_type":"markdown","source":"### Distribition of primary usage of buildings"},{"metadata":{"trusted":true,"id":"05A-PRVKGtkJ","colab_type":"code","outputId":"1e730e9f-f5e2-4f27-c2d8-4a218a938e20","colab":{"base_uri":"https://localhost:8080/","height":302}},"cell_type":"code","source":"ax = sns.barplot(pd.unique(full_train_df['primary_use']), full_train_df['primary_use'].value_counts())\nax.set(xlabel='Primary Usage', ylabel='# of records', title='Primary Usage vs. # of records')\nax.set_xticklabels(ax.get_xticklabels(), rotation=50, ha=\"right\")\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"id":"vFU3e4sVGtkM","colab_type":"text"},"cell_type":"markdown","source":"### Distribution of meter types"},{"metadata":{"trusted":true,"id":"jZzEyKsWGtkN","colab_type":"code","outputId":"c29200a2-059f-40e4-8c66-0c90034fbc7d","colab":{"base_uri":"https://localhost:8080/","height":295}},"cell_type":"code","source":"meter_types = {0: 'electricity', 1: 'chilledwater', 2: 'steam', 3: 'hotwater'}\nax = sns.barplot(np.vectorize(meter_types.get)(pd.unique(full_train_df['meter'])), full_train_df['meter'].value_counts())\nax.set(xlabel='Meter Type', ylabel='# of records', title='Meter type vs. # of records')\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"id":"j0Uo6EUpGtkO","colab_type":"text"},"cell_type":"markdown","source":"### Average meter reading for training dataset"},{"metadata":{"trusted":true,"id":"2Gwfm0nOGtkP","colab_type":"code","outputId":"f0fa0b38-5500-4314-dc31-0c2d1becaadd","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"# Average meter reading\nprint (f'Average meter reading: {full_train_df.meter_reading.mean()} kWh')","execution_count":0,"outputs":[]},{"metadata":{"id":"IBMDQdh7GtkS","colab_type":"text"},"cell_type":"markdown","source":"### Distribution of meter readings for each meter type"},{"metadata":{"trusted":true,"id":"s86xfYW0GtkS","colab_type":"code","outputId":"5e91b0d1-f53f-48fd-e557-aead54dfc350","colab":{"base_uri":"https://localhost:8080/","height":295}},"cell_type":"code","source":"ax = sns.barplot(np.vectorize(meter_types.get)(full_train_df.groupby(['meter'])['meter_reading'].mean().keys()), full_train_df.groupby(['meter'])['meter_reading'].mean())\nax.set(xlabel='Meter Type', ylabel='Meter reading', title='Meter type vs. Meter Reading')\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"id":"JvHqBooYGtkW","colab_type":"text"},"cell_type":"markdown","source":"### Distribution of buildings built in each year for both training and test datasets"},{"metadata":{"trusted":true,"id":"_U7EE17TGtkX","colab_type":"code","outputId":"039e9987-b329-49e7-9cdf-63f8d33ea264","colab":{"base_uri":"https://localhost:8080/","height":404}},"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(14, 6))\nax.set(xlabel='Year Built', ylabel='# Of Buildings', title='Buildings built in each year')\nfull_train_df['year_built'].value_counts(dropna=False).sort_index().plot(ax=ax)\nfull_test_df['year_built'].value_counts(dropna=False).sort_index().plot(ax=ax)\nax.legend(['Train', 'Test']);","execution_count":0,"outputs":[]},{"metadata":{"id":"gooZBRTiGtkY","colab_type":"text"},"cell_type":"markdown","source":"### Distribution of square feet area of buildings"},{"metadata":{"trusted":true,"id":"viNwnUJgGtkZ","colab_type":"code","outputId":"43cbd3b2-48ef-45d1-f93d-e819eb526944","colab":{"base_uri":"https://localhost:8080/","height":458}},"cell_type":"code","source":"fig, ax = plt.subplots(1,1,figsize=(15, 7))\nfull_train_df.groupby(['building_id'])['square_feet'].mean().plot(ax=ax)\nax.set(xlabel='Building ID', ylabel='Area in Square Feet', title='Square Feet area of buildings')\nplt.show()","execution_count":0,"outputs":[]},{"metadata":{"id":"sUApd3emGtkb","colab_type":"text"},"cell_type":"markdown","source":"## Feature Engineering\nThe joined dataframe (full_train_df) now has 20,216,100 rows, and 16 features in training dataset."},{"metadata":{"id":"4f9J8YntGtkc","colab_type":"text"},"cell_type":"markdown","source":"### Analysing missing data\nFirst let's count and fill missing data in training datasets"},{"metadata":{"trusted":true,"id":"SjkcMs54Gtkc","colab_type":"code","outputId":"09374a5b-012b-4676-809f-916ad6d19c7a","colab":{"base_uri":"https://localhost:8080/","height":700}},"cell_type":"code","source":"pd.DataFrame(full_train_df.isna().sum().sort_values(ascending=False), columns=['NaN Count'])","execution_count":0,"outputs":[]},{"metadata":{"id":"Jy_o3JgnGtkd","colab_type":"text"},"cell_type":"markdown","source":"Missing data can be filled in many ways. Here are few techniques to fill missing values: \n\n* Ignore the data row\n* Back-fill or forward-fill to propagate next or previous values respectively\n* Replace with some constant value outside fixed value range-999,-1 etc.\n* Replace with mean, median value\n\nFor now, we will go with last method. So let's fill all the missing data with it's average(mean) values of corresponding columns."},{"metadata":{"trusted":true,"id":"hI3dC_PuGtke","colab_type":"code","colab":{}},"cell_type":"code","source":"def mean_without_overflow_fast(col):\n    col /= len(col)\n    return col.mean() * len(col)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"VywkA7xTGtkf","colab_type":"code","colab":{}},"cell_type":"code","source":"missing_values = (100-full_train_df.count() / len(full_train_df) * 100).sort_values(ascending=False)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"MCSeC7gpGtkg","colab_type":"code","outputId":"1384cfa5-da41-46bc-c199-9d5532ea3d66","colab":{"base_uri":"https://localhost:8080/","height":51}},"cell_type":"code","source":"%%time\nmissing_features = full_train_df.loc[:, missing_values > 0.0]\nmissing_features = missing_features.apply(mean_without_overflow_fast)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"Op4oMVL9Gtkh","colab_type":"code","colab":{}},"cell_type":"code","source":"# Both train and test are interpolated with mean of train\nfor key in full_train_df.loc[:, missing_values > 0.0].keys():\n    if key == 'year_built' or key == 'floor_count':\n        full_train_df[key].fillna(math.floor(missing_features[key]), inplace=True)\n        full_test_df[key].fillna(math.floor(missing_features[key]), inplace=True)\n    else:\n        full_train_df[key].fillna(missing_features[key], inplace=True)\n        full_test_df[key].fillna(missing_features[key], inplace=True)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"6af6dq8EGtki","colab_type":"code","outputId":"ce5afe73-b9ee-4bf5-e6b8-e71ddbc42da3","colab":{"base_uri":"https://localhost:8080/","height":309}},"cell_type":"code","source":"full_train_df.tail()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"QvVycGCnGtkj","colab_type":"code","outputId":"75433a2f-30c7-4dbe-a74b-a892448d0d1a","colab":{"base_uri":"https://localhost:8080/","height":309}},"cell_type":"code","source":"full_test_df.tail()","execution_count":0,"outputs":[]},{"metadata":{"id":"09vDynhUGtkk","colab_type":"text"},"cell_type":"markdown","source":"So all the missing values for training and testing data is now filled with the mean of corresponding feature columns."},{"metadata":{"id":"KwqPinx0ICky","colab_type":"code","outputId":"926ab0b6-4fc9-40aa-a18a-419e03433fde","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"full_train_df.isna().sum().sum(), full_test_df.isna().sum().sum()","execution_count":0,"outputs":[]},{"metadata":{"id":"bpDAogasGtkl","colab_type":"text"},"cell_type":"markdown","source":"### Adding few more features"},{"metadata":{"id":"a70ZT77QGtkl","colab_type":"text"},"cell_type":"markdown","source":"First let's expand timestamp to multiple components"},{"metadata":{"trusted":true,"id":"c8NBa6BiGtkm","colab_type":"code","outputId":"8b96869f-4d68-4ed1-b715-452405e4d5a1","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"full_train_df['timestamp'].dtype","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"SwmPGN74Gtkp","colab_type":"code","colab":{}},"cell_type":"code","source":"full_train_df[\"timestamp\"] = pd.to_datetime(full_train_df[\"timestamp\"])\nfull_test_df[\"timestamp\"] = pd.to_datetime(full_test_df[\"timestamp\"])","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"usfDJG0hGtkr","colab_type":"code","colab":{}},"cell_type":"code","source":"def transform(df):\n    df['hour'] = np.uint8(df['timestamp'].dt.hour)\n    df['day'] = np.uint8(df['timestamp'].dt.day)\n    df['weekday'] = np.uint8(df['timestamp'].dt.weekday)\n    df['month'] = np.uint8(df['timestamp'].dt.month)\n    df['year'] = np.uint8(df['timestamp'].dt.year-1900)\n    \n    df['square_feet'] = np.log(df['square_feet'])\n    \n    return df","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"ahmzuwfzGtkt","colab_type":"code","colab":{}},"cell_type":"code","source":"full_train_df = transform(full_train_df)\nfull_test_df = transform(full_test_df)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"7r3Q8FTCGtkv","colab_type":"code","colab":{}},"cell_type":"code","source":"dates_range = pd.date_range(start='2015-12-31', end='2019-01-01')\nus_holidays = calendar().holidays(start=dates_range.min(), end=dates_range.max())\nfull_train_df['is_holiday'] = (full_train_df['timestamp'].dt.date.astype('datetime64').isin(us_holidays)).astype(np.int8)\nfull_test_df['is_holiday'] = (full_test_df['timestamp'].dt.date.astype('datetime64').isin(us_holidays)).astype(np.int8)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"mDvpKJ8wGtky","colab_type":"code","colab":{}},"cell_type":"code","source":"# Assuming 5 days a week for all the given buildings\nfull_train_df.loc[(full_train_df['weekday'] == 5) | (full_train_df['weekday'] == 6) , 'is_holiday'] = 1\nfull_test_df.loc[(full_test_df['weekday']) == 5 | (full_test_df['weekday'] == 6) , 'is_holiday'] = 1","execution_count":0,"outputs":[]},{"metadata":{"id":"DdlmV1nCGtk0","colab_type":"text"},"cell_type":"markdown","source":"### Removing weired data on site_id 0\nAs you can see above, this data looks weired until May 20. It is reported in this discussion by @barnwellguy that All electricity meter is 0 until May 20 for site_id == 0. Let's remove these data from training data."},{"metadata":{"trusted":true,"id":"2Ps7gTY1Gtk1","colab_type":"code","outputId":"6bec1b13-a80a-4c39-f6f2-d6bac6daf510","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"full_train_df.shape","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"YohOwBxGGtk3","colab_type":"code","colab":{}},"cell_type":"code","source":"full_train_df = full_train_df.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"LwyIPZX-Gtk6","colab_type":"code","outputId":"aaf24d32-1e35-4825-ab79-8ba28dcb8ba7","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"full_train_df.shape","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"Jd6ZekRdGtk8","colab_type":"code","outputId":"8292d506-f927-4ba0-f4da-5039e9b9013c","colab":{"base_uri":"https://localhost:8080/","height":215}},"cell_type":"code","source":"full_test_df = full_test_df.drop(['timestamp'], axis=1)\nfull_train_df = full_train_df.drop(['timestamp'], axis=1)\nprint (f'Shape of training dataset: {full_train_df.shape}')\nprint (f'Shape of testing dataset: {full_test_df.shape}')","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"00vEYxclGtk-","colab_type":"code","outputId":"f3ec679a-2654-4ff7-d524-f4313faf1ee8","colab":{"base_uri":"https://localhost:8080/","height":224}},"cell_type":"code","source":"full_train_df.tail()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"MskB7pm1GtlB","colab_type":"code","outputId":"b33927c6-4d83-415c-f590-af9eac8f7295","colab":{"base_uri":"https://localhost:8080/","height":215}},"cell_type":"code","source":"## Reducing memory\nfull_train_df = reduce_mem_usage(full_train_df)\nfull_test_df = reduce_mem_usage(full_test_df)\ngc.collect()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"gAGmpzeRGtlD","colab_type":"code","colab":{}},"cell_type":"code","source":"# def degToCompass(num):\n#     val=int((num/22.5)+.5)\n#     arr=[i for i in range(0,16)]\n#     return arr[(val % 16)]","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"-RJWJRs8GtlF","colab_type":"code","colab":{}},"cell_type":"code","source":"# full_train_df['wind_direction'] = full_train_df['wind_direction'].apply(degToCompass)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"h_EQgHnxGtlI","colab_type":"code","colab":{}},"cell_type":"code","source":"# beaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n#           (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n\n# for item in beaufort:\n#     full_train_df.loc[(full_train_df['wind_speed']>=item[1]) & (full_train_df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"_ADM_PvOGtlK","colab_type":"code","colab":{}},"cell_type":"code","source":"# le = LabelEncoder()\n# full_train_df['primary_use'] = le.fit_transform(full_train_df['primary_use'])\n\ncategoricals = ['site_id', 'building_id', 'primary_use', 'hour', 'weekday', 'meter',  'wind_direction', 'is_holiday']\n# drop_cols = ['sea_level_pressure', 'wind_speed']\nnumericals = ['square_feet', 'year_built', 'air_temperature', 'cloud_coverage',\n              'dew_temperature', 'precip_depth_1_hr', 'floor_count', 'air_temperature_mean_lag72',\n       'cloud_coverage_mean_lag72', 'dew_temperature_mean_lag72',\n       'precip_depth_1_hr_mean_lag72']\n\nfeat_cols = categoricals + numericals","execution_count":0,"outputs":[]},{"metadata":{"id":"R6Ak3rwdRRZR","colab_type":"code","outputId":"07582a0e-6d67-47e5-b81e-51c26c874dca","colab":{"base_uri":"https://localhost:8080/","height":317},"trusted":false},"cell_type":"code","source":"full_train_df[numericals].describe()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"Ejdov_dlGtlL","colab_type":"code","outputId":"cc9509e9-2795-4867-8eb1-93f4e2963df7","colab":{"base_uri":"https://localhost:8080/","height":224}},"cell_type":"code","source":"full_train_df.tail()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"OE7tMuawGtlN","colab_type":"code","outputId":"c7060b5b-a5df-49e5-89a0-0c5f97c9863c","colab":{"base_uri":"https://localhost:8080/","height":51}},"cell_type":"code","source":"full_train_df = reduce_mem_usage(full_train_df)\ngc.collect()","execution_count":0,"outputs":[]},{"metadata":{"id":"kLagcT2NPfPA","colab_type":"code","outputId":"3c3f7b77-0f34-4369-8188-30bebce47d4a","colab":{"base_uri":"https://localhost:8080/","height":153},"trusted":false},"cell_type":"code","source":"cat_card = full_train_df[categoricals].nunique().apply(lambda x: min(x, 50)).to_dict()\ncat_card","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"4YITzxgmGtlO","colab_type":"code","colab":{}},"cell_type":"code","source":"target = full_train_df[\"meter_reading\"]\nfull_train_df.to_pickle('full_train_df.pkl')\n#del full_train_df[\"meter_reading\"]","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"w3zmjlwAGtlQ","colab_type":"code","colab":{}},"cell_type":"code","source":"# full_train_df.drop(drop_cols, axis=1)\n# gc.collect()","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"id":"az4qvdLIGtlR","colab_type":"code","outputId":"f090cd58-52ee-42fd-b8f8-267b56f0727b","colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":"# Save the testing dataset to freeup the RAM. We'll read after training\nfull_test_df.to_pickle('full_test_df.pkl')\ndel full_test_df\ngc.collect()","execution_count":0,"outputs":[]},{"metadata":{"id":"AM8W4c5rS1Ay","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"import pickle","execution_count":0,"outputs":[]},{"metadata":{"id":"vNvvFHS5ULWl","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"with open('full_train_df.pkl', 'rb') as f:\n    full_train_df = pickle.load(f)","execution_count":0,"outputs":[]},{"metadata":{"id":"UC5SJBOLlyAw","colab_type":"code","outputId":"55530cd7-a938-4ffa-ef00-68e65ff1737b","colab":{"base_uri":"https://localhost:8080/","height":153},"trusted":false},"cell_type":"code","source":"full_train_df.columns","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"text","id":"Z-coGMWe3sEF"},"cell_type":"markdown","source":"# fastai tabular (sample)"},{"metadata":{"trusted":true,"colab_type":"code","id":"3sWTG9nq3sEG","colab":{}},"cell_type":"code","source":"from fastai.tabular import *","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"ARdBpZOa3sEI","colab":{},"trusted":false},"cell_type":"code","source":"from fastai.basic_train import *","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"c91cf8b8-dd96-47ff-bea4-b0f538913545","id":"M6Rs5Mhi3sEJ","colab":{"base_uri":"https://localhost:8080/","height":51},"trusted":false},"cell_type":"code","source":"ls","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"TFbRCGrk3sEL","colab":{},"trusted":false},"cell_type":"code","source":"#learner = learn.load(f'model1.bin')","execution_count":0,"outputs":[]},{"metadata":{"id":"VxpiwVoW5YI-","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"full_train_df_sample = full_train_df.sample(frac=0.1)","execution_count":0,"outputs":[]},{"metadata":{"id":"k7mZYQJD5xI1","colab_type":"code","outputId":"590581d4-b293-4540-a689-0436154ee1da","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"full_train_df_sample.shape","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"NvmiaVeJ3sEM","colab":{}},"cell_type":"code","source":"procs = [FillMissing, Categorify] # Took out Normalize to address NaN prob when getting mean","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"NTk-ye9I3sEO","colab":{}},"cell_type":"code","source":"valid_idx = range(len(full_train_df_sample)- int(full_train_df_sample.shape[0] * 0.1), len(full_train_df_sample))","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"W7O05I_u3sEP","colab":{}},"cell_type":"code","source":"#folds = 2\n#seed = 666\n\n#kf = StratifiedKFold(n_splits=folds, shuffle=False, random_state=seed)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"A3JoTjdi3sEQ","colab":{}},"cell_type":"code","source":"#index_splits = next(kf.split(full_train_df, full_train_df['building_id']))","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"xc6SHt-23sER","colab":{}},"cell_type":"code","source":"dep_var = \"meter_reading\"\ncat_names = categoricals\npath = DATA","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"rVb3DJbJ3sET","colab":{}},"cell_type":"code","source":"#full_train_df[dep_var] = target","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"R_ZLXi033sEU","colab":{}},"cell_type":"code","source":"#del target","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"586e4fb2-9ec1-4c83-89c7-e5e5fe20dbbe","id":"CYEzruUS3sEW","colab":{"base_uri":"https://localhost:8080/","height":221},"trusted":false},"cell_type":"code","source":"full_train_df_sample[dep_var]","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","outputId":"c6062f72-5c24-4d79-d0cc-f7d82d9a00fa","id":"bBK81A9G3sEX","colab":{"base_uri":"https://localhost:8080/","height":54}},"cell_type":"code","source":"# Not including test set first since takes too much space\ndata = TabularDataBunch.from_df(path, full_train_df_sample, dep_var, valid_idx=valid_idx, procs=procs, cat_names=cat_names) #, test_df=full_test_df)\nprint(data.train_ds.cont_names)  # `cont_names` defaults to: set(df)-set(cat_names)-{dep_var}","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","outputId":"73960f0d-e4da-4ad0-e4c8-e4219d81aa87","id":"o6JR76Jp3sEZ","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"(cat_x,cont_x),y = next(iter(data.train_dl))\nfor o in (cat_x, cont_x, y): print(to_np(o[:5]))","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"1HeWANyg3sEa","colab":{},"trusted":false},"cell_type":"code","source":"learn = tabular_learner(data, layers=[200,100], emb_szs=cat_card, metrics=rmse)\n","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"EgrvZyWI3sEb","outputId":"f4ae9692-25cf-45cd-962d-2a59227b2de1","colab":{"base_uri":"https://localhost:8080/","height":80},"trusted":false},"cell_type":"code","source":"learn.fit_one_cycle(1, 1e-2)","execution_count":0,"outputs":[]},{"metadata":{"id":"UHqH-Ghz7tgg","colab_type":"code","outputId":"6984040a-bc42-4bc7-ff5f-3e25320d9517","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":false},"cell_type":"code","source":"pred_batch1 = learn.pred_batch()\npred_batch1[: 5]","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"oU_WGlHH3sEd","colab":{},"trusted":false},"cell_type":"code","source":"learn.save('model_sample')","execution_count":0,"outputs":[]},{"metadata":{"id":"UQvH0DbQLV6y","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"learn.export('model_sample_export')","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"WIs4STpd3sEg","colab":{},"trusted":false},"cell_type":"code","source":"# Checking if predictions will be similar after loading\nlearn1 = tabular_learner(data, layers=[200,100], emb_szs=cat_card, metrics=rmse)","execution_count":0,"outputs":[]},{"metadata":{"id":"cCIbfcRQ8-8n","colab_type":"code","outputId":"dad15890-3b00-45c8-b584-65f6b6027f2d","colab":{"base_uri":"https://localhost:8080/","height":1000},"trusted":false},"cell_type":"code","source":"learn1.load('model_sample')","execution_count":0,"outputs":[]},{"metadata":{"id":"fUuz5iunB3Tv","colab_type":"code","outputId":"1859c021-ef71-43d9-aa68-cf8a188621bb","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":false},"cell_type":"code","source":"ls /content/ashrae-energy-prediction/","execution_count":0,"outputs":[]},{"metadata":{"id":"X-jfHHKV9JjC","colab_type":"code","outputId":"ef8fa04d-a5dd-496c-d60d-d692ed13746c","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":false},"cell_type":"code","source":"pred_batch2 = learn1.pred_batch()\npred_batch2[: 5]","execution_count":0,"outputs":[]},{"metadata":{"id":"bQZDA0MSM8p7","colab_type":"code","outputId":"15833d71-d147-45fc-d917-fc1fd4e1db71","colab":{"base_uri":"https://localhost:8080/","height":119},"trusted":false},"cell_type":"code","source":"learn2.data.train_ds","execution_count":0,"outputs":[]},{"metadata":{"id":"NF8sgM0SMaB2","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"tabList_sample = TabularList.from_df(full_train_df.head(10), cat_names=cat_names, procs=procs)\nlearn2 = load_learner(DATA, 'model_sample_export', test=tabList_sample)","execution_count":0,"outputs":[]},{"metadata":{"id":"UCEzBDZkM3IL","colab_type":"code","outputId":"bf6160c3-a2df-4bfc-a4f1-50e98ba116c0","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":false},"cell_type":"code","source":"pred_batch3 = learn2.pred_batch(ds_type=DatasetType.Test)\npred_batch3[: 5]","execution_count":0,"outputs":[]},{"metadata":{"id":"9Sz5gjLTWfEd","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"full_test_df = pd.read_pickle('full_test_df.pkl')","execution_count":0,"outputs":[]},{"metadata":{"id":"yI6Vpnn_esCj","colab_type":"code","outputId":"38b90ddd-bd2a-400b-8032-3a2161861b78","colab":{"base_uri":"https://localhost:8080/","height":71},"trusted":false},"cell_type":"code","source":"nan_cols = full_test_df.columns.values[full_test_df.isna().sum().nonzero()[0]].tolist()","execution_count":0,"outputs":[]},{"metadata":{"id":"BvXYLapqYMvt","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"for col_ in nan_cols:\n    median = full_test_df[col_].median()\n    full_test_df[col_] = full_test_df[col_].fillna(median)\n","execution_count":0,"outputs":[]},{"metadata":{"id":"Q__dfO9UQIru","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"# Need to input data as tabList when learner is loaded\n# Note that the learner is loaded without the datasets\ntabList = TabularList.from_df(full_test_df, cat_names=cat_names, procs=procs)\nlearn2 = load_learner(DATA, 'model_sample_export', test=tabList)","execution_count":0,"outputs":[]},{"metadata":{"id":"TfIZrfEqtPc_","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"del tabList\ndel full_test_df","execution_count":0,"outputs":[]},{"metadata":{"id":"8jIegG2BQUJ_","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"pred_test = learn2.get_preds(ds_type=DatasetType.Test)","execution_count":0,"outputs":[]},{"metadata":{"id":"2bJAOi5iV2qU","colab_type":"code","outputId":"3a70d28f-f7b8-473b-bce3-0fc2b139e31e","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"len(pred_test[0]), full_test_df.shape","execution_count":0,"outputs":[]},{"metadata":{"id":"4A_XXMtDWG2t","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"pred_test_np = pred_test[0].numpy()","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"fa5c3891-ddeb-4bb7-dc19-7f3d940fd256","id":"E-1jCNNP3sEh","colab":{"base_uri":"https://localhost:8080/","height":122},"trusted":false},"cell_type":"code","source":"from google.colab import drive\ndrive.mount('/content/gdrive')","execution_count":0,"outputs":[]},{"metadata":{"trusted":false,"colab_type":"code","id":"xxVG9_nP3sFL","colab":{}},"cell_type":"code","source":"res = pred_test_np","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"f9e82602-dfc2-44b3-8980-e6a1d7975989","id":"aYelSNB63sFR","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"submission = pd.read_csv(f'{DATA}sample_submission.csv')\nsubmission.shape","execution_count":0,"outputs":[]},{"metadata":{"trusted":false,"colab_type":"code","id":"G8SV2NZT3sFS","outputId":"8c1d1ab0-ff96-4281-f385-0a214fe6a67b","colab":{"base_uri":"https://localhost:8080/","height":419}},"cell_type":"code","source":"submission = pd.read_csv(f'{DATA}sample_submission.csv')\n# Remember, we predicted the log consumption, so we have to get the exponential to convert it back!\nsubmission['meter_reading'] = np.expm1(res)\nsubmission.loc[submission['meter_reading']<0, 'meter_reading'] = 0\nsubmission.to_csv('submission_fastai_20191103.csv', index=False)\nsubmission","execution_count":0,"outputs":[]},{"metadata":{"id":"FSpvRpMN_vYH","colab_type":"code","outputId":"3ecd8059-b69d-422c-b99b-f27ae5c7f158","colab":{"base_uri":"https://localhost:8080/","height":204},"trusted":false},"cell_type":"code","source":"submission.head()","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"3uQoghou3sFU","colab":{},"trusted":false},"cell_type":"code","source":"!cp submission_fastai_20191103.csv /content/gdrive/'My Drive'/ashrae/submissions/submission_fastai_20191103.csv","execution_count":0,"outputs":[]},{"metadata":{"id":"gq1nxnIeuBYC","colab_type":"code","outputId":"c19894b1-3a05-4a5d-c7c3-61224fa9f8a1","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"ls /content/gdrive/'My Drive'/ashrae/submissions/","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"_X_c7jRf3sFV","outputId":"71ee3db0-a839-44b2-fa45-5312255b951a","colab":{"base_uri":"https://localhost:8080/","height":68},"trusted":false},"cell_type":"code","source":"!kaggle competitions submit -c ashrae-energy-prediction -f submission_fastai_20191103.csv -m \"Hello fastai tabular FCNN (corrected w/ exponential)\"","execution_count":0,"outputs":[]},{"metadata":{"id":"uC3keheMAF_3","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"del pred_test\ndel learn2\ndel learn","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"text","id":"T1EiHyjyC448"},"cell_type":"markdown","source":"# fastai tabular (all)"},{"metadata":{"trusted":true,"colab_type":"code","id":"Nul__975C44-","colab":{}},"cell_type":"code","source":"from fastai.tabular import *","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"shmbDsJSC44_","colab":{},"trusted":false},"cell_type":"code","source":"from fastai.basic_train import *","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"c91cf8b8-dd96-47ff-bea4-b0f538913545","id":"gIprtETFC45A","colab":{"base_uri":"https://localhost:8080/","height":51},"trusted":false},"cell_type":"code","source":"ls","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"GsIcKIwXC45B","colab":{},"trusted":false},"cell_type":"code","source":"#learner = learn.load(f'model1.bin')","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"pxyhAhMaC45D","colab":{},"trusted":false},"cell_type":"code","source":"full_train_df_sample = full_train_df.sample(frac=1)","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"af769fce-96f0-4cd3-8bb6-6d0f9f46e996","id":"ApRNoPtKC45E","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"full_train_df_sample.shape","execution_count":0,"outputs":[]},{"metadata":{"id":"NGBkOVqFLlsP","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"del full_train_df","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"VTotkA6ZC45G","colab":{}},"cell_type":"code","source":"procs = [FillMissing, Categorify] # Took out Normalize to address NaN prob when getting mean","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"odd1xMqtC45H","colab":{}},"cell_type":"code","source":"#valid_idx = range(len(full_train_df_sample)- int(full_train_df_sample.shape[0] * 0.1), len(full_train_df_sample))\nvalid_idx = np.random.permutation(len(full_train_df_sample))[: int(full_train_df_sample.shape[0] * 0.05)]","execution_count":0,"outputs":[]},{"metadata":{"id":"KFOFHFzOEAPn","colab_type":"code","outputId":"41b8fae9-86d9-4bd1-fae7-b3adc1ed5c0a","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"valid_idx","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"wxzloKiXC45K","colab":{}},"cell_type":"code","source":"#folds = 2\n#seed = 666\n\n#kf = StratifiedKFold(n_splits=folds, shuffle=False, random_state=seed)","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"Os5eyABXC45L","colab":{}},"cell_type":"code","source":"#index_splits = next(kf.split(full_train_df, full_train_df['building_id']))","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"E9HZOPOjC45O","colab":{}},"cell_type":"code","source":"dep_var = \"meter_reading\"\ncat_names = categoricals\npath = DATA","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"vvBAQhB9C45P","colab":{}},"cell_type":"code","source":"#full_train_df[dep_var] = target","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","id":"p77G7Vv9C45Q","colab":{}},"cell_type":"code","source":"#del target","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"d108af6b-1a9d-4d62-c4d8-5bdf6cc39fe1","id":"OLPzVdrNC45R","colab":{"base_uri":"https://localhost:8080/","height":221},"trusted":false},"cell_type":"code","source":"full_train_df_sample[dep_var]","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","outputId":"f4270155-21e8-4c03-f953-79ee124bb8c2","id":"RmqLtTb1C45U","colab":{"base_uri":"https://localhost:8080/","height":54}},"cell_type":"code","source":"# Not including test set first since takes too much space\ndata = TabularDataBunch.from_df(path, full_train_df_sample, dep_var, valid_idx=valid_idx, procs=procs, cat_names=cat_names, bs=128) #, test_df=full_test_df)\nprint(data.train_ds.cont_names)  # `cont_names` defaults to: set(df)-set(cat_names)-{dep_var}","execution_count":0,"outputs":[]},{"metadata":{"id":"Uy2lgjYOEO8z","colab_type":"code","colab":{},"trusted":false},"cell_type":"code","source":"del full_train_df_sample","execution_count":0,"outputs":[]},{"metadata":{"trusted":true,"colab_type":"code","outputId":"b70bdf66-2a3f-4cf5-973d-1de81d508fe5","id":"8tQtegbqC45V","colab":{"base_uri":"https://localhost:8080/","height":204}},"cell_type":"code","source":"(cat_x,cont_x),y = next(iter(data.train_dl))\nfor o in (cat_x, cont_x, y): print(to_np(o[:5]))","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"iaFHtIVsC45W","colab":{},"trusted":false},"cell_type":"code","source":"learn = tabular_learner(data, layers=[200,100], emb_szs=cat_card, metrics=rmse)\n","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"256cb50f-b0f4-4d32-d4c3-f53014935ada","id":"h05DmcalC45Y","colab":{"base_uri":"https://localhost:8080/","height":80},"trusted":false},"cell_type":"code","source":"learn.fit_one_cycle(1, 1e-2)","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"bf02f240-f52f-467a-f547-1b8bbe7c6030","id":"tzuq6cCtC45Z","colab":{"base_uri":"https://localhost:8080/","height":102},"trusted":false},"cell_type":"code","source":"pred_batch1 = learn.pred_batch()\npred_batch1[: 5]","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"kKSoS_zFC45b","colab":{},"trusted":false},"cell_type":"code","source":"learn.export('model_all_export')","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"xsnZvsfbEh6r","colab":{},"trusted":false},"cell_type":"code","source":"# Checking if predictions will be similar after loading\n#learn1 = tabular_learner(data, layers=[200,100], emb_szs=cat_card, metrics=rmse)","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"TE9NAV-rC45e","colab":{},"trusted":false},"cell_type":"code","source":"#learn1.load('model_sample')","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"ngGqHHofC45f","colab":{},"trusted":false},"cell_type":"code","source":"#ls /content/ashrae-energy-prediction/","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"fCMEtBTiC45h","colab":{},"trusted":false},"cell_type":"code","source":"#pred_batch2 = learn1.pred_batch()\n#pred_batch2[: 5]","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"2TrW9rrcC45i","colab":{},"trusted":false},"cell_type":"code","source":"#learn2.data.train_ds","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"bRQmG8olC45k","colab":{},"trusted":false},"cell_type":"code","source":"#tabList_sample = TabularList.from_df(full_train_df.head(10), cat_names=cat_names, procs=procs)\n#learn2 = load_learner(DATA, 'model_sample_export', test=tabList_sample)","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"wCMvwk2MC45l","colab":{},"trusted":false},"cell_type":"code","source":"#pred_batch3 = learn2.pred_batch(ds_type=DatasetType.Test)\n#pred_batch3[: 5]","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"hGgvp23LC45m","colab":{},"trusted":false},"cell_type":"code","source":"full_test_df = pd.read_pickle('full_test_df.pkl')","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"5ec1bf64-ab21-49b4-b2c6-9f1ae6ae003e","id":"Gp0PnjO8C45n","colab":{"base_uri":"https://localhost:8080/","height":71},"trusted":false},"cell_type":"code","source":"nan_cols = full_test_df.columns.values[full_test_df.isna().sum().nonzero()[0]].tolist()","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"yqYaz-NrC45p","colab":{},"trusted":false},"cell_type":"code","source":"for col_ in nan_cols:\n    median = full_test_df[col_].median()\n    full_test_df[col_] = full_test_df[col_].fillna(median)\n","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"9oLe8c9HC45q","colab":{},"trusted":false},"cell_type":"code","source":"## Need to input data as tabList when learner is loaded\n## Note that the learner is loaded without the datasets\ntabList = TabularList.from_df(full_test_df, cat_names=cat_names, procs=procs)\nlearn2 = load_learner(DATA, 'model_all_export', test=tabList)","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"X53sodq0C45r","colab":{},"trusted":false},"cell_type":"code","source":"del tabList\ndel full_test_df","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"YgzoIthdC45s","outputId":"20b02f04-3aad-4e9b-f2f4-a2d511c00576","colab":{"base_uri":"https://localhost:8080/","height":307},"trusted":false},"cell_type":"code","source":"pred_test = learn2.get_preds(ds_type=DatasetType.Test)","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"a17f3812-19f1-4a98-a0c4-33a00bf63270","id":"MzPUohW_C45v","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"len(pred_test[0])#, full_test_df.shape","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"gB59NhU2C45w","colab":{},"trusted":false},"cell_type":"code","source":"pred_test_np = pred_test[0].numpy()","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"dcfa68c3-1731-4207-f526-7e7522891b95","id":"nNsUUeOZC45x","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"from google.colab import drive\ndrive.mount('/content/gdrive')","execution_count":0,"outputs":[]},{"metadata":{"trusted":false,"colab_type":"code","id":"qjtE2PD7C45y","colab":{}},"cell_type":"code","source":"res = pred_test_np","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","outputId":"35301006-5421-4568-fc12-b37befa96169","id":"IkAd_AJxC45z","colab":{"base_uri":"https://localhost:8080/","height":34},"trusted":false},"cell_type":"code","source":"submission = pd.read_csv(f'{DATA}sample_submission.csv')\nsubmission.shape","execution_count":0,"outputs":[]},{"metadata":{"trusted":false,"colab_type":"code","outputId":"af378b56-c9de-47a7-b72a-e2c8dd40160a","id":"KNnCFv8FC450","colab":{"base_uri":"https://localhost:8080/","height":419}},"cell_type":"code","source":"submission = pd.read_csv(f'{DATA}sample_submission.csv')\nsubmission['meter_reading'] = np.expm1(res)\nsubmission.loc[submission['meter_reading'] < 0, 'meter_reading'] = 0\nsubmission.to_csv('submission_fastai_full_20191103.csv', index=False)\nsubmission","execution_count":0,"outputs":[]},{"metadata":{"colab_type":"code","id":"1vlCNRVZC46C","colab":{},"trusted":false},"cell_type":"code","source":"","execution_count":0,"outputs":[]}],"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"colab":{"name":"ASHRAE_ EDA + FE + fastai.ipynb","provenance":[],"collapsed_sections":["Z-coGMWe3sEF"],"machine_shape":"hm"},"accelerator":"GPU"},"nbformat":4,"nbformat_minor":1}