{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Desperate attempt to log a commit."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys, warnings, math\nwarnings.filterwarnings('ignore')\nimport gc\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom datetime import datetime\n\nsns.set()\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(\"../input/ashrae-energy-prediction/train.csv\") as f:\n    head = [next(f) for x in range(10)]\nprint(head)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#https://hackersandslackers.com/downcast-numerical-columns-python-pandas/\nimport feather\n\nprint('train data:')\ntrain = pd.read_csv('../input/ashrae-energy-prediction/train.csv', \n                    dtype={'building_id':np.uint16, 'meter':np.uint8, 'meter_reading':np.float64})\ntrain['timestamp'] = pd.to_datetime(train['timestamp'], format=\"%Y %m %d %H:%M:%S\")\nprint(train.info(memory_usage='deep'))\ntrain.to_feather('train.feather')\n\nprint('-'*20);print('test data:')\ntest = pd.read_csv('../input/ashrae-energy-prediction/test.csv', \n                   dtype={'row_id':np.uint16,'building_id':np.uint16,'meter':np.uint16})\ntest['timestamp'] = pd.to_datetime(test['timestamp'], format=\"%Y %m %d %H:%M:%S\")\nprint(test.info(memory_usage='deep'))\ntest.to_feather('test.feather')\n\nprint('-'*20);print('weather_train data:')\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv',dtype={'site_id':np.uint16})\nweather_train['timestamp'] = pd.to_datetime(weather_train['timestamp'],infer_datetime_format=True)\nweather_train[['air_temperature', 'cloud_coverage',\n       'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']] = weather_train[['air_temperature', 'cloud_coverage',\n       'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']].apply(pd.to_numeric,downcast='float')\nprint(weather_train.info(memory_usage='deep'))\nweather_train.to_feather('weather_train.feather')\n\nprint('-'*20);print('weather_test data:')\nweather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv',dtype={'site_id':np.uint16})\nweather_test['timestamp'] = pd.to_datetime(weather_test['timestamp'],infer_datetime_format=True)\nweather_test[['air_temperature', 'cloud_coverage',\n       'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']] = weather_test[['air_temperature', 'cloud_coverage',\n       'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']].apply(pd.to_numeric,downcast='float')\nprint(weather_test.info(memory_usage='deep'))\nweather_test.to_feather('weather_test.feather')\n\nprint('-'*20);print('building_metadata data:')\nbuilding_metadata = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nbuilding_metadata['primary_use'] = building_metadata['primary_use'].astype('category')\nprint(building_metadata.info(memory_usage='deep'))\nbuilding_metadata.to_feather('building_metadata.feather')\n\nsample_submission = pd.read_csv('../input/ashrae-energy-prediction/sample_submission.csv')\nsample_submission.to_feather('sample_submission.feather')\n\ndel train, test, weather_train, weather_test, building_metadata, sample_submission\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Interpolating weather data to fill in null values ... "},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = pd.read_feather('weather_train.feather')\nweather_train = weather_train.groupby('site_id') \\\n                             .apply(lambda group: group.interpolate(limit_direction='both'))\n#weather_train = weather_train.set_index('timestamp').interpolate(method='time') \n\nprint(weather_train.isnull().sum())\n#weather_train.groupby('site_id').apply(lambda group: group.isna().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_feather('train.feather')\n#weather_train = pd.read_feather('weather_train.feather')\n#train['meter_reading'] = np.log1p(train['meter_reading'])\nbuilding_metadata = pd.read_feather('building_metadata.feather')\ntrain = train.merge(building_metadata, on='building_id', how='left')\ntrain = train.merge(weather_train, on=['site_id', 'timestamp'], how='left')\ndel weather_train, building_metadata\ngc.collect()\n\nprint(train.isnull().sum())\ntrain.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#taken from here:https://www.kaggle.com/kaushal2896/ashrae-eda-fe-lightgbm-1-12\ndef mean_without_overflow_fast(col):\n    col /= len(col)\n    return col.mean() * len(col)\n\nmissing_values = (100-train.count() / len(train) * 100).sort_values(ascending=False)\nmissing_features = train.loc[:, missing_values > 0.0]\nmissing_features = missing_features.apply(mean_without_overflow_fast)\n\nfor key in train.loc[:, missing_values > 0.0].keys():\n    if key == 'year_built' or key == 'floor_count':\n        train[key].fillna(math.floor(missing_features[key]), inplace=True)\n    else:\n        train[key].fillna(missing_features[key], inplace=True)\n\ntrain.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Frequency of primary_use\ntrain.groupby(['primary_use']).agg({'site_id':'nunique'}).rename(columns={'site_id':'N'}) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['square_feet'].hist(bins=32) #is this is wrong?\nplt.xlabel(\"square_feet\")\nplt.ylabel(\"Frequency\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Most buildings appear to be have floor area below 100,000 sq.ft. "},{"metadata":{"trusted":true},"cell_type":"code","source":"get_mean = train.groupby(['primary_use']).agg({'meter_reading':'mean'}).rename(columns={'meter_reading':'mr_mean'}) \nget_mean.sort_values(by='mr_mean')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So Education and Services appear to be consuming major part of the energy. Is the data really sampled every hour? Are there any gaps? Checking ..."},{"metadata":{"trusted":true},"cell_type":"code","source":"da = (train['timestamp'].iloc[-1] - train['timestamp'].iloc[0])\nprint(\"Number of hours between start and end dates: \", da.total_seconds()/3600 + 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The year 2016 was a leap year, so 24(h) x 366(d) = 8784. So our day periodicity is 24. OK, now extracting dates that have all measurements of the day."},{"metadata":{"trusted":true},"cell_type":"code","source":"count_full = train.groupby('building_id')['timestamp'].nunique()\n#Remember count_full is a Series object\ncount_full = count_full[count_full==count_full.max()]\n#ids with whole length\nprint(count_full.index)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Getting the part of the train data with full set of timestamps."},{"metadata":{"trusted":true},"cell_type":"code","source":"trfull = train[train['building_id'].isin(count_full.index)]\ntrfull.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Guess I can get the train data out of the way to save memory space ..."},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_date = trfull[trfull['building_id']==0].groupby(trfull['timestamp'].dt.floor('d')).count()\nnum_date.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_date['timestamp'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So each date has a complete set of measurements - or so it seems. Let's check the number of meter types at each site."},{"metadata":{"trusted":true},"cell_type":"code","source":"trfull.groupby('site_id').apply(lambda x: x['meter'].nunique())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Not every type of meter is present at each site. What about the number of buildings at each site ..."},{"metadata":{"trusted":true},"cell_type":"code","source":"trfull.groupby('site_id').apply(lambda x: x['building_id'].nunique())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Checking a particular site and meter..."},{"metadata":{"trusted":true},"cell_type":"code","source":"#sns.set(rc={\"lines.linewidth\": 0.5})\ntrfull[((trfull['site_id']==3) & (trfull['meter']==0))] \\\n             .plot(x='timestamp',y='meter_reading',figsize=(12,6))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Aggregating consumption at a particular site..."},{"metadata":{"trusted":true},"cell_type":"code","source":"site14 = trfull[trfull['site_id']==14]\nfig, axes = plt.subplots(4,1,figsize=(14, 18))\nfor i in range(4):\n    site14[site14['meter']==i][['timestamp', 'meter_reading']].set_index('timestamp') \\\n            .resample('H').sum()['meter_reading'] \\\n            .plot(ax=axes[i], alpha=0.8, label='By hour') \\\n            .set_ylabel('Summation meter reading')\n    axes[i].legend();\n    axes[i].set_title('Meter: ' + str(i));\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now aggregating for all sites ..."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(4,1,figsize=(14, 18))\nfor i in range(4):\n    trfull[trfull['meter']==i][['timestamp', 'meter_reading']].set_index('timestamp') \\\n            .resample('H').sum()['meter_reading'] \\\n            .plot(ax=axes[i], alpha=0.8, label='By hour') \\\n            .set_ylabel('Summation meter reading')\n    axes[i].legend();\n    axes[i].set_title('Meter: ' + str(i));\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Oops! there is something wrong with meter type 3 aggregation. Fortunately [this notebook](https://www.kaggle.com/nroman/eda-for-ashrae) has already identified the culprit - it's building_id==1099 and meter==2. Thanks @Roman :). Let's get it out of the way."},{"metadata":{"trusted":true},"cell_type":"code","source":"trfull = trfull[~((trfull['meter'] == 2) & (trfull['building_id'] == 1099))]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Also as reported [here](https://www.kaggle.com/c/ashrae-energy-prediction/discussion/113054#656588) all electricity meter readings are 0 until May 20 for site_id == 0. So removing those as well."},{"metadata":{"trusted":true},"cell_type":"code","source":"trfull = trfull.query('not (building_id <= 104 & meter == 0 & timestamp <= \"2016-05-20\")')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Aggregating for all sites again ..."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(4,1,figsize=(14, 18))\nfor i in range(4):\n    trfull[trfull['meter']==i].groupby('timestamp')['meter_reading'].sum().plot(ax=axes[i])\n    axes[i].set_title('Meter: ' + str(i))\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\n\ndef getFrames(dftrn, dftst, idx):\n    #train part\n    dftrn = dftrn[dftrn['meter']==idx]\n    dftrn.set_index('timestamp',inplace=True)\n    dftrn['primary_use'] = le.fit_transform(dftrn['primary_use'])\n    cols = list(dftrn.columns)\n    cols.remove('meter_reading')\n    #test part\n    dftst = dftst[dftst['meter']==idx]\n    dftst['index1'] = dftst.index\n    dftst.set_index('timestamp',inplace=True)\n    dftst['primary_use'] = le.fit_transform(dftst['primary_use'])\n    return dftrn[cols], dftrn['meter_reading'], dftst[cols], dftst['index1']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#error metrics\nfrom sklearn.metrics import make_scorer\nfrom sklearn.metrics import mean_squared_error\n\ndef rmse(y_test, y_pred):\n    return np.sqrt(mean_squared_error(y_test, y_pred))\nmy_scorer = make_scorer(rmse,greater_is_better=False)\n\n#from sklearn.multioutput import MultiOutputRegressor\nfrom sklearn.linear_model import ElasticNet\nfrom sklearn.ensemble import RandomForestRegressor\n\n#model = ElasticNet(\n#        alpha=1.0,\n#        l1_ratio=0.3,\n#        fit_intercept=True,\n#        normalize=False,\n#        precompute=False,\n#        max_iter=16,\n#        copy_X=True,\n#        tol=0.1,\n#        warm_start=False,\n#        positive=False,\n#        random_state=None,\n#        selection='random'\n#    )\nmodel = RandomForestRegressor(random_state = 1, n_jobs = -1)\n\nfrom sklearn.model_selection import TimeSeriesSplit, cross_val_score\ntscv = TimeSeriesSplit(n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = pd.read_feather('weather_test.feather')\ncols = list(weather_test.columns)\nweather_test = weather_test.groupby('site_id') \\\n                           .apply(lambda group: group.interpolate(limit_direction='both'))\n#weather_test = weather_test.set_index('timestamp').interpolate(method='time') \nweather_test.groupby('site_id').apply(lambda group: group.isna().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(weather_test.isnull().sum())\nweather_test.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_feather('test.feather')\nbuilding_metadata = pd.read_feather('building_metadata.feather')\ntest = test.merge(building_metadata, on='building_id', how='left')\n#weather_test = pd.read_feather('weather_test.feather')\ntest = test.merge(weather_test, on=['site_id', 'timestamp'], how='left')\ndel weather_test, building_metadata\ngc.collect()\n\nprint(test.isnull().sum())\ntest.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_values = (100-test.count() / len(test) * 100).sort_values(ascending=False)\nmissing_features = test.loc[:, missing_values > 0.0]\nmissing_features = missing_features.apply(mean_without_overflow_fast)\n\nfor key in test.loc[:, missing_values > 0.0].keys():\n    if key == 'year_built' or key == 'floor_count':\n        test[key].fillna(math.floor(missing_features[key]), inplace=True)\n    else:\n        test[key].fillna(missing_features[key], inplace=True)\n        \ntest.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(4):\n    X, y, _, _ = getFrames(trfull, test, i)\n    scores = cross_val_score(model, X, y, cv=tscv, scoring=my_scorer)\n    print(\"Meter-{0:d} Loss: {1:.3f} (+/- {2:.3f})\".format(i, scores.mean(), scores.std()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\n\n#params = {\n#    'alpha':(0.1, 0.3, 0.5, 0.7, 0.9),\n#    'l1_ratio':(0.1, 0.3, 0.5, 0.7, 0.9) \n#}\nparams = {\n    'max_features' : [\"auto\", \"sqrt\", \"log2\"],\n    'min_samples_split' : np.linspace(0.1, 1.0, 10)\n}\npredict_list = []\n\nfor i in range(4):\n    X, y, tst, ref = getFrames(trfull, test, i)\n    print('-'*20);print(\"Meter-{0:d}\".format(i))\n    gs = GridSearchCV(model, param_grid=params, cv=tscv, scoring=my_scorer, verbose=1)\n    gs.fit(X,y)\n    yp = gs.predict(tst)\n    predict_list.append(np.vstack((ref.values,yp)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p = np.hstack(predict_list)\np = p.T\np = p[p[:,0].argsort()]\n#print(p)\n#print(p.shape)\n\nsub = pd.DataFrame(p, columns=['row_id','meter_reading'])\nsub.loc[sub['meter_reading']<0, 'meter_reading'] = 0\nsub['row_id'] = sub['row_id'].astype(int)\nsub.tail()\n#from collections import Counter\n#print([item for item, count in Counter(p).items() if count > 1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv',index=False, float_format='%.4f')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}