{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport statsmodels.api as sm\nimport statsmodels.formula.api as smf\nimport os\nimport math\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nunimportant_cols = ['wind_direction', 'wind_speed', 'sea_level_pressure']\ntarget = 'meter_reading'\n\ndef load_data(source='train', path='/kaggle/input/ashrae-energy-prediction'):\n    ''' load and merge all tables '''\n    assert source in ['train', 'test']\n\n    building = pd.read_csv(f'{path}/building_metadata.csv', dtype={'building_id':np.uint16, 'site_id':np.uint8})\n    weather  = pd.read_csv(f'{path}/weather_{source}.csv', parse_dates=['timestamp'],\n                                                           dtype={'site_id':np.uint8, 'air_temperature':np.float16,\n                                                                  'cloud_coverage':np.float16, 'dew_temperature':np.float16,\n                                                                  'precip_depth_1_hr':np.float16},\n                                                           usecols=lambda c: c not in unimportant_cols)\n    df = pd.read_csv(f'{path}/{source}.csv', dtype={'building_id':np.uint16, 'meter':np.uint8}, parse_dates=['timestamp'])\n    \n    df = df.merge(building, on='building_id', how='left')\n    df = df.merge(weather, on=['site_id', 'timestamp'], how='left')\n    return df\n\n# load and display some samples\ntrain = load_data('train')\ntrain['hour'] = np.uint8(train['timestamp'].dt.hour)\ntrain['weekday'] = np.uint8(train['timestamp'].dt.weekday)\ntrain['log_square_feet'] = np.float16(np.log(train['square_feet']))\ntrain['log_meter_reading'] = np.float16(np.log(train['meter_reading']))\nsubset = train.sample(n = 2000000, replace = False)\ntrain = subset.iloc[:-200000]\ntrain = train.sort_values(by=['timestamp'])\ntrain = train.reset_index()\ntrain = train.drop(columns = 'index')\ntrain.head(7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train.log_meter_reading < 0] = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_ratios = train.count()/len(train)\ntrain.loc[:, data_ratios < 1.0].mean()\ntrain = train.fillna(train.loc[:, data_ratios < 1.0].mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"\nformula = 'log_meter_reading ~ log_square_feet + building_id + meter + air_temperature + dew_temperature + floor_count + primary_use - 1 + year_built + hour + site_id + weekday + cloud_coverage + precip_depth_1_hr'\nmodel = smf.glm(formula = formula, data=train, family=sm.families.Gaussian(),missing = 'drop')\nresult = model.fit()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = subset.iloc[-200000:]\ntest = test.sort_values(by=['timestamp'])\ntest = test.reset_index()\ntest = test.drop(columns = 'index')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_ratios = test.count()/len(test)\ntest.loc[:, data_ratios < 1.0].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test.fillna(test.loc[:, data_ratios < 1.0].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = result.predict(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from math import exp\nfrom math import log\nfrom math import sqrt\ndef RMSLE(a, b):\n    assert(len(a) == len(b))\n    sum = 0\n    d = 0\n    for i in range(len(a)):\n        sum += ((log(exp(a[i]) + 1) - log(exp(b[i]) + 1)) ** 2)\n        \n    return sqrt((1 / (len(a))) * (sum))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib.pyplot import hist\n\ntest.loc[test.log_meter_reading < 0] = 0\npreds.loc[preds < 0] = 0\nhist(test.log_meter_reading)\nhist(preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nRMSLE(preds,test.log_meter_reading)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}