{"cells":[{"metadata":{"_uuid":"792267de-11b3-4ae0-939c-3b5641bf53dc","_cell_guid":"b0a6b03a-eb6a-4de4-809e-12d16c429c31","trusted":true},"cell_type":"markdown","source":"> Note: The below script works, however I messed up loading data from one  of the previous kernel versions, hence cannot execut it anymore.\nPLease uncomment the below script or follow the fork https://www.kaggle.com/poltigo/ashrae-fastai-solving-the-memory-issue"},{"metadata":{"_uuid":"9feec0d9-09c2-4bc2-ae70-beb3310f253a","_cell_guid":"375dd28c-d291-42f3-bf20-3fb32faf16db","trusted":true},"cell_type":"code","source":"# import numpy as np \n# import pandas as pd \n# import warnings\n# warnings.filterwarnings('ignore') # Suppress warnings \n# import gc\n\n# import matplotlib.pyplot as plt\n# %matplotlib inline\n# import seaborn as sns\n\n# pd.set_option('max_columns', 100)\n# pd.set_option('display.float_format', '{:.2f}'.format)\n\n# import os,random, math, psutil, pickle\n\n# from sklearn.metrics import mean_squared_error\n# from tqdm import tqdm\n\n# from fastai.tabular import *\n# import torch\n# print(torch.cuda.is_available())\n\n# # %% [code]\n# root = '../input/ashrae-energy-prediction/'\n# train_df = pd.read_csv(root + 'train.csv')\n# train_df[\"timestamp\"] = pd.to_datetime(train_df[\"timestamp\"], format='%Y-%m-%d %H:%M:%S')\n\n# weather_train_df = pd.read_csv(root + 'weather_train.csv')\n# test_df = pd.read_csv(root + 'test.csv')\n# weather_test_df = pd.read_csv(root + 'weather_test.csv')\n# building_meta_df = pd.read_csv(root + 'building_metadata.csv')\n# sample_submission = pd.read_csv(root + 'sample_submission.csv')\n\n# # %% [code]\n# print(train_df.shape)\n# print(test_df.shape)\n\n# # %% [markdown]\n# # **Important: fastai does not work with float16 format unless explicitly specified; thus the below function was modified to exclude float16**\n\n# # %% [code]\n# ## Function to reduce the DF size\n# def reduce_mem_usage(df, verbose=True):\n#     numerics = ['int8', 'int16', 'int32', 'int64', 'float32', 'float64']\n#     start_mem = df.memory_usage().sum() / 1024**2    \n#     for col in df.columns:\n#         col_type = df[col].dtypes\n#         if col_type in numerics:\n#             c_min = df[col].min()\n#             c_max = df[col].max()\n#             if str(col_type)[:3] == 'int':\n#                 if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n#                     df[col] = df[col].astype(np.int8)\n#                 elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n#                     df[col] = df[col].astype(np.int16)\n#                 elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n#                     df[col] = df[col].astype(np.int32)\n#                 elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n#                     df[col] = df[col].astype(np.int64)  \n#             else:\n#                 if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n#                     df[col] = df[col].astype(np.float32)\n#                 else:\n#                     df[col] = df[col].astype(np.float64)    \n#     end_mem = df.memory_usage().sum() / 1024**2\n#     if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n#     return df\n\n# # %% [code]\n# ## REducing memory\n# train_df = reduce_mem_usage(train_df)\n# test_df = reduce_mem_usage(test_df)\n\n# weather_train_df = reduce_mem_usage(weather_train_df)\n# weather_test_df = reduce_mem_usage(weather_test_df)\n# building_meta_df = reduce_mem_usage(building_meta_df)\n\n# # %% [code]\n# # creating the index of test_df to be used for inference\n# range_test = np.array_split(test_df.index, 5)\n# print(range_test)\n\n# # %% [code]\n# train_df['timestamp'] = pd.to_datetime(train_df['timestamp'])\n# test_df['timestamp'] = pd.to_datetime(test_df['timestamp'])\n\n# weather_train_df['timestamp'] = pd.to_datetime(weather_train_df['timestamp'])\n# weather_test_df['timestamp'] = pd.to_datetime(weather_test_df['timestamp'])\n\n# building_meta_df['primary_use'] = building_meta_df['primary_use'].astype('category')\n\n# temp_df = train_df[['building_id']]\n# temp_df = temp_df.merge(building_meta_df, on=['building_id'], how='left')\n# del temp_df['building_id']\n# train_df = pd.concat([train_df, temp_df], axis=1)\n\n# temp_df = test_df[['building_id']]\n# temp_df = temp_df.merge(building_meta_df, on=['building_id'], how='left')\n\n# del temp_df['building_id']\n# test_df = pd.concat([test_df, temp_df], axis=1)\n# del temp_df, building_meta_df\n\n# temp_df = train_df[['site_id','timestamp']]\n# temp_df = temp_df.merge(weather_train_df, on=['site_id','timestamp'], how='left')\n\n# del temp_df['site_id'], temp_df['timestamp']\n# train_df = pd.concat([train_df, temp_df], axis=1)\n\n# temp_df = test_df[['site_id','timestamp']]\n# temp_df = temp_df.merge(weather_test_df, on=['site_id','timestamp'], how='left')\n\n# del temp_df['site_id'], temp_df['timestamp']\n# test_df = pd.concat([test_df, temp_df], axis=1)\n\n# del temp_df, weather_train_df, weather_test_df\n\n# gc.collect()\n\n# # %% [code]\n# # Adding small value to meeter reading to avoid log(0) error\n# train_df.loc[train_df.meter_reading == 0, ['meter_reading']] = train_df.meter_reading + 0.000001\n\n# # %% [markdown]\n# # **Converting variables to log values because of their uneven distribution**\n\n# # %% [code]\n# train_df.meter_reading = np.log1p(train_df[\"meter_reading\"])\n# train_df.square_feet = np.log1p(train_df[\"square_feet\"])\n# test_df.square_feet = np.log1p(test_df[\"square_feet\"])\n# train_df['square_feet'] = train_df['square_feet'].astype('float32')\n# test_df['square_feet'] = test_df['square_feet'].astype('float32')\n\n# # %% [code]\n# # sns.distplot((train_df.meter_reading))\n# # sns.distplot(train_df.square_feet)\n\n# # %% [markdown]\n# # **Handling missing values**\n\n# # %% [code]\n# # # Missing items\n# # (train_df.isnull().sum()/ len(train_df) *100).sort_values(ascending = False).head(10)\n\n# # %% [code]\n# # # Missing items\n# # (test_df.isnull().sum()/ len(test_df) *100).sort_values(ascending = False).head(10)\n\n# # %% [code]\n# def average_imputation(df, column_name):\n#     imputation = df.groupby(['timestamp'])[column_name].mean()\n    \n#     df.loc[df[column_name].isnull(), column_name] = df[df[column_name].isnull()][[column_name]].apply(lambda x: imputation[df['timestamp'][x.index]].values)\n#     del imputation\n#     return df\n\n# # %% [code]\n# train_df = average_imputation(train_df, 'wind_speed')\n# #train_df = average_imputation(train_df, 'wind_direction')\n# test_df = average_imputation(test_df, 'wind_speed')\n# #test_df = average_imputation(test_df, 'wind_direction')\n\n# # %% [code]\n# beaufort = [(0, 0, 0.3), (1, 0.3, 1.6), (2, 1.6, 3.4), (3, 3.4, 5.5), (4, 5.5, 8), (5, 8, 10.8), (6, 10.8, 13.9), \n#           (7, 13.9, 17.2), (8, 17.2, 20.8), (9, 20.8, 24.5), (10, 24.5, 28.5), (11, 28.5, 33), (12, 33, 200)]\n\n# # %% [code]\n# for item in beaufort:\n#     train_df.loc[(train_df['wind_speed']>=item[1]) & (train_df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n# for item in beaufort:\n#     test_df.loc[(test_df['wind_speed']>=item[1]) & (test_df['wind_speed']<item[2]), 'beaufort_scale'] = item[0]\n\n# # %% [code]\n# # def degToCompass(num):\n# #     val=int((num/22.5)+.5)\n# #     arr=[i for i in range(0,16)]\n# #     return arr[(val % 16)]\n\n# # %% [code]\n# #train_df['wind_direction'] = train_df['wind_direction'].apply(degToCompass)\n# train_df['beaufort_scale'] = train_df['beaufort_scale'].astype('int8')\n# #train_df[\"wind_direction\"] = train_df['wind_direction'].astype(np.uint8)\n\n# # %% [code]\n# #test_df['wind_direction'] = test_df['wind_direction'].apply(degToCompass)\n# test_df['beaufort_scale'] = test_df['beaufort_scale'].astype('int8')\n# #test_df[\"wind_direction\"] = test_df['wind_direction'].astype(np.uint8)\n\n# # %% [code]\n# def fill_miss(df):\n#     miss_col = ['floor_count','year_built','cloud_coverage']\n#     for col in miss_col:\n#         df[col].fillna(-999, inplace = True)\n\n# # %% [code]\n# fill_miss(train_df)\n# fill_miss(test_df)\n\n# # %% [code]\n# def impute_miss(df):\n#     miss_col = ['precip_depth_1_hr', 'sea_level_pressure',  'dew_temperature', 'air_temperature']\n#     for col in miss_col:\n#         df[col].fillna(df[col].mean(), inplace = True)\n\n# # %% [code]\n# impute_miss(train_df)\n# impute_miss(test_df)\n\n# # %% [markdown]\n# # convert columns with previously missing values to more efficient formats\n\n# # %% [code]\n# convert = ['year_built', 'floor_count', 'cloud_coverage', 'precip_depth_1_hr']\n# def type_convert(df, columns = convert):\n#     for col in columns:\n#         df[col] = df[col].astype('int8')\n\n# # %% [code]\n# type_convert(train_df)\n# type_convert(test_df)\n\n# # %% [code]\n# # train_df['month_datetime'] = train_df['timestamp'].dt.month.astype(np.int8)\n# # train_df['weekofyear_datetime'] = train_df['timestamp'].dt.weekofyear.astype(np.int8)\n# # train_df['dayofyear_datetime'] = train_df['timestamp'].dt.dayofyear.astype(np.int16)\n# train_df['hour_datetime'] = train_df['timestamp'].dt.hour.astype('int8')  \n# train_df['day_week'] = train_df['timestamp'].dt.dayofweek.astype('int8')\n# # train_df['day_month_datetime'] = train_df['timestamp'].dt.day.astype(np.int8)\n# # train_df['week_month_datetime'] = train_df['timestamp'].dt.day/7\n# # train_df['week_month_datetime'] = train_df['week_month_datetime'].apply(lambda x: math.ceil(x)).astype(np.int8)\n    \n# # test_df['month_datetime'] = test_df['timestamp'].dt.month.astype(np.int8)\n# # test_df['weekofyear_datetime'] = test_df['timestamp'].dt.weekofyear.astype(np.int8)\n# # test_df['dayofyear_datetime'] = test_df['timestamp'].dt.dayofyear.astype(np.int16)\n# test_df['hour_datetime'] = test_df['timestamp'].dt.hour.astype('int8')\n# test_df['day_week'] = test_df['timestamp'].dt.dayofweek.astype('int8')\n# # test_df['day_month_datetime'] = test_df['timestamp'].dt.day.astype(np.int8)\n# # test_df['week_month_datetime'] = test_df['timestamp'].dt.day/7\n# # test_df['week_month_datetime'] = test_df['week_month_datetime'].apply(lambda x: math.ceil(x)).astype(np.int8)\n\n# # %% [code]\n# train_df.drop(columns = ['timestamp', 'wind_speed'], inplace = True)\n# test_df.drop(columns = ['timestamp', 'wind_speed'], inplace = True)\n\n# # %% [code]\n# test_cols = [\"row_id\",\"site_id\", \"building_id\", \"primary_use\", \"hour_datetime\", \"day_week\",  \"meter\", \n#              \"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\n#               \"dew_temperature\", \"precip_depth_1_hr\", \"floor_count\", 'beaufort_scale']\n\n# # %% [code]\n# test_df = test_df[test_cols]\n\n# # %% [code]\n# train_cols = [\"site_id\", \"building_id\", \"primary_use\", \"hour_datetime\", \"day_week\",  \"meter\", \n#              \"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\n#               \"dew_temperature\", \"precip_depth_1_hr\", \"floor_count\", 'beaufort_scale', \"meter_reading\"]\n\n# # %% [code]\n# train_df = train_df[train_cols]\n\n# # %% [code]\n# train_df.to_pickle('train_df.pkl')\n# test_df.to_pickle('test_df.pkl')\n# del train_df, test_df\n# gc.collect()\n\n# # %% [markdown]\n# # **Stage 2 - Load pickled data and train the model**\n\n# # %% [code]\n# train_df = pd.read_pickle('train_df.pkl')\n# #test_df = pd.read_pickle('test_df.pkl')\n# #train_df = train_df[0:100000]\n# #test_df = test_df[0:1500000]\n# gc.collect()\n\n# # %% [code]\n# # #cat_names = list(train_df.select_dtypes(include = ['category', 'bool']).columns)\n# # cat_names = ['primary_use', 'meter', 'building_id', 'site_id', \n# #              'month_datetime', 'weekofyear_datetime','dayofyear_datetime',\n# #              'hour_datetime','day_week',  'day_month_datetime',\n# #              'week_month_datetime', 'floor_count','year_built', 'beaufort_scale']\n# cat_names = [\"site_id\", \"building_id\", \"primary_use\", \"hour_datetime\", \"day_week\",  \"meter\"]\n# print(cat_names)\n\n# # %% [code]\n# # #cont_names = list(train_df.select_dtypes(exclude = ['category', 'bool', 'datetime64[ns]']).columns)\n# # #cont_names.remove('meter_reading')\n# # cont_names = ['square_feet', 'air_temperature',  'dew_temperature', 'cloud_coverage',\n# #               'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction' ]\n\n\n# cont_names = [\"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\n#               \"dew_temperature\", \"precip_depth_1_hr\", \"floor_count\", 'beaufort_scale']\n\n# print(cont_names)\n\n# # %% [code]\n# #Path / default location for saving/loading models\n# path = ''\n\n# #The dependent variable/target\n# dep_var = 'meter_reading'\n\n# # %% [code]\n# procs = [FillMissing, Categorify, Normalize]\n\n# # %% [code]\n# # #Start index for creating a validation set from train_data\n# # start_indx = len(train_df) - int(len(train_df) * 0.2)\n\n# # #End index for creating a validation set from train_data\n# # end_indx = len(train_df)\n\n# # valid_idx = range(start_indx, end_indx)\n\n# # %% [code]\n# data = (TabularList.from_df(train_df, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs)\n#                 #.split_by_idx(valid_idx)\n#                 .split_by_rand_pct(valid_pct = 0.2)\n#                 .label_from_df(cols=dep_var, label_cls=FloatList, log = False)\n#                 #.add_test(TabularList.from_df(test_df, path=path, cat_names=cat_names, cont_names=cont_names))\n#                 #.add_test(test)\n#                 .databunch())\n\n# # %% [code]\n# #max_log_y = np.log(np.max(train_df['meter_reading'])*1.2)\n# #y_range = torch.tensor([0, max_log_y])\n# max_y = (np.max(train_df['meter_reading'])*1.2)\n# y_range = torch.tensor([0, max_y])\n\n# # %% [code]\n# del train_df\n# gc.collect()\n\n# # %% [code]\n# data.show_batch(rows=5)\n\n# # %% [code]\n# data.show_batch(rows=5, ds_type=DatasetType.Valid)\n\n# # %% [code]\n# #data.show_batch(rows=5, ds_type=DatasetType.Test)\n\n# # %% [code]\n# learn = tabular_learner(data, layers=[800,400], ps=[0.001,0.01], emb_drop=0.04, y_range = y_range, emb_szs={'building_id': 50}, metrics= rmse)\n# #metrics= exp_rmspe\n\n# # %% [code]\n# learn.model\n\n# # %% [code]\n# learn.lr_find()\n\n# # %% [code]\n# learn.recorder.plot()\n\n# # %% [code]\n# gc.collect()\n\n# # %% [code]\n# learn.fit_one_cycle(1, max_lr = 3e-1, wd = 0.2)\n\n# # %% [code]\n# learn.recorder.plot_losses()\n\n# # %% [code]\n# learn.export()\n\n# # %% [code]\n# learn.destroy()\n\n# # %% [markdown]\n# # **Stage 3 - Batch inference from the test dataset**\n\n# # %% [code]\n# output = []\n# for r in range(len(range_test)):\n#     test_df = pd.read_pickle('test_df.pkl')\n#     test_df = test_df.ix[range_test[r]]\n#     test = TabularList.from_df(test_df, path=path, cat_names=cat_names, cont_names=cont_names)\n#     del test_df\n#     gc.collect()\n#     # beginning of inference\n#     learn = load_learner(\".\", test= test)\n#     preds,_ = learn.get_preds(ds_type=DatasetType.Test)\n#     preds = np.expm1(preds.numpy())\n#     preds = pd.DataFrame(preds)\n#     output.append(preds)   \n\n# # %% [code]\n# meter_reading = pd.concat(output, ignore_index = True)\n\n# # %% [code]\n# meter_reading.reset_index(inplace = True)\n\n# # %% [code]\n# meter_reading.columns = ['row_id', 'meter_reading']\n\n# # %% [code]\n# meter_reading.head()\n\n# # %% [code]\n# meter_reading.to_csv('submission.csv', index=False)\n\n# # %% [code]\n# # submission = pd.DataFrame()\n# # submission['row_id'] = row_id\n# # submission['meter_reading'] = output\n# # submission.head()\n# # submission.to_csv('submission.csv', index=False)","execution_count":0,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}