{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os, gc, warnings\nimport random\nimport datetime\n\nfrom tqdm.notebook import tqdm\n# matplotlib and seaborn for plotting\nimport matplotlib\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# import plotly.offline as py\n# py.init_notebook_mode(connected=True)\n# from plotly.offline import init_notebook_mode, iplot\n# init_notebook_mode(connected=True)\n# import plotly.graph_objs as go\n# import plotly.offline as offline\n# offline.init_notebook_mode()\n\nfrom pandas.plotting import register_matplotlib_converters\nregister_matplotlib_converters()\n\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport sklearn\n\nimport lightgbm as lgb\n\nimport pickle\n\nwarnings.filterwarnings('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-10T00:35:49.966533Z","iopub.execute_input":"2022-01-10T00:35:49.966899Z","iopub.status.idle":"2022-01-10T00:35:49.975103Z","shell.execute_reply.started":"2022-01-10T00:35:49.966864Z","shell.execute_reply":"2022-01-10T00:35:49.973957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/ashrae-energy-prediction'\n# Input data files are available in the \"../input/\" directory.\nfor dirname, _, filenames in os.walk(path):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2022-01-10T00:35:49.980407Z","iopub.execute_input":"2022-01-10T00:35:49.980766Z","iopub.status.idle":"2022-01-10T00:35:49.993025Z","shell.execute_reply.started":"2022-01-10T00:35:49.980731Z","shell.execute_reply":"2022-01-10T00:35:49.992079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# unimportant features (see importance below)\nunimportant_cols = ['wind_direction', 'wind_speed', 'sea_level_pressure']\ntarget = 'meter_reading'\n\ndef load_data(source='train', path=path):\n    ''' load and merge all tables '''\n    assert source in ['train', 'test']\n    \n    building = pd.read_csv(f'{path}/building_metadata.csv', dtype={'building_id':np.uint16, 'site_id':np.uint8})\n    weather  = pd.read_csv(f'{path}/weather_{source}.csv', parse_dates=['timestamp'],\n                                                           dtype={'site_id':np.uint8, 'air_temperature':np.float16,\n                                                                  'cloud_coverage':np.float16, 'dew_temperature':np.float16,\n                                                                  'precip_depth_1_hr':np.float16},\n                                                           usecols=lambda c: c not in unimportant_cols)\n    df = pd.read_csv(f'{path}/{source}.csv', dtype={'building_id':np.uint16, 'meter':np.uint8}, parse_dates=['timestamp'])\n    df = df.merge(building, on='building_id', how='left')\n    df = df.merge(weather, on=['site_id', 'timestamp'], how='left')\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-10T00:35:49.998567Z","iopub.execute_input":"2022-01-10T00:35:49.999056Z","iopub.status.idle":"2022-01-10T00:35:50.009002Z","shell.execute_reply.started":"2022-01-10T00:35:49.998942Z","shell.execute_reply":"2022-01-10T00:35:50.007960Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain = load_data('train')\ntrain.head(7)","metadata":{"execution":{"iopub.status.busy":"2022-01-10T00:35:50.011665Z","iopub.execute_input":"2022-01-10T00:35:50.012812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest = load_data('test')\ntest.sample(7)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#  FULL EDA","metadata":{}},{"cell_type":"markdown","source":"**Data Wrangling**","metadata":{}},{"cell_type":"code","source":"buildings = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nbuildings.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"NULL COUNT:\" )\nprint(buildings.isnull().sum())\nprint(\"DUPLICATES COUNT:\" )\nprint(buildings.duplicated().sum())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n  \nfig = px.box(buildings, x = \"primary_use\", y=\"square_feet\")\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Meter Readings Visual Train","metadata":{}},{"cell_type":"code","source":"meter_arr = train[\"meter\"].unique()\nfor meter in meter_arr:\n    mask = train[\"meter\"] == meter\n    plt.figure(figsize=(20, 5))\n    sns.scatterplot(data = train[mask], x = \"meter_reading\", y = \"air_temperature\")\n    plt.xlabel(\"Meter: {}\".format(meter))\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Removing Outliers**","metadata":{}},{"cell_type":"code","source":"mask1 = train[\"meter\"] == 0\nmask2 = train[\"meter_reading\"] > 40000\nmask = np.logical_and(mask1, mask2)\nprint(train.shape)\ntrain[mask][\"meter_reading\"] = np.mean(train[mask1][\"meter_reading\"])\nprint(train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mask1 = train[\"meter\"] == 3\nmask2 = train[\"meter_reading\"] > 140000\nmask = np.logical_and(mask1, mask2)\nprint(train.shape)\ntrain[mask][\"meter_reading\"] = np.mean(train[mask1][\"meter_reading\"])\nprint(train.shape)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Train meter 1 and 2 have no visual outliers","metadata":{}},{"cell_type":"code","source":"train.corr().meter","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**exploring test file**","metadata":{}},{"cell_type":"code","source":"#The goal is to predict the meter reading column accurately\n# not much exploartion is needed in the test file \n# exploring should be after the prediction","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"le = LabelEncoder()\ntrain[\"primary_use\"] = le.fit_transform(train[\"primary_use\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['hour'] = np.uint8(train['timestamp'].dt.hour)\ntrain['day'] = np.uint8(train['timestamp'].dt.day)\ntrain['weekday'] = np.uint8(train['timestamp'].dt.weekday)\ntrain['month'] = np.uint8(train['timestamp'].dt.month)\ntrain['year'] = np.uint8(train['timestamp'].dt.year-2000)\ntrain = train.drop(\"timestamp\", axis = 1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"emptylist = []\nfor col in train.columns:\n    max1 = train[col].max()\n    min1 = train[col].min()\n    if not np.isfinite(train[col]).all(): \n        emptylist.append(col)\n        train[col].fillna(min1-1,inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(16,8))\n\ncorr = train[[col for col in train.columns if col != 'year']].sample(10000).corr(method='spearman')\n_ = sns.heatmap(corr, annot=True, xticklabels=corr.columns.values, yticklabels=corr.columns.values)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = np.log1p(train[\"meter_reading\"])\ntrain = train.drop([\"meter_reading\"],axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split \nxtrain, xval, ytrain, yval = train_test_split(train, target, test_size=0.3, random_state=42)\nRFRxtrain, RFRxtest, RFRytrain, RFRytest = train_test_split(train, target, test_size=0.995, random_state=42)\nRFRxtrain, RFRxtest, RFRytrain, RFRytest = train_test_split(RFRxtrain, RFRytrain, test_size=0.3, random_state=42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import metrics\nfrom sklearn.linear_model  import SGDRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.preprocessing import StandardScaler \nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error,  mean_absolute_error\nfrom sklearn.model_selection import cross_validate, KFold, RandomizedSearchCV, GridSearchCV, train_test_split\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = DecisionTreeRegressor(max_depth=30)\nmodel.fit(xtrain, ytrain)\n# This takes around ... minutes to run","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ypred_val = model.predict(xval)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nplt.scatter(yval[0:10000], ypred_val[0:10000], s=30)\nplt.title('Predicted vs. Actual meter reading')\nplt.xlabel('Actual meter reading')\nplt.ylabel('Predicted meter reading')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SGDreg   =  SGDRegressor(random_state = 42)\nSGDreg.fit(xtrain, ytrain.ravel())\ny_pred=SGDreg.predict(xval)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 5))\nplt.scatter(yval[0:10000], y_pred[0:10000], s=30)\nplt.title('Predicted vs. Actual meter reading')\nplt.xlabel('Actual meter reading')\nplt.ylabel('Predicted meter reading')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_memory_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_memory_usage(RFRxtrain)\nreduce_memory_usage(RFRxtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RFRreg = RandomForestRegressor(n_estimators = 100, random_state = 0)\nRFRreg.fit(RFRxtrain, RFRytrain)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RFR_ypred = RFRreg.predict(RFRxtest)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.scatter(RFRytest[0:10000], RFR_ypred[0:10000], s=30)\nplt.title('Predicted vs. Actual meter reading')\nplt.xlabel('Actual meter reading')\nplt.ylabel('Predicted meter reading')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}