{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%matplotlib inline\nimport pandas as pd\nimport seaborn as sns\nimport statsmodels.formula.api as smf\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn import metrics\nfrom sklearn.model_selection  import train_test_split\nimport numpy as np\nfrom scipy.stats import norm # for scientific Computing\nfrom scipy import stats, integrate\nimport matplotlib.pyplot as plt","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-09T19:10:13.685200Z","iopub.execute_input":"2022-01-09T19:10:13.685644Z","iopub.status.idle":"2022-01-09T19:10:15.151628Z","shell.execute_reply.started":"2022-01-09T19:10:13.685553Z","shell.execute_reply":"2022-01-09T19:10:15.150838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train =  pd.read_csv('../input/ashrae-great-energy-prediction/train.csv')\nASHRAE_test = pd.read_csv('../input/ashrae-energy-prediction/test.csv')\nASHRAE_train = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\nbuilding_meta = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nweather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')\n#weather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\n#building = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\n#weather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:11.677898Z","iopub.execute_input":"2022-01-09T19:30:11.678567Z","iopub.status.idle":"2022-01-09T19:30:34.136131Z","shell.execute_reply.started":"2022-01-09T19:30:11.678529Z","shell.execute_reply":"2022-01-09T19:30:34.135118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ASHRAE_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:39.163919Z","iopub.execute_input":"2022-01-09T19:30:39.164174Z","iopub.status.idle":"2022-01-09T19:30:39.174664Z","shell.execute_reply.started":"2022-01-09T19:30:39.164148Z","shell.execute_reply":"2022-01-09T19:30:39.173635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:39.981089Z","iopub.execute_input":"2022-01-09T19:30:39.981533Z","iopub.status.idle":"2022-01-09T19:30:40.005371Z","shell.execute_reply.started":"2022-01-09T19:30:39.981502Z","shell.execute_reply":"2022-01-09T19:30:40.004449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Function to reduce the DF size\ndef reduce_memory_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:40.633130Z","iopub.execute_input":"2022-01-09T19:30:40.633905Z","iopub.status.idle":"2022-01-09T19:30:40.645785Z","shell.execute_reply.started":"2022-01-09T19:30:40.633864Z","shell.execute_reply":"2022-01-09T19:30:40.644760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_memory_usage(building_meta)\nreduce_memory_usage(weather_train)\nreduce_memory_usage(ASHRAE_train)\n\nreduce_memory_usage(weather_test)\nreduce_memory_usage(ASHRAE_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:41.481301Z","iopub.execute_input":"2022-01-09T19:30:41.481562Z","iopub.status.idle":"2022-01-09T19:30:42.833913Z","shell.execute_reply.started":"2022-01-09T19:30:41.481536Z","shell.execute_reply":"2022-01-09T19:30:42.833062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ASHRAE_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:30:59.924422Z","iopub.execute_input":"2022-01-09T19:30:59.925043Z","iopub.status.idle":"2022-01-09T19:31:01.450913Z","shell.execute_reply.started":"2022-01-09T19:30:59.925004Z","shell.execute_reply":"2022-01-09T19:31:01.450049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Size of the building dataset is', building_meta.shape)\nprint('Size of the weather_train dataset is', weather_train.shape)\nprint('Size of the train dataset is', ASHRAE_train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:31:23.613439Z","iopub.execute_input":"2022-01-09T19:31:23.614022Z","iopub.status.idle":"2022-01-09T19:31:23.621673Z","shell.execute_reply.started":"2022-01-09T19:31:23.613980Z","shell.execute_reply":"2022-01-09T19:31:23.620837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ASHRAE_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:32:08.531750Z","iopub.execute_input":"2022-01-09T19:32:08.532041Z","iopub.status.idle":"2022-01-09T19:32:10.049811Z","shell.execute_reply.started":"2022-01-09T19:32:08.532009Z","shell.execute_reply":"2022-01-09T19:32:10.048252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"building_meta.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:32:23.587833Z","iopub.execute_input":"2022-01-09T19:32:23.588728Z","iopub.status.idle":"2022-01-09T19:32:23.616517Z","shell.execute_reply.started":"2022-01-09T19:32:23.588670Z","shell.execute_reply":"2022-01-09T19:32:23.615732Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primary_use_numbersOfUniqueValue = building_meta['primary_use'].nunique()\n \nprint('Number of unique values in column \"primary_use\" of the building_meta : ')\nprint(primary_use_numbersOfUniqueValue)\nprimary_use_element = building_meta['primary_use'].unique()\n \nprint('Unique element in column \"primary_use\" of the building_meta : ')\nprint(primary_use_element)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:32:47.413030Z","iopub.execute_input":"2022-01-09T19:32:47.413774Z","iopub.status.idle":"2022-01-09T19:32:47.420563Z","shell.execute_reply.started":"2022-01-09T19:32:47.413735Z","shell.execute_reply":"2022-01-09T19:32:47.419995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Columns of the building dataset is', building_meta.columns)\nprint('Columns of the weather_train dataset is', weather_train.columns)\nprint('Columns of the train dataset is', ASHRAE_train.columns)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:33:06.516615Z","iopub.execute_input":"2022-01-09T19:33:06.517242Z","iopub.status.idle":"2022-01-09T19:33:06.523729Z","shell.execute_reply.started":"2022-01-09T19:33:06.517177Z","shell.execute_reply":"2022-01-09T19:33:06.522950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(15,7))\nsns.heatmap(building_meta.isnull(), yticklabels=False,cmap='viridis')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:33:27.674965Z","iopub.execute_input":"2022-01-09T19:33:27.675814Z","iopub.status.idle":"2022-01-09T19:33:28.002134Z","shell.execute_reply.started":"2022-01-09T19:33:27.675759Z","shell.execute_reply":"2022-01-09T19:33:28.000405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Percentage of missing values in the building_meta dataset\")\nbuilding_meta.isna().sum()/len(building_meta)*100\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:33:53.094075Z","iopub.execute_input":"2022-01-09T19:33:53.094345Z","iopub.status.idle":"2022-01-09T19:33:53.103338Z","shell.execute_reply.started":"2022-01-09T19:33:53.094317Z","shell.execute_reply":"2022-01-09T19:33:53.102755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Percentage of missing values in the train dataset\")\nASHRAE_train.isna().sum()/len(ASHRAE_train)*100","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:34:03.345435Z","iopub.execute_input":"2022-01-09T19:34:03.346408Z","iopub.status.idle":"2022-01-09T19:34:04.362155Z","shell.execute_reply.started":"2022-01-09T19:34:03.346350Z","shell.execute_reply":"2022-01-09T19:34:04.361256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.merge(df1, df2, on='employee')\nBuildingTrainMerge=building_meta.merge(ASHRAE_train,left_on='building_id',right_on='building_id',how='left')\nBuildingTrainMerge.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:34:23.414602Z","iopub.execute_input":"2022-01-09T19:34:23.415446Z","iopub.status.idle":"2022-01-09T19:34:28.557218Z","shell.execute_reply.started":"2022-01-09T19:34:23.415408Z","shell.execute_reply":"2022-01-09T19:34:28.556283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train=BuildingTrainMerge.merge(weather_train,left_on=['site_id','timestamp'],right_on=['site_id','timestamp'],how='left')\nBTW_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:34:37.241346Z","iopub.execute_input":"2022-01-09T19:34:37.241620Z","iopub.status.idle":"2022-01-09T19:34:42.714164Z","shell.execute_reply.started":"2022-01-09T19:34:37.241591Z","shell.execute_reply":"2022-01-09T19:34:42.713171Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:35:04.167674Z","iopub.execute_input":"2022-01-09T19:35:04.167945Z","iopub.status.idle":"2022-01-09T19:35:04.173834Z","shell.execute_reply.started":"2022-01-09T19:35:04.167917Z","shell.execute_reply":"2022-01-09T19:35:04.173000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Percentage of missing values in the BTW_train dataset\")\nBTW_train.isna().sum()/len(BTW_train)*100","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:35:17.335322Z","iopub.execute_input":"2022-01-09T19:35:17.335617Z","iopub.status.idle":"2022-01-09T19:35:19.942756Z","shell.execute_reply.started":"2022-01-09T19:35:17.335588Z","shell.execute_reply":"2022-01-09T19:35:19.941796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.hist('sea_level_pressure')\nBTW_train[['sea_level_pressure']].describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:35:39.658016Z","iopub.execute_input":"2022-01-09T19:35:39.658427Z","iopub.status.idle":"2022-01-09T19:35:49.298252Z","shell.execute_reply.started":"2022-01-09T19:35:39.658387Z","shell.execute_reply":"2022-01-09T19:35:49.297438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.hist('cloud_coverage')\nBTW_train[['cloud_coverage']].describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:37:26.261896Z","iopub.execute_input":"2022-01-09T19:37:26.262190Z","iopub.status.idle":"2022-01-09T19:37:31.166831Z","shell.execute_reply.started":"2022-01-09T19:37:26.262161Z","shell.execute_reply":"2022-01-09T19:37:31.166008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.hist('precip_depth_1_hr')\nBTW_train[['precip_depth_1_hr']].describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:38:16.480579Z","iopub.execute_input":"2022-01-09T19:38:16.480891Z","iopub.status.idle":"2022-01-09T19:38:21.880055Z","shell.execute_reply.started":"2022-01-09T19:38:16.480859Z","shell.execute_reply":"2022-01-09T19:38:21.879165Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.hist('wind_speed')\nBTW_train[['wind_speed']].describe()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:39:16.273145Z","iopub.execute_input":"2022-01-09T19:39:16.274153Z","iopub.status.idle":"2022-01-09T19:39:21.776169Z","shell.execute_reply.started":"2022-01-09T19:39:16.274105Z","shell.execute_reply":"2022-01-09T19:39:21.775520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.hist(column='air_temperature')\nBTW_train[['air_temperature']].describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:39:21.777671Z","iopub.execute_input":"2022-01-09T19:39:21.778541Z","iopub.status.idle":"2022-01-09T19:39:27.209502Z","shell.execute_reply.started":"2022-01-09T19:39:21.778495Z","shell.execute_reply":"2022-01-09T19:39:27.208671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.boxplot(x = 'meter', y = 'meter_reading', data = BTW_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:40:16.233183Z","iopub.execute_input":"2022-01-09T19:40:16.233484Z","iopub.status.idle":"2022-01-09T19:40:26.327359Z","shell.execute_reply.started":"2022-01-09T19:40:16.233447Z","shell.execute_reply":"2022-01-09T19:40:26.326512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def outlier_function(df, col_name):\n    ''' this function detects first and third quartile and interquartile range for a given column of a dataframe\n    then calculates upper and lower limits to determine outliers conservatively\n    returns the number of lower and uper limit and number of outliers respectively\n    '''\n    first_quartile = np.percentile(\n        np.array(df[col_name].tolist()), 25)\n    third_quartile = np.percentile(\n        np.array(df[col_name].tolist()), 75)\n    IQR = third_quartile - first_quartile\n                      \n    upper_limit = third_quartile+(3*IQR)\n    lower_limit = first_quartile-(3*IQR)\n    outlier_count = 0\n                      \n    for value in df[col_name].tolist():\n        if (value < lower_limit) | (value > upper_limit):\n            outlier_count +=1\n    return lower_limit, upper_limit, outlier_count","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:41:22.906858Z","iopub.execute_input":"2022-01-09T19:41:22.907396Z","iopub.status.idle":"2022-01-09T19:41:22.916033Z","shell.execute_reply.started":"2022-01-09T19:41:22.907344Z","shell.execute_reply":"2022-01-09T19:41:22.914874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"{} percent of {} are outliers.\"\n      .format((\n              (100 * outlier_function(BTW_train, 'meter_reading')[2])\n               / len(BTW_train['meter_reading'])),\n              'meter_reading'))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:44:16.099127Z","iopub.execute_input":"2022-01-09T19:44:16.100038Z","iopub.status.idle":"2022-01-09T19:44:25.660774Z","shell.execute_reply.started":"2022-01-09T19:44:16.099996Z","shell.execute_reply":"2022-01-09T19:44:25.659918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of the meter reading in meters without zeros\nplt.figure(figsize=(12,10))\n\n#list of different meters\nmeters = sorted(BTW_train['meter'].unique().tolist())\n\n# plot meter_reading distribution for each meter\nfor meter_type in meters:\n    subset = BTW_train[BTW_train['meter'] == meter_type]\n    sns.kdeplot(np.log1p(subset[\"meter_reading\"]), \n                label=meter_type, linewidth=2)\n\n# set title, legends and labels\nplt.ylabel(\"Density\")\nplt.xlabel(\"Meter_reading\")\nplt.legend(['electricity', 'chilled water', 'steam', 'hot water'])\nplt.title(\"Density of Logartihm(Meter Reading + 1) Among Different Meters\", size=14)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:45:28.113752Z","iopub.execute_input":"2022-01-09T19:45:28.114038Z","iopub.status.idle":"2022-01-09T19:46:49.983762Z","shell.execute_reply.started":"2022-01-09T19:45:28.114006Z","shell.execute_reply":"2022-01-09T19:46:49.983251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:47:15.904896Z","iopub.execute_input":"2022-01-09T19:47:15.905236Z","iopub.status.idle":"2022-01-09T19:47:15.912334Z","shell.execute_reply.started":"2022-01-09T19:47:15.905198Z","shell.execute_reply":"2022-01-09T19:47:15.911456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corrmat=BTW_train.corr()\nfig,ax=plt.subplots(figsize=(12,10))\nsns.heatmap(corrmat,annot=True,annot_kws={'size': 12})","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:47:46.765688Z","iopub.execute_input":"2022-01-09T19:47:46.766387Z","iopub.status.idle":"2022-01-09T19:47:59.217657Z","shell.execute_reply.started":"2022-01-09T19:47:46.766349Z","shell.execute_reply":"2022-01-09T19:47:59.216728Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train = BTW_train.drop(columns=['year_built', 'floor_count', 'wind_direction', 'dew_temperature'])\nBTW_train ['timestamp'] =  pd.to_datetime(BTW_train['timestamp'])\nBTW_train['Month']=pd.DatetimeIndex(BTW_train['timestamp']).month\nBTW_train['Day']=pd.DatetimeIndex(BTW_train['timestamp']).day","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:47:59.218983Z","iopub.execute_input":"2022-01-09T19:47:59.219188Z","iopub.status.idle":"2022-01-09T19:48:07.785210Z","shell.execute_reply.started":"2022-01-09T19:47:59.219164Z","shell.execute_reply":"2022-01-09T19:48:07.784247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train= BTW_train.groupby(['meter',BTW_train['building_id'],'primary_use',BTW_train['Month'], BTW_train['Day']]).agg({'meter_reading':'sum', 'air_temperature': 'mean', 'wind_speed': 'mean', 'precip_depth_1_hr': 'mean', 'cloud_coverage': 'mean', 'square_feet': 'mean'})\n","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:48:13.318577Z","iopub.execute_input":"2022-01-09T19:48:13.319298Z","iopub.status.idle":"2022-01-09T19:48:19.194567Z","shell.execute_reply.started":"2022-01-09T19:48:13.319259Z","shell.execute_reply":"2022-01-09T19:48:19.193688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:48:22.019342Z","iopub.execute_input":"2022-01-09T19:48:22.020052Z","iopub.status.idle":"2022-01-09T19:48:22.026889Z","shell.execute_reply.started":"2022-01-09T19:48:22.020006Z","shell.execute_reply":"2022-01-09T19:48:22.026007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train = BTW_train.reset_index()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:48:31.916719Z","iopub.execute_input":"2022-01-09T19:48:31.917317Z","iopub.status.idle":"2022-01-09T19:48:31.960980Z","shell.execute_reply.started":"2022-01-09T19:48:31.917271Z","shell.execute_reply":"2022-01-09T19:48:31.960293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:48:48.551361Z","iopub.execute_input":"2022-01-09T19:48:48.551654Z","iopub.status.idle":"2022-01-09T19:48:49.229021Z","shell.execute_reply.started":"2022-01-09T19:48:48.551621Z","shell.execute_reply":"2022-01-09T19:48:49.228159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train['wind_speed'] = BTW_train['wind_speed'].astype('float32')\nBTW_train['air_temperature'] = BTW_train['air_temperature'].astype('float32')\nBTW_train['precip_depth_1_hr'] = BTW_train['precip_depth_1_hr'].astype('float32')\nBTW_train['cloud_coverage'] = BTW_train['cloud_coverage'].astype('float32')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:49:00.693387Z","iopub.execute_input":"2022-01-09T19:49:00.693871Z","iopub.status.idle":"2022-01-09T19:49:00.717843Z","shell.execute_reply.started":"2022-01-09T19:49:00.693826Z","shell.execute_reply":"2022-01-09T19:49:00.716925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train['precip_depth_1_hr'].fillna(method='ffill', inplace = True)\nBTW_train['cloud_coverage'].fillna(method='bfill', inplace = True)\n\nBTW_train['wind_speed'].fillna(BTW_train['wind_speed'].mean(), inplace=True)\nBTW_train['air_temperature'].fillna(BTW_train['air_temperature'].mean(), inplace=True)\nBTW_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:49:13.701954Z","iopub.execute_input":"2022-01-09T19:49:13.702251Z","iopub.status.idle":"2022-01-09T19:49:13.778604Z","shell.execute_reply.started":"2022-01-09T19:49:13.702223Z","shell.execute_reply":"2022-01-09T19:49:13.777785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:49:29.812579Z","iopub.execute_input":"2022-01-09T19:49:29.813436Z","iopub.status.idle":"2022-01-09T19:49:29.818834Z","shell.execute_reply.started":"2022-01-09T19:49:29.813394Z","shell.execute_reply":"2022-01-09T19:49:29.817994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:49:37.447385Z","iopub.execute_input":"2022-01-09T19:49:37.447669Z","iopub.status.idle":"2022-01-09T19:49:37.455265Z","shell.execute_reply.started":"2022-01-09T19:49:37.447638Z","shell.execute_reply":"2022-01-09T19:49:37.454411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_train.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:50:01.596610Z","iopub.execute_input":"2022-01-09T19:50:01.597362Z","iopub.status.idle":"2022-01-09T19:50:01.603205Z","shell.execute_reply.started":"2022-01-09T19:50:01.597321Z","shell.execute_reply":"2022-01-09T19:50:01.602441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_linearR = pd.get_dummies(BTW_train, columns=['primary_use'])","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:50:05.262880Z","iopub.execute_input":"2022-01-09T19:50:05.263625Z","iopub.status.idle":"2022-01-09T19:50:05.388107Z","shell.execute_reply.started":"2022-01-09T19:50:05.263587Z","shell.execute_reply":"2022-01-09T19:50:05.387507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BTW_linearR.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:50:15.955982Z","iopub.execute_input":"2022-01-09T19:50:15.956857Z","iopub.status.idle":"2022-01-09T19:50:15.962402Z","shell.execute_reply.started":"2022-01-09T19:50:15.956818Z","shell.execute_reply":"2022-01-09T19:50:15.961746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X =BTW_linearR[['building_id', 'meter', 'air_temperature', 'wind_speed', 'precip_depth_1_hr', 'cloud_coverage',\n       'square_feet', 'primary_use_Education', 'primary_use_Entertainment/public assembly',\n       'primary_use_Food sales and service', 'primary_use_Healthcare',\n       'primary_use_Lodging/residential',\n       'primary_use_Manufacturing/industrial', 'primary_use_Office',\n       'primary_use_Other', 'primary_use_Parking',\n       'primary_use_Public services', 'primary_use_Religious worship',\n       'primary_use_Retail', 'primary_use_Services',\n       'primary_use_Technology/science', 'primary_use_Utility',\n       'primary_use_Warehouse/storage', 'Month', 'Day']]\n\n# Create target variable\ny = BTW_linearR['meter_reading']\n\n# Train, test, split\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X,y, test_size = .20, random_state= 0)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:50:34.286229Z","iopub.execute_input":"2022-01-09T19:50:34.286798Z","iopub.status.idle":"2022-01-09T19:50:34.487000Z","shell.execute_reply.started":"2022-01-09T19:50:34.286762Z","shell.execute_reply":"2022-01-09T19:50:34.486155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit\n# Import model\nfrom sklearn.linear_model import LinearRegression\n\n# Create linear regression object\nregressor = LinearRegression()\n\n# Fit model to training data\nregressor.fit(X_train,y_train)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:50:57.011589Z","iopub.execute_input":"2022-01-09T19:50:57.011893Z","iopub.status.idle":"2022-01-09T19:50:57.617915Z","shell.execute_reply.started":"2022-01-09T19:50:57.011862Z","shell.execute_reply":"2022-01-09T19:50:57.616872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Predicting test set results\ny_pred = regressor.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:51:05.811241Z","iopub.execute_input":"2022-01-09T19:51:05.811513Z","iopub.status.idle":"2022-01-09T19:51:05.835808Z","shell.execute_reply.started":"2022-01-09T19:51:05.811486Z","shell.execute_reply":"2022-01-09T19:51:05.834792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Accuracy %d', regressor.score(X_test, y_test))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:51:15.138932Z","iopub.execute_input":"2022-01-09T19:51:15.139242Z","iopub.status.idle":"2022-01-09T19:51:15.173527Z","shell.execute_reply.started":"2022-01-09T19:51:15.139215Z","shell.execute_reply":"2022-01-09T19:51:15.170708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Calculate R Sqaured\nprint('R^2 =',metrics.explained_variance_score(y_test,y_pred))#Calculate R Sqaured\nprint('R^2 =',metrics.explained_variance_score(y_test,y_pred))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:51:29.852110Z","iopub.execute_input":"2022-01-09T19:51:29.852407Z","iopub.status.idle":"2022-01-09T19:51:29.864014Z","shell.execute_reply.started":"2022-01-09T19:51:29.852372Z","shell.execute_reply":"2022-01-09T19:51:29.863151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdf = pd.DataFrame(data = regressor.coef_, index = X.columns, columns = ['Coefficients'])\ncdf","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:51:36.471190Z","iopub.execute_input":"2022-01-09T19:51:36.471754Z","iopub.status.idle":"2022-01-09T19:51:36.482850Z","shell.execute_reply.started":"2022-01-09T19:51:36.471707Z","shell.execute_reply":"2022-01-09T19:51:36.481885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cdf.Coefficients.nlargest(10).plot(kind='barh')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:51:50.972324Z","iopub.execute_input":"2022-01-09T19:51:50.972742Z","iopub.status.idle":"2022-01-09T19:51:51.221296Z","shell.execute_reply.started":"2022-01-09T19:51:50.972687Z","shell.execute_reply":"2022-01-09T19:51:51.220407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import statsmodels.api as sm\nfrom scipy import stats\nX =BTW_linearR[['building_id', 'meter', 'air_temperature', 'wind_speed', 'precip_depth_1_hr', 'cloud_coverage',\n       'square_feet', 'primary_use_Education', 'primary_use_Entertainment/public assembly',\n       'primary_use_Food sales and service', 'primary_use_Healthcare',\n       'primary_use_Lodging/residential',\n       'primary_use_Manufacturing/industrial', 'primary_use_Office',\n       'primary_use_Other', 'primary_use_Parking',\n       'primary_use_Public services', 'primary_use_Religious worship',\n       'primary_use_Retail', 'primary_use_Services',\n       'primary_use_Technology/science', 'primary_use_Utility',\n       'primary_use_Warehouse/storage', 'Month', 'Day']]\n\n# Create target variable\ny = BTW_linearR['meter_reading']\n \n \n \nX2 = sm.add_constant(X)\nest = sm.OLS(y, X2)\nest2 = est.fit()\nprint(est2.summary())","metadata":{"execution":{"iopub.status.busy":"2022-01-09T19:52:09.114986Z","iopub.execute_input":"2022-01-09T19:52:09.115445Z","iopub.status.idle":"2022-01-09T19:52:11.799621Z","shell.execute_reply.started":"2022-01-09T19:52:09.115416Z","shell.execute_reply":"2022-01-09T19:52:11.798656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}