{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-09T14:36:16.306334Z","iopub.execute_input":"2022-01-09T14:36:16.306884Z","iopub.status.idle":"2022-01-09T14:36:16.341014Z","shell.execute_reply.started":"2022-01-09T14:36:16.306763Z","shell.execute_reply":"2022-01-09T14:36:16.340339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pylab as plt\nimport seaborn as sns\nimport plotly.express as px\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:36:16.343955Z","iopub.execute_input":"2022-01-09T14:36:16.344795Z","iopub.status.idle":"2022-01-09T14:36:18.662551Z","shell.execute_reply.started":"2022-01-09T14:36:16.344725Z","shell.execute_reply":"2022-01-09T14:36:18.661574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load data\ntrain = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\n#train['timestamp'] = pd.to_datetime(train['timestamp']) #the train dataset contains a 'timestamp' column we convert to a datetime object for ease of use\n#test = pd.read_csv('../input/ashrae-energy-prediction/test.csv')\n#test['timestamp'] = pd.to_datetime(test['timestamp'])\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\n#weather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')\nbuild_meta = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\n#sample_submission = pd.read_csv('../input/ashrae-energy-prediction/sample_submission.csv')\ndf_test= pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:36:18.664179Z","iopub.execute_input":"2022-01-09T14:36:18.664448Z","iopub.status.idle":"2022-01-09T14:37:07.960573Z","shell.execute_reply.started":"2022-01-09T14:36:18.664416Z","shell.execute_reply":"2022-01-09T14:37:07.959354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The challenge \nWe must construct accurate projections of metered building energy use in the following areas: chilled water, electric, hot water, and steam meters in this competition. Over 1,000 structures were studied over a three-year period. The goal is that, with more accurate estimations of these energy-saving projects, big-scale investors and financial institutions will be more motivated to invest in this area, allowing advances in building efficiency to be made.","metadata":{}},{"cell_type":"markdown","source":"# 1.Full EDA ","metadata":{}},{"cell_type":"markdown","source":"> # 1.1.train EDA","metadata":{}},{"cell_type":"code","source":"#summary of a train data\ntrain.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:07.962307Z","iopub.execute_input":"2022-01-09T14:37:07.962657Z","iopub.status.idle":"2022-01-09T14:37:07.992889Z","shell.execute_reply.started":"2022-01-09T14:37:07.962609Z","shell.execute_reply":"2022-01-09T14:37:07.991912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#the sum of the null values \ntrain.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:07.996047Z","iopub.execute_input":"2022-01-09T14:37:07.996613Z","iopub.status.idle":"2022-01-09T14:37:10.347274Z","shell.execute_reply.started":"2022-01-09T14:37:07.996567Z","shell.execute_reply":"2022-01-09T14:37:10.346404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dimensionality of the train data.\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:10.348448Z","iopub.execute_input":"2022-01-09T14:37:10.348675Z","iopub.status.idle":"2022-01-09T14:37:10.35446Z","shell.execute_reply.started":"2022-01-09T14:37:10.348648Z","shell.execute_reply":"2022-01-09T14:37:10.353475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:10.355753Z","iopub.execute_input":"2022-01-09T14:37:10.356023Z","iopub.status.idle":"2022-01-09T14:37:10.378157Z","shell.execute_reply.started":"2022-01-09T14:37:10.355993Z","shell.execute_reply":"2022-01-09T14:37:10.377456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# statistical details about train data\ntrain.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:10.379348Z","iopub.execute_input":"2022-01-09T14:37:10.379856Z","iopub.status.idle":"2022-01-09T14:37:12.810383Z","shell.execute_reply.started":"2022-01-09T14:37:10.37982Z","shell.execute_reply":"2022-01-09T14:37:12.809336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train[\"timestamp\"].min() , train[\"timestamp\"].max(),sep =\"\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:12.81164Z","iopub.execute_input":"2022-01-09T14:37:12.811926Z","iopub.status.idle":"2022-01-09T14:37:18.73176Z","shell.execute_reply.started":"2022-01-09T14:37:12.811894Z","shell.execute_reply":"2022-01-09T14:37:18.730849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# statistical details about meter_reading\ntrain['meter_reading'].describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:18.733454Z","iopub.execute_input":"2022-01-09T14:37:18.733938Z","iopub.status.idle":"2022-01-09T14:37:19.710292Z","shell.execute_reply.started":"2022-01-09T14:37:18.733893Z","shell.execute_reply":"2022-01-09T14:37:19.709414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zeros = train[train['meter_reading']== 0].shape[0] / train.shape[0]\nprint('Zero reading percentile : {}'.format(round(zeros,3)))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:19.711919Z","iopub.execute_input":"2022-01-09T14:37:19.712406Z","iopub.status.idle":"2022-01-09T14:37:20.016267Z","shell.execute_reply.started":"2022-01-09T14:37:19.712362Z","shell.execute_reply":"2022-01-09T14:37:20.015236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The reading value is zero in 9% of the time. We know that some buildings have many meter types, thus we may assume that using seasonal energy meters will result in zeros. We don't need to use cooled water on chilly days, to be more specific. Similarly, hot water use might vary from time to time. Those data, however, might be absent because no energy usage makes no sense!","metadata":{}},{"cell_type":"code","source":"#converting the meters to what they stand for \ntrain['meter'].replace({0:\"electricity\",1:\"chilledwater\",2:\"steam\",3:\"hotwater\"},inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:20.018189Z","iopub.execute_input":"2022-01-09T14:37:20.018527Z","iopub.status.idle":"2022-01-09T14:37:21.399902Z","shell.execute_reply.started":"2022-01-09T14:37:20.018482Z","shell.execute_reply":"2022-01-09T14:37:21.398993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# to show the precentage of meters that having zeroes \nmeter_dict = {}\nfor i in train[\"meter\"].unique():\n    percent = round(train[train['meter_reading']== 0][\"meter\"].value_counts()[i] /train[\"meter\"].value_counts()[i],2)\n    meter_dict[i] = percent\nzero_meter = pd.Series(meter_dict)\nsns.barplot(x=zero_meter.index, y= zero_meter,palette =\"Set2\")\nplt.title(\"Meters percentage having zero readings\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:21.401297Z","iopub.execute_input":"2022-01-09T14:37:21.401528Z","iopub.status.idle":"2022-01-09T14:37:35.629003Z","shell.execute_reply.started":"2022-01-09T14:37:21.401501Z","shell.execute_reply":"2022-01-09T14:37:35.62812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"timestamp2\"] = pd.to_datetime(train[\"timestamp\"])\ntrain[\"month\"] = train.timestamp2.dt.month","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:35.632575Z","iopub.execute_input":"2022-01-09T14:37:35.632853Z","iopub.status.idle":"2022-01-09T14:37:41.298135Z","shell.execute_reply.started":"2022-01-09T14:37:35.632823Z","shell.execute_reply":"2022-01-09T14:37:41.297287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axs = plt.subplots(2,2, sharey=True, tight_layout=True,figsize=(10,6))\n\naxs[0][0].hist(x =\"month\",data =train[(train.meter_reading == 0) & (train.meter==\"electricity\")],bins =12,color = \"navajowhite\")\naxs[0][0].set_title(\"For electricity\")\n\naxs[0][1].hist(x =\"month\",data =train[(train.meter_reading == 0) & (train.meter==\"chilledwater\")],bins =12,color = \"skyblue\")\naxs[0][1].set_title(\"For chilled water\")\n\naxs[1][0].hist(x =\"month\",data =train[(train.meter_reading == 0) & (train.meter==\"steam\")],bins =12,color = \"slategrey\")\naxs[1][0].set_title(\"For steam\")\n\naxs[1][1].hist(x =\"month\",data =train[(train.meter_reading == 0) & (train.meter==\"hotwater\")],bins =12,color = \"lightcoral\")\naxs[1][1].set_title(\"For hot water\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:41.299541Z","iopub.execute_input":"2022-01-09T14:37:41.29976Z","iopub.status.idle":"2022-01-09T14:37:54.681703Z","shell.execute_reply.started":"2022-01-09T14:37:41.299733Z","shell.execute_reply":"2022-01-09T14:37:54.681045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As seen in the graphs, zero readings vary with time and meter type. The first five months of the year have witnessed electricity meter zeros. The number of zero readings decreases as the month progresses. Seasonal variations in chilled and hot water zeros can be found. The distribution of zeros in steam and hot water is remarkably similar.","metadata":{}},{"cell_type":"code","source":"# Function for remove Outlier\n\ndef removeOutliers(data, outlierConstant = 1.5):\n    values = np.array(data)\n    upper_quartile = np.percentile(values, 75)\n    lower_quartile = np.percentile(values, 25)\n    IQR = (upper_quartile - lower_quartile) * outlierConstant\n    quartile = (lower_quartile - IQR, upper_quartile + IQR)\n    results = []\n    for i in values.tolist():\n        if i >= quartile[0] and i <= quartile[1]:\n            results.append(i)\n    return results","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:54.682719Z","iopub.execute_input":"2022-01-09T14:37:54.683082Z","iopub.status.idle":"2022-01-09T14:37:54.689479Z","shell.execute_reply.started":"2022-01-09T14:37:54.683053Z","shell.execute_reply":"2022-01-09T14:37:54.688793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_noOutlier = removeOutliers(train.meter_reading)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:37:54.690661Z","iopub.execute_input":"2022-01-09T14:37:54.690878Z","iopub.status.idle":"2022-01-09T14:38:05.921191Z","shell.execute_reply.started":"2022-01-09T14:37:54.690852Z","shell.execute_reply":"2022-01-09T14:38:05.920278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 3,figsize=(12,5))\n\nsns.distplot(train.meter_reading,hist = False,ax = axes[0])\naxes[0].set_title(\"Target distribution\")\n\nsns.distplot(target_noOutlier,ax=axes[1])\naxes[1].set_title(\"Target distribution without outlier\")\n\nsns.distplot(np.log1p(train.meter_reading),ax = axes[2])\naxes[2].set_title(\"Target with Log Transform\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:38:05.92244Z","iopub.execute_input":"2022-01-09T14:38:05.92311Z","iopub.status.idle":"2022-01-09T14:41:49.61999Z","shell.execute_reply.started":"2022-01-09T14:38:05.923067Z","shell.execute_reply":"2022-01-09T14:41:49.618953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['meter_reading_log'] = np.log1p(train['meter_reading'])","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:41:49.621721Z","iopub.execute_input":"2022-01-09T14:41:49.621995Z","iopub.status.idle":"2022-01-09T14:41:50.114047Z","shell.execute_reply.started":"2022-01-09T14:41:49.621959Z","shell.execute_reply":"2022-01-09T14:41:50.113221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The meter reading variable is highly right-skewed, with a large number of zeros. A skewed distribution is transformed into a more normalized dataset via logarithmic transformation. We have outliers in addition to zeros readings!","metadata":{}},{"cell_type":"code","source":"import seaborn as sns\nsns.countplot(x= \"meter\",data = train,color = 'b')","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:41:50.115549Z","iopub.execute_input":"2022-01-09T14:41:50.115772Z","iopub.status.idle":"2022-01-09T14:42:15.074042Z","shell.execute_reply.started":"2022-01-09T14:41:50.115746Z","shell.execute_reply":"2022-01-09T14:42:15.073395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Graphs shows distribution meter_reading in each meter category\n\nsns.kdeplot(train.loc[(train['meter']=='electricity'), \n            \"meter_reading_log\"], color='yellow', shade=False, label='electricity')\n\nsns.kdeplot(train.loc[(train['meter']=='chilledwater'), \n            \"meter_reading_log\"], color='b', shade=False, label='chilledwater')\n\nsns.kdeplot(train.loc[(train['meter']=='steam'), \n            \"meter_reading_log\"], color='gray', shade=False, label='steam')\n\nsns.kdeplot(train.loc[(train['meter']=='hotwater'), \n            \"meter_reading_log\"], color='r', shade=False, label='hotwater')\n\nplt.xlabel('meter_reading_log') \nplt.ylabel('Probability Density')\nplt.legend()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:42:15.075214Z","iopub.execute_input":"2022-01-09T14:42:15.0757Z","iopub.status.idle":"2022-01-09T14:43:46.177658Z","shell.execute_reply.started":"2022-01-09T14:42:15.07566Z","shell.execute_reply":"2022-01-09T14:43:46.176616Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # 2.1 weather train data EDA","metadata":{}},{"cell_type":"code","source":"# summary of a weather_train data\nweather_train.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:46.179549Z","iopub.execute_input":"2022-01-09T14:43:46.179794Z","iopub.status.idle":"2022-01-09T14:43:46.221933Z","shell.execute_reply.started":"2022-01-09T14:43:46.179764Z","shell.execute_reply":"2022-01-09T14:43:46.220934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dimensionality of the weather_train data.\nweather_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:46.223149Z","iopub.execute_input":"2022-01-09T14:43:46.223393Z","iopub.status.idle":"2022-01-09T14:43:46.228905Z","shell.execute_reply.started":"2022-01-09T14:43:46.22336Z","shell.execute_reply":"2022-01-09T14:43:46.228362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"weather_train.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:46.229799Z","iopub.execute_input":"2022-01-09T14:43:46.230036Z","iopub.status.idle":"2022-01-09T14:43:46.255475Z","shell.execute_reply.started":"2022-01-09T14:43:46.230008Z","shell.execute_reply":"2022-01-09T14:43:46.25452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"8 features are numeric, only timestamp is not numeric we will convert to DateTime.","metadata":{}},{"cell_type":"code","source":"# statistical details about weather_train\nweather_train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:46.257333Z","iopub.execute_input":"2022-01-09T14:43:46.257779Z","iopub.status.idle":"2022-01-09T14:43:46.336583Z","shell.execute_reply.started":"2022-01-09T14:43:46.257729Z","shell.execute_reply":"2022-01-09T14:43:46.335656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We appear to have outliers. we are going to check location using a box plot. In addition, the values varied significantly from one another. To get clear results, scale them before plotting.","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(7,1,figsize=(10,30)) \ncolumns = weather_train.drop([\"site_id\",\"timestamp\"],axis=1).columns\nfor i,col in enumerate(list(columns)):\n\n    plot = weather_train.boxplot(col, by=\"site_id\", ax=axes.flatten()[i])\n\nplt.tight_layout() \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:46.338169Z","iopub.execute_input":"2022-01-09T14:43:46.339069Z","iopub.status.idle":"2022-01-09T14:43:51.025395Z","shell.execute_reply.started":"2022-01-09T14:43:46.33902Z","shell.execute_reply":"2022-01-09T14:43:51.024371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Apart from outliers, we have no or limited data for cloud coverage, precipitation depth, and sea level pressure for several sites. There are a lot of outliers in Precip depth.","metadata":{}},{"cell_type":"code","source":"sns.heatmap(weather_train.corr(),linewidths=.5,cmap=\"YlGnBu\")","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.026801Z","iopub.execute_input":"2022-01-09T14:43:51.027087Z","iopub.status.idle":"2022-01-09T14:43:51.423909Z","shell.execute_reply.started":"2022-01-09T14:43:51.02705Z","shell.execute_reply":"2022-01-09T14:43:51.423099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We have strong correlation between air_temp and dew_temp.","metadata":{}},{"cell_type":"markdown","source":"> # 3.1 building meta EDA","metadata":{}},{"cell_type":"code","source":"# summary of a build_meta data\nbuild_meta.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.425501Z","iopub.execute_input":"2022-01-09T14:43:51.425977Z","iopub.status.idle":"2022-01-09T14:43:51.44107Z","shell.execute_reply.started":"2022-01-09T14:43:51.425943Z","shell.execute_reply":"2022-01-09T14:43:51.440235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dimensionality of the build_meta data.\nbuild_meta.shape","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.442777Z","iopub.execute_input":"2022-01-09T14:43:51.44329Z","iopub.status.idle":"2022-01-09T14:43:51.449074Z","shell.execute_reply.started":"2022-01-09T14:43:51.443245Z","shell.execute_reply":"2022-01-09T14:43:51.44832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"build_meta.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.450003Z","iopub.execute_input":"2022-01-09T14:43:51.45036Z","iopub.status.idle":"2022-01-09T14:43:51.468709Z","shell.execute_reply.started":"2022-01-09T14:43:51.450327Z","shell.execute_reply":"2022-01-09T14:43:51.467858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# statistical details about build_meta\nbuild_meta.describe()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.470306Z","iopub.execute_input":"2022-01-09T14:43:51.471237Z","iopub.status.idle":"2022-01-09T14:43:51.499192Z","shell.execute_reply.started":"2022-01-09T14:43:51.471184Z","shell.execute_reply":"2022-01-09T14:43:51.498096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"build_meta.primary_use.unique()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.500798Z","iopub.execute_input":"2022-01-09T14:43:51.501118Z","iopub.status.idle":"2022-01-09T14:43:51.50871Z","shell.execute_reply.started":"2022-01-09T14:43:51.501077Z","shell.execute_reply":"2022-01-09T14:43:51.507904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"floor_count and year_built have lots of missing values.We can take primary_use as categorical value.","metadata":{}},{"cell_type":"code","source":"build_meta.groupby('site_id').agg({\"building_id\":[min,max]})","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.509991Z","iopub.execute_input":"2022-01-09T14:43:51.510439Z","iopub.status.idle":"2022-01-09T14:43:51.535712Z","shell.execute_reply.started":"2022-01-09T14:43:51.510401Z","shell.execute_reply":"2022-01-09T14:43:51.53485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(1, 2,figsize=(16,5))\nsite_build = build_meta.groupby('site_id').building_id.size()\nsns.barplot(x=site_build.index , y= site_build,color=\"salmon\",ax=axes[0])\naxes[0].set_ylabel(\"Number of building\")\nsns.countplot(y=\"primary_use\",data=build_meta ,ax=axes[1],color=\"salmon\")\nfig.subplots_adjust(wspace=0.5)\nplt.ylabel(\"\")\ndel site_build","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:51.537178Z","iopub.execute_input":"2022-01-09T14:43:51.537578Z","iopub.status.idle":"2022-01-09T14:43:52.086399Z","shell.execute_reply.started":"2022-01-09T14:43:51.537545Z","shell.execute_reply":"2022-01-09T14:43:52.08572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.heatmap(build_meta.corr(),linewidths=.3,annot =True)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:52.087701Z","iopub.execute_input":"2022-01-09T14:43:52.088211Z","iopub.status.idle":"2022-01-09T14:43:52.454023Z","shell.execute_reply.started":"2022-01-09T14:43:52.088141Z","shell.execute_reply":"2022-01-09T14:43:52.453429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(3,1,figsize=(10,10)) \ncolumns = build_meta.drop([\"primary_use\",\"site_id\",\"building_id\"],axis=1).columns\nfor i,col in enumerate(list(columns)):\n    plot = build_meta.boxplot(col, by=\"site_id\", ax=axes.flatten()[i])\n\nplt.tight_layout() \n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:52.455295Z","iopub.execute_input":"2022-01-09T14:43:52.45564Z","iopub.status.idle":"2022-01-09T14:43:53.591642Z","shell.execute_reply.started":"2022-01-09T14:43:52.45561Z","shell.execute_reply":"2022-01-09T14:43:53.591038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reduce memory size func \n\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:53.592881Z","iopub.execute_input":"2022-01-09T14:43:53.593236Z","iopub.status.idle":"2022-01-09T14:43:53.607052Z","shell.execute_reply.started":"2022-01-09T14:43:53.593197Z","shell.execute_reply":"2022-01-09T14:43:53.606211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = reduce_mem_usage(train)\n\nweather_train = reduce_mem_usage(weather_train)\n\nbuild_meta = reduce_mem_usage(build_meta)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:53.608697Z","iopub.execute_input":"2022-01-09T14:43:53.609515Z","iopub.status.idle":"2022-01-09T14:43:54.278444Z","shell.execute_reply.started":"2022-01-09T14:43:53.609475Z","shell.execute_reply":"2022-01-09T14:43:54.277464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"check which features are repeated in the datasets ","metadata":{}},{"cell_type":"code","source":"print(\"Train merge on : \")\nprint(\"build : \",set(train.columns).intersection(set(build_meta.columns)))\nprint(\"weather :\",set(build_meta.columns).intersection(set(weather_train.columns)),end =\"\")\nprint(set(train.columns).intersection(set(weather_train.columns)))  ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:54.279661Z","iopub.execute_input":"2022-01-09T14:43:54.279887Z","iopub.status.idle":"2022-01-09T14:43:54.287568Z","shell.execute_reply.started":"2022-01-09T14:43:54.279858Z","shell.execute_reply":"2022-01-09T14:43:54.286487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge dataset to make an one train dataset \n\ntrain = train.merge(build_meta, on='building_id', how='left')\nalltrain = train.merge(weather_train, on=['site_id','timestamp'], how='left')\n\ndel build_meta,weather_train,train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:43:54.289259Z","iopub.execute_input":"2022-01-09T14:43:54.289626Z","iopub.status.idle":"2022-01-09T14:44:06.813973Z","shell.execute_reply.started":"2022-01-09T14:43:54.289524Z","shell.execute_reply":"2022-01-09T14:44:06.813137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltrain","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:06.819257Z","iopub.execute_input":"2022-01-09T14:44:06.819506Z","iopub.status.idle":"2022-01-09T14:44:06.856971Z","shell.execute_reply.started":"2022-01-09T14:44:06.819475Z","shell.execute_reply":"2022-01-09T14:44:06.856164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltrain[\"timestamp\"] = pd.to_datetime(alltrain[\"timestamp\"])","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:06.858399Z","iopub.execute_input":"2022-01-09T14:44:06.858638Z","iopub.status.idle":"2022-01-09T14:44:10.851037Z","shell.execute_reply.started":"2022-01-09T14:44:06.858608Z","shell.execute_reply":"2022-01-09T14:44:10.849999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#alltrain.info() \n\nnumeric_col = alltrain.select_dtypes(np.number).columns.tolist()\ncategoric_col =alltrain.select_dtypes(\"object\").columns.tolist()\ndate =alltrain.select_dtypes(\"datetime\").columns.tolist()\nprint(\"numeric columns : {} \".format(numeric_col))\nprint(\"categorical columns : {} \".format(categoric_col))\nprint(\"datetime columns : {} \".format(date))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:10.852426Z","iopub.execute_input":"2022-01-09T14:44:10.852706Z","iopub.status.idle":"2022-01-09T14:44:14.244667Z","shell.execute_reply.started":"2022-01-09T14:44:10.852673Z","shell.execute_reply":"2022-01-09T14:44:14.243802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltrain.drop(['building_id','site_id'],axis =1).describe() ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:14.246295Z","iopub.execute_input":"2022-01-09T14:44:14.246611Z","iopub.status.idle":"2022-01-09T14:44:50.083043Z","shell.execute_reply.started":"2022-01-09T14:44:14.246567Z","shell.execute_reply":"2022-01-09T14:44:50.082105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"use = alltrain.groupby(\"primary_use\").meter_reading.mean()\nsns.barplot(y=use.index,x=use)\ndel use","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:50.084455Z","iopub.execute_input":"2022-01-09T14:44:50.084806Z","iopub.status.idle":"2022-01-09T14:44:52.819981Z","shell.execute_reply.started":"2022-01-09T14:44:50.08475Z","shell.execute_reply":"2022-01-09T14:44:52.819114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feet_range = pd.cut(alltrain.square_feet, bins=np.arange(0, 1000000, 100000))\nsquare_meter_read = pd.concat([feet_range,alltrain.meter_reading],axis =1)\ns = square_meter_read.groupby(\"square_feet\").meter_reading.mean()\nsns.barplot(y=s.index,x=s)\ndel feet_range,square_meter_read,s","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:52.821109Z","iopub.execute_input":"2022-01-09T14:44:52.821376Z","iopub.status.idle":"2022-01-09T14:44:53.942051Z","shell.execute_reply.started":"2022-01-09T14:44:52.821341Z","shell.execute_reply":"2022-01-09T14:44:53.941115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:53.943324Z","iopub.execute_input":"2022-01-09T14:44:53.943555Z","iopub.status.idle":"2022-01-09T14:44:54.49864Z","shell.execute_reply.started":"2022-01-09T14:44:53.943527Z","shell.execute_reply":"2022-01-09T14:44:54.497964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#  expanded on over time\nalltrain.rename(columns = {\"timestamp\":\"date\"},inplace= True)\nalltrain[\"month\"] = np.uint8(alltrain[\"date\"].dt.month)\nalltrain[\"week\"] =alltrain[\"date\"].dt.weekofyear\nalltrain[\"weekday\"] = np.uint8(alltrain[\"date\"].dt.weekday)\nalltrain[\"day\"] = np.uint8(alltrain[\"date\"].dt.day)\nalltrain[\"hour\"] = np.uint8(alltrain[\"date\"].dt.hour)\nalltrain[\"weekend\"] = [1 if x in [5,6] else 0 for x in alltrain.day]","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:44:54.501782Z","iopub.execute_input":"2022-01-09T14:44:54.502605Z","iopub.status.idle":"2022-01-09T14:45:25.184451Z","shell.execute_reply.started":"2022-01-09T14:44:54.502567Z","shell.execute_reply":"2022-01-09T14:45:25.18336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltrain['part_of_day'] = (alltrain['date'].dt.hour % 24 + 4) // 4\nalltrain['part_of_day'].replace({1: 'Late Night',\n                      2: 'Early Morning',\n                      3: 'Morning',\n                      4: 'Noon',\n                      5: 'Evening',\n                      6: 'Night'}, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:45:25.186011Z","iopub.execute_input":"2022-01-09T14:45:25.186381Z","iopub.status.idle":"2022-01-09T14:45:28.825727Z","shell.execute_reply.started":"2022-01-09T14:45:25.186336Z","shell.execute_reply":"2022-01-09T14:45:28.824958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reduce_mem_usage(alltrain)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:45:28.826866Z","iopub.execute_input":"2022-01-09T14:45:28.827109Z","iopub.status.idle":"2022-01-09T14:45:40.298114Z","shell.execute_reply.started":"2022-01-09T14:45:28.827082Z","shell.execute_reply":"2022-01-09T14:45:40.29721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for x in [\"weekend\",\"part_of_day\",\"weekday\"]:\n    sns.boxplot(x = x, y= \"meter_reading_log\",data = alltrain,palette = \"pastel\")\n    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:45:40.299687Z","iopub.execute_input":"2022-01-09T14:45:40.30002Z","iopub.status.idle":"2022-01-09T14:46:08.02448Z","shell.execute_reply.started":"2022-01-09T14:45:40.299975Z","shell.execute_reply":"2022-01-09T14:46:08.023868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Features Engineering","metadata":{}},{"cell_type":"markdown","source":"> # 1.2. Imputation","metadata":{}},{"cell_type":"markdown","source":"dealing with the missing values ","metadata":{}},{"cell_type":"code","source":"def findnull(data):\n    nullseries = data.isnull().sum()\n    onlynull= nullseries[nullseries > 0]\n    print(onlynull)\n    sns.barplot(x=onlynull.index , y=onlynull*100/len(data))\n    plt.ylabel(\"PERCENTAGE NULL DATA\")\n    plt.xlabel(\"COLUMN NAME\")\n    plt.xticks(rotation=90)\n    plt.figure(figsize=(20,6))","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:08.025767Z","iopub.execute_input":"2022-01-09T14:46:08.026663Z","iopub.status.idle":"2022-01-09T14:46:08.032826Z","shell.execute_reply.started":"2022-01-09T14:46:08.026621Z","shell.execute_reply":"2022-01-09T14:46:08.032216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"findnull(alltrain)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:08.033948Z","iopub.execute_input":"2022-01-09T14:46:08.034321Z","iopub.status.idle":"2022-01-09T14:46:15.97484Z","shell.execute_reply.started":"2022-01-09T14:46:08.034291Z","shell.execute_reply":"2022-01-09T14:46:15.974287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltrain.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:15.975951Z","iopub.execute_input":"2022-01-09T14:46:15.97635Z","iopub.status.idle":"2022-01-09T14:46:15.983044Z","shell.execute_reply.started":"2022-01-09T14:46:15.976313Z","shell.execute_reply":"2022-01-09T14:46:15.982212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def impute_weather(df):\n    columns = ['air_temperature', 'cloud_coverage', 'dew_temperature','precip_depth_1_hr',\n'sea_level_pressure', 'wind_direction','wind_speed'] \n    for col in columns:\n    \n        imputaion = df.groupby(['site_id','part_of_day','month'])[col].transform('mean')\n        df[col].fillna(imputaion,inplace = True)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:15.984603Z","iopub.execute_input":"2022-01-09T14:46:15.984839Z","iopub.status.idle":"2022-01-09T14:46:15.993422Z","shell.execute_reply.started":"2022-01-09T14:46:15.984812Z","shell.execute_reply":"2022-01-09T14:46:15.992548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"impute_weather(alltrain)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:15.994829Z","iopub.execute_input":"2022-01-09T14:46:15.995076Z","iopub.status.idle":"2022-01-09T14:46:40.997112Z","shell.execute_reply.started":"2022-01-09T14:46:15.995049Z","shell.execute_reply":"2022-01-09T14:46:40.996372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"findnull(alltrain)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:40.99849Z","iopub.execute_input":"2022-01-09T14:46:40.999223Z","iopub.status.idle":"2022-01-09T14:46:48.890356Z","shell.execute_reply.started":"2022-01-09T14:46:40.999186Z","shell.execute_reply":"2022-01-09T14:46:48.889557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltrain.floor_count.fillna(0,inplace = True)  ","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:48.89174Z","iopub.execute_input":"2022-01-09T14:46:48.892057Z","iopub.status.idle":"2022-01-09T14:46:48.960933Z","shell.execute_reply.started":"2022-01-09T14:46:48.892021Z","shell.execute_reply":"2022-01-09T14:46:48.960123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # 2.2 Handling Outliers","metadata":{}},{"cell_type":"code","source":"alltrain[alltrain.meter_reading==0].groupby(\"site_id\").size().sort_values()[-5:]","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:48.962172Z","iopub.execute_input":"2022-01-09T14:46:48.962421Z","iopub.status.idle":"2022-01-09T14:46:51.254837Z","shell.execute_reply.started":"2022-01-09T14:46:48.962389Z","shell.execute_reply":"2022-01-09T14:46:51.253978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltrain.groupby([\"site_id\",\"building_id\"]).meter_reading.sum().sort_values()[:-6:-1]","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:51.256417Z","iopub.execute_input":"2022-01-09T14:46:51.257022Z","iopub.status.idle":"2022-01-09T14:46:52.716245Z","shell.execute_reply.started":"2022-01-09T14:46:51.256976Z","shell.execute_reply":"2022-01-09T14:46:52.715445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px \nfig = px.histogram(np.log1p(alltrain[alltrain['site_id']==13].meter_reading),nbins =10, width=800, height=400)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:46:52.717586Z","iopub.execute_input":"2022-01-09T14:46:52.717934Z","iopub.status.idle":"2022-01-09T14:47:11.366667Z","shell.execute_reply.started":"2022-01-09T14:46:52.717892Z","shell.execute_reply":"2022-01-09T14:47:11.365351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from plotly.subplots import make_subplots\nimport plotly.graph_objects as go\nfig = make_subplots(rows=4, cols=1,subplot_titles=(\"Plot electricity\", \"Plot chilled water\", \"Plot steam\", \"Plot hot water\"))\nmax_cons1 = alltrain[(alltrain['building_id']==1099) & (alltrain.meter == \"electricity\")]\nmax_cons2 = alltrain[(alltrain['building_id']==1099) & (alltrain.meter == \"chilledwater\")]\nmax_cons3 = alltrain[(alltrain['building_id']==1099) & (alltrain.meter == \"steam\")]\nmax_cons4 = alltrain[(alltrain['building_id']==1099) & (alltrain.meter == \"hotwater\")]\nfig.add_trace(\n    go.Scatter(x=max_cons1.date,y=max_cons1.meter_reading),\n    row=1, col=1\n)\nfig.add_trace(\n    go.Scatter(x=max_cons2.date,y=max_cons2.meter_reading),\n    row=2, col=1\n)\nfig.add_trace(\n    go.Scatter(x=max_cons3.date,y=max_cons3.meter_reading),\n    row=3, col=1\n)\nfig.add_trace(\n    go.Scatter(x=max_cons4.date,y=max_cons4.meter_reading),\n    row=4, col=1\n)\nfig.update_layout(height=600, width=800, title_text=\"Side By Side Subplots\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:47:11.369235Z","iopub.execute_input":"2022-01-09T14:47:11.369622Z","iopub.status.idle":"2022-01-09T14:47:23.386037Z","shell.execute_reply.started":"2022-01-09T14:47:11.369569Z","shell.execute_reply":"2022-01-09T14:47:23.385112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # 3.2 Encoding ","metadata":{}},{"cell_type":"markdown","source":"( convert the categorical feauter to numerical )","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\n\nalltrain['primary_use'] = le.fit_transform(alltrain['primary_use']).astype(np.int8)","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:47:23.387836Z","iopub.execute_input":"2022-01-09T14:47:23.388229Z","iopub.status.idle":"2022-01-09T14:47:30.328777Z","shell.execute_reply.started":"2022-01-09T14:47:23.388176Z","shell.execute_reply":"2022-01-09T14:47:30.327695Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # 4.2 one-hot encoding data ","metadata":{}},{"cell_type":"code","source":"pd.get_dummies(alltrain, columns=[\"primary_use\"]).head()","metadata":{"execution":{"iopub.status.busy":"2022-01-09T14:47:30.330497Z","iopub.execute_input":"2022-01-09T14:47:30.331058Z","iopub.status.idle":"2022-01-09T14:47:35.350972Z","shell.execute_reply.started":"2022-01-09T14:47:30.331006Z","shell.execute_reply":"2022-01-09T14:47:35.350141Z"},"trusted":true},"execution_count":null,"outputs":[]}]}