{"cells":[{"metadata":{},"cell_type":"markdown","source":"# <a id=\"0\">Table Of Contents</a>\n- <a href='#1'>1. Imports</a>\n- <a href=\"#2\">2 Read In Data</a>\n- <a href=\"#3\">3. Glipmse of Data</a>\n- <a href=\"#4\">4. Reducing Memory Size</a>\n- <a href=\"#5\">5. Exploratory Data Analysis</a>\n    - <a href=\"#5-1\">5.1 Data Description</a>\n    - <a href=\"#5-2\">5.2 Examine the Distribution of Target Variable</a>\n    - <a href=\"#5-3\" >5.3 Missing Data and Zeroes Visualised</a>\n    - <a href=\"#5-4\">5.4 Examining Missing Values</a>\n    - <a href=\"#5-5\">5.5 Column Types</a>\n    - <a href=\"#5-6\">5.6 Correlations</a>\n- <a href=\"#6\">6. Plotting</a>\n- <a href=\"#7\">7. Simple Single Series Analysis</a>\n- <a href=\"#8\">8. Outlier Distribution</a>\n    - <a href=\"#8-1\">8.1 Group Data on Daily Basis</a>\n    - <a href=\"#8-2\">8.2 Aggregating Data For Buildings</a>\n    - <a href=\"#8-3\">8.3 Identifying Outliers</a>\n- <a href=\"#9\">9. Feature Engineering and Modelling</a>\n- <a href=\"#10\">10. Pickling Train and Test Data</a>\n- <a href=\"#11\">11. Creating Features</a>\n- <a href=\"#12\">12. Variables Encoding</a>"},{"metadata":{},"cell_type":"markdown","source":"  # <a id=\"1\">1. Imports </a>\n  #### <a href=\"#0\">Top</a>"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Importing libraries\nimport numpy as np\nimport pandas as pd\nimport gc\n\nimport warnings\nwarnings.filterwarnings('ignore')\n#matplotlib and seaborn for plotting\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.patches as patches\n\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\npd.set_option('max_columns',150)\n\npy.init_notebook_mode(connected=True)\nfrom plotly.offline import init_notebook_mode,iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n \nimport os,random, math,psutil, pickle","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"2\">2. Read in Data </a>    \n###### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input/ashrae-energy-prediction\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nroot=\"../input/ashrae-energy-prediction/\"\ntrain_df=pd.read_csv(root+\"train.csv\")\nwtrain_df=pd.read_csv(root+\"weather_train.csv\")\ntest_df=pd.read_csv(root+\"test.csv\")\nwtest_df=pd.read_csv(root+\"weather_test.csv\")\nbuilding_df=pd.read_csv(root+\"building_metadata.csv\")\nsample_df=pd.read_csv(root+\"sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#changing the datatype of timestamp column of training set\ntrain_df[\"timestamp\"]=pd.to_datetime(train_df[\"timestamp\"], format=\"%Y-%m-%d %H:%M:%S\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"3\">3 Glimpse of Data</a>  \n###### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Size of train_df data is\", train_df.shape)\nprint(\"Size of wtrain_df data is\",wtrain_df.shape)\nprint(\"Size of wtest_df data is\",wtest_df.shape)\nprint(\"Size of test_df data is\",test_df.shape )\nprint(\"Size of building_df data is\", building_df.shape)\nprint(\"Size of sample_df data is\", sample_df.shape)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"4\"> 4. Reducing Memory Size </a>\n###### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Calculating Memory utilization in MB's\nprint(\"Memory utilization train_df data is\", train_df.memory_usage().sum()/1024**2,\"MB\")\nprint(\"Memory utilization of wtrain_train_df data is\",wtrain_df.memory_usage().sum()/1024**2,\"MB\")\nprint(\"Memory utilization of wtest_train_df data is\",wtest_df.memory_usage().sum()/1024**2,\"MB\")\nprint(\"Memory utilization of test_df data is\",test_df.memory_usage().sum()/1024**2,\"MB\")\nprint(\"Memory utilization of building_metadata_df data is\",building_df.memory_usage().sum()/1024**2,\"MB\")\nprint(\"Memory utilization of sample_submission_df data is\",sample_df.memory_usage().sum()/1024**2,\"MB\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function to reduce the dataframe size\n\ndef reduce_mem(df, verbose=True):\n    numerics=['int16','int32','int64','float16','float32','float64']\n    start_mem=df.memory_usage().sum()/1024**2\n    for col in df.columns:\n        col_type=df[col].dtypes\n        if col_type in numerics:\n            c_min=df[col].min()\n            c_max=df[col].max()\n            if str(col_type)[:3]==\"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col]=df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col]=df[col].astype(np.int16)\n                elif c_min >np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col]=df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col]=df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col]=df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col]=df[col].astype(np.float32)\n                elif c_min > np.finfo(np.float64).min and c_max < np.finfo(np.float64).max:\n                    df[col]=df[col].astype(np.float64)\n    end_mem=df.memory_usage().sum()/1024**2\n    if verbose: print(\"Memory usage decreased to {:5.2f} MB ({:.1f}% reduction)\".format(end_mem,100*(start_mem-end_mem)/start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=reduce_mem(train_df)\nwtrain_df=reduce_mem(wtrain_df)\nwtest_df=reduce_mem(wtest_df)\ntest_df=reduce_mem(test_df)\nbuilding_df=reduce_mem(building_df)\nsample_df=reduce_mem(sample_df)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"5\">5. Exploratory Data Analysis</a>\n###### <a href=\"#0\">Top</a>"},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"5-1\">5.1 Data Description</a>\n###### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Column Name of Dataframes\ndef colname(df):\n    name=df.columns\n    return name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.columns.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wtrain_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wtrain_df.columns.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wtest_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"wtest_df.columns.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_df.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_df.columns.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"5-2\">5.2 Examine the Distribution of Target Variable</a>\n##### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# target is meter_reading\n#individual subplots for {0: electricity, 1: chilledwater, 2: steam, 3: hotwater}\ngrouped=train_df.groupby(['meter'])[\"meter_reading\"]\nncols=2\nnrows=int(np.ceil(grouped.ngroups/ncols))\nfig,axes=plt.subplots(nrows=nrows,ncols=ncols,figsize=(30,15))\nfor (key,ax) in zip(grouped.groups.keys(),axes.flatten()):\n    if key==0:\n        label=\"Electricity\"\n    elif key==1:\n        label=\"Chilled Water\"\n    elif key==2:\n        label=\"Steam\"\n    elif key==3:\n        label=\"Hot Water\"\n    grouped.get_group(key).plot(ax=ax, label=label)\n    ax.legend(loc=\"upper left\")\n    ax.set_xlabel(\"Meter Reading\")\n    ax.set_ylabel(\"Frequency\")\nplt.show();\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking distribution of meterreading as a whole\nplt.figure(figsize=(15,8))\nplt.plot(train_df[\"meter_reading\"])\nplt.title(\"Distribution of Meter Reading as Whole\")\nplt.xlabel(\"Meter Reading\")\nplt.ylabel(\"Frequency\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"meter_reading\"].plot(kind=\"hist\",\n                              bins=30,\n                              figsize=(30,8),\n                              title=\"Distribution of Target Variable (meter_reading)\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"5-3\">5.3 Missing Data and Zeroes Visualised</a>\n###### <a href=\"#0\">Top</a>\n"},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":" #Goal: for each building and each meter type,visualise where target is missing and where target is zero vs time\n#Load Data\ntrain=train_df.set_index([\"timestamp\"])\n#Plot missing values per building per meter type\n\nf,a=plt.subplots(1,4,figsize=(20,30))\nfor meter in np.arange(4):\n    df=train[train.meter==meter].copy().reset_index()\n    df['timestamp']=pd.to_timedelta(df.timestamp).dt.total_seconds()/3600\n    df.timestamp=df.timestamp.astype(int)\n    #print(df.timestamp.min())\n    df.timestamp-=df.timestamp.min()\n    #print(df.timestamp.max())\n    missmap=np.empty((1449,df.timestamp.max()+1))\n    #print(missmap.shape)\n    missmap.fill(np.nan)\n    for l in df.values:\n        if l[2]!=meter:continue\n        missmap[int(l[1]),int(l[0])]=0 if l[3]==0 else 1\n    a[meter].set_title(f'meter{meter}')\n    sns.heatmap(missmap,cmap='Paired',ax=a[meter],cbar=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"5-4\">5.4 Examining Missing Values</a>\n##### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking missing values for train_df\ntotal=train_df.isnull().sum().sort_values(ascending=False)\npercent=(train_df.isnull().sum()/train_df.isnull().count()*100).sort_values(ascending=False)\nmissing_train=pd.concat([total,percent], axis=1, keys=[\"Total\",\"Percentage\"])\nmissing_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking missing values for wtrain_df\ntotal=wtrain_df.isnull().sum().sort_values(ascending=False)\npercent=(wtrain_df.isnull().sum()/wtrain_df.isnull().count()*100).sort_values(ascending=False)\nmissing_wtrain=pd.concat([total,percent],axis=1,keys=[\"Total\",\"Percent\"])\nmissing_wtrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking missing values for wtest_df\ntotal=wtest_df.isnull().sum().sort_values(ascending=False)\npercent=(wtest_df.isnull().sum()/wtest_df.isnull().count()*100).sort_values(ascending=False)\nmissing_wtest=pd.concat([total,percent],axis=1,keys=[\"Total\",\"Percent\"])\nmissing_wtest","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking missing values in building_df\ntotal=building_df.isnull().sum().sort_values(ascending=False)\npercent=(building_df.isnull().sum()/building_df.isnull().count()*100).sort_values(ascending=False)\nmissing_building=pd.concat([total,percent],axis=1,keys=[\"Total\",\"Percent\"])\nmissing_building","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"5-5\">5.5 Column Types</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Columns in train_df\nprint(f'Column types in train_df::\\n{train_df.dtypes.value_counts()}')\n\n#Column in wtrain_df\nprint(f'\\nColumn types in wtrain_df::\\n{wtrain_df.dtypes.value_counts()}')\n\n#Column in wtest_df\nprint(f'\\nColumn types in wtest_df::\\n{wtest_df.dtypes.value_counts()}')\n\n#Column in wtrain_df and building_df\nprint(f'\\nColumn types in building_df::\\n{building_df.dtypes.value_counts()}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of unique classes in each categorical/object column in train_df\nprint(f'No. of unique classes in column of object type in train_df:\\n{train_df.select_dtypes(\"object\").apply(pd.Series.nunique,axis=0)}')\n\n#Number of unique classes in each categorical/object column in wtrain_df\nprint(f'\\nNo. of unique classes in column of object type in wtrain_df:\\n{wtrain_df.select_dtypes(\"object\").apply(pd.Series.nunique,axis=0)}')\n\n#Number of unique classes in each categorical/object column in wtest_df\nprint(f'\\nNo. of unique classes in column of object type in wtest_df:\\n{wtest_df.select_dtypes(\"object\").apply(pd.Series.nunique,axis=0)}')\n\n#Number of unique classes in each categorical/object column in building_df\nprint(f'\\nNo. of unique classes in column of object type in building_df:\\n{building_df.select_dtypes(\"object\").apply(pd.Series.nunique,axis=0)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"5-6\">5.6 Correlations</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#finding the correleations in train_df\n\ncor=train_df.corr()\nprint(cor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting the coorelations in seaborn\n\nplt.figure(figsize=(20,8))\nsns.heatmap(cor, annot=True,cmap='RdYlBu', vmax=0.6)\nplt.title(\"Correlation Heatmap\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"6\">6. Plotting</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Finding unique buildings in train_df\nprint(f'No of unique buildings in the data:{train_df.building_id.nunique()}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting the distribution of meter_reading using hisogram\ntrain_df.meter_reading.hist(figsize=(7,5))\nplt.xlabel(\"Meter Reading\")\nplt.ylabel(\"Frequency\")\nplt.title(\"Histogram Meter Reading\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting on wtrain_df and wtest_df\n#wtrain_df.head()\n#Plotting air_temperature distribution of wtrain_df and wtest_df\n\ndef plot_dist(col, kde=True):\n    ''' plot dist curve on wtrain_df ad wtest_df for a given column name'''\n    \n    fig,ax=plt.subplots(figsize=(20,5))\n    sns.distplot(wtrain_df[col].dropna(), color=\"green\", ax=ax).set_title(f'Distribution of {col}',fontsize=16)\n    sns.distplot(wtest_df[col].dropna(),ax=ax,color=\"purple\").set_title(f'Distribution of {col}',fontsize=16)\n    plt.xlabel(col, fontsize=16)\n    plt.legend(['train','test'])\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"name=list(colname(wtrain_df))\n#print(name)\n#Plottting distribution of air_temperature\nplot_dist(\"air_temperature\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting Disribution of cloud_coverage\nplot_dist(\"cloud_coverage\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting dew_temperature\nplot_dist('dew_temperature')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting sea_level_pressure\nplot_dist('sea_level_pressure')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting distribution of wind_direction\nplot_dist('wind_direction')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"7\">7. Simple Single Series Analysis</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"from statsmodels.tsa.seasonal import seasonal_decompose\nfrom statsmodels.tsa.stattools import adfuller","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Check the stationarity of the timeseries\n\ndef check_stn(ts):\n    #Determining rolling statistics\n    rolmean=ts.rolling(window=12, center=False).mean()\n    rolstd=ts.rolling(window=12,center=False).std()\n    \n    #Plot rolling statistics\n    plt.figure(figsize=(20,8))\n    orig=plt.plot(ts, label=\"Original\")\n    mean=plt.plot(rolmean, label=\"Rolling Mean\")\n    std=plt.plot(rolstd,label=\"Rolling Std\")\n    plt.xlabel(\"timestamp\")\n    plt.ylabel(\"meter_reading\")\n    plt.legend(loc=\"best\")\n    plt.title(\"Plotting Rolling Mean and Rolling Std\")\n    plt.show()\n    \n    #Perform Dickey-Fuller Test\n    print(\"Result of Dickey-Fuller Test\")\n    ts=ts.iloc[:,0].values\n    dftest=adfuller(ts, autolag=\"AIC\")\n    dfoutput=pd.Series(dftest[0:4],index=[\"Test Statistics\",\"p-value\",\"No.Of Lags Used\",\"No. Of Observations Used\"])\n    for key, value in dftest[4].items():\n        dfoutput[f'Critical Value of {key}']=value\n    print(dfoutput)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ts=pd.DataFrame(train_df.groupby(['timestamp']).meter_reading.sum())\ncheck_stn(ts)\n#Note: For TimeSeries to be Stationary value of Test Statistics has to be less then the Critical Value.\n#Conclusion: This is a stationary time series","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Decomposition of TimeSeries\nimport statsmodels.api as sm\n#multiplicative \nres=sm.tsa.seasonal_decompose(ts.values,period=12, model=\"multiplicative\")\nfig=res.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Additive\nres=sm.tsa.seasonal_decompose(ts.values,period=12, model=\"additive\")\nfig=res.plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"8\">8. Outlier Distribution</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_mean=train_df.groupby(['timestamp'])[\"meter_reading\"].mean()\nplt.figure(figsize=(10,5))\nplt.plot(y_mean)\nplt.xlabel(\"timestamp\")\nplt.ylabel(\"Mean Of meter_reading\")\nplt.title(\"Plotting Mean of meter_reading and timestamp\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.plot(y_mean.rolling(window=10).std())\nax=plt.axhline(y=0.009,color=\"red\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,5))\nplt.plot(y_mean.rolling(window=10).std())\nplt.axhline(y=0.009,color=\"red\")\nplt.axhspan(0, 905, color='green', alpha=0.1)\nplt.axhspan(906,1505, color='red', alpha=0.1);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"8-1\">8.1 Group Data on Daily Basis</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"meter\"]=pd.Categorical(train_df.meter).rename_categories({0:\"Electricity\",1:\"Chilled_Water\",2:\"Steam\",3:\"Hot_Water\"})\ndaily_train=train_df.copy()\ndaily_train['date']=daily_train.timestamp.dt.date\ndaily_train=daily_train.groupby([\"date\",\"building_id\",\"meter\"]).sum()\ndaily_train.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"8-2\">8.2 Aggregating Data For Buildings</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"daily_train_agg=daily_train.groupby([\"date\",\"meter\"]).agg([\"sum\",\"mean\",\"idxmax\",\"max\"])\ndaily_train_agg=daily_train_agg.reset_index()\nlevel_0=daily_train_agg.columns.droplevel(0)\nlevel_1=daily_train_agg.columns.droplevel(1)\nlevel_0=['' if x=='' else '_'+ x for x in level_0]\ndaily_train_agg.columns=level_1 + level_0\ndaily_train_agg.rename_axis(None,axis=1)\ndaily_train_agg.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,8))\nax=sns.lineplot(x=\"date\",y=\"meter_reading_sum\", data=daily_train_agg, hue=\"meter\")\nax.set_title(\"Total Energy per day\")\nax.set_xlabel(\"Date\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,8))\nax=sns.lineplot(x=\"date\",y=\"meter_reading_max\", data=daily_train_agg, hue=\"meter\")\nax.set_title(\"Maximum Energy per day\")\nax.set_xlabel(\"Date\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"8-3\">8.3 Identifying Outliers</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"daily_train_agg[\"building_id_max\"]=[x[1] for x in daily_train_agg['meter_reading_idxmax'] ]\ndaily_train_agg.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# No of days a building has the maximum electricity consumption\n\nout_elec=daily_train_agg[daily_train_agg[\"meter\"]==\"Electricity\"][\"building_id_max\"].value_counts()\nprint(f'Number of time buildings having maximum consumption of Electricity:\\n{out_elec}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Maximum electricity is consumed by only 6 buildings\nplt.figure(figsize=(20,8))\ndaily_train_elec=daily_train_agg[daily_train_agg['meter']==\"Electricity\"]\n#print(f'Data type of building_id_max is :: {daily_train_elec.building_id_max.dtype}') #Note: It has to be categorical\ndaily_train_elec[\"building_id_max\"]=pd.Categorical(daily_train_elec[\"building_id_max\"])\n\nax=sns.scatterplot(x=\"date\",y=\"meter_reading_max\",hue=\"building_id_max\", data=daily_train_elec,alpha=0.8)\nax.set_xlim((daily_train_elec[\"date\"].min()+pd.to_timedelta(-1,unit=\"M\"),daily_train_elec[\"date\"].max()+pd.to_timedelta(2,unit=\"M\")))\nplt.xlabel(\"Date\", fontsize=16)\nplt.ylabel(\"meter_reading_max\",fontsize=16)\nplt.title(\"Building Having High Consumption of Electricity\", fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of days the buidling having higher consumption for ChilledWater\ndaily_train_chilled=daily_train_agg[daily_train_agg[\"meter\"]==\"Chilled_Water\"]\ndaily_train_chilled[\"building_id_max\"]=pd.Categorical(daily_train_chilled[\"building_id_max\"])\nbld_count=daily_train_chilled[\"building_id_max\"].value_counts()\nprint(f'No of days building has high demand for Chilled_Water:\\n{bld_count}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting the buildings meter_reading for building having demand of Chilled_Water\nplt.figure(figsize=(20,8))\nax=sns.scatterplot(x=\"date\",y=\"meter_reading_max\",hue=\"building_id_max\", data=daily_train_chilled)\nplt.xlim((daily_train_chilled[\"date\"].min()+pd.to_timedelta(-2,unit=\"M\"), daily_train_chilled[\"date\"].max()+pd.to_timedelta(2,unit=\"M\")))\nplt.xlabel(\"Date\",fontsize=16)\nplt.ylabel(\"meter_reading\",fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#No of days building having high demand for Steam\ndaily_train_steam=daily_train_agg[daily_train_agg[\"meter\"]==\"Steam\"]\ndaily_train_steam[\"building_id_max\"]=pd.Categorical(daily_train_steam[\"building_id_max\"])\nsteam_count=daily_train_steam[\"building_id_max\"].value_counts()\nprint(f'Number of days building having high demands for Steam among all buildings:\\n{steam_count}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting the meter_reading for buildings having high demand for Steam for each date.\nplt.figure(figsize=(20,8))\nax=sns.scatterplot(x=\"date\",y=\"meter_reading_max\", data=daily_train_steam, hue=\"building_id_max\")\nplt.xlim((daily_train_steam[\"date\"].min()+pd.to_timedelta(-2,unit=\"M\"),daily_train_steam[\"date\"].max()+pd.to_timedelta(2,unit=\"M\")))\nplt.xlabel(\"Date\", fontsize=16)\nplt.ylabel(\"meter_reading_max\",fontsize=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#No of days each building have high demand for Hot_Water\n\ndaily_train_hw=daily_train_agg[daily_train_agg[\"meter\"]=='Hot_Water']\ndaily_train_hw[\"building_id_max\"]=pd.Categorical(daily_train_hw[\"building_id_max\"])\nhw_count=daily_train_hw[\"building_id_max\"].value_counts()\nprint(f'No of days each building has high demand for Hot_Water:\\n{hw_count}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Plotting meter_reading for each building datewise having high demand for Hot_Water\nplt.figure(figsize=(20,8))\nax=sns.scatterplot(x=\"date\",y='meter_reading_max',data=daily_train_hw,hue=\"building_id_max\")\nplt.xlabel(\"Date\", fontsize=16)\nplt.ylabel(\"meter_reading_max\",fontsize=16)\nplt.title(\"Plotting meter_reading_max of each builiding having high demand for Hot_water\")\nplt.xlim((daily_train_hw[\"date\"].min()+pd.to_timedelta(-2,unit=\"M\"),daily_train_hw[\"date\"].max()+pd.to_timedelta(2,unit=\"M\")))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"9\">9. Feature Engineering and Modelling</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nfrom sklearn import metrics\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"9-1\">9.1 Merging buidling_df and train_df</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"wtrain_df[\"timestamp\"]=pd.to_datetime(wtrain_df[\"timestamp\"])\nwtest_df[\"timestamp\"]=pd.to_datetime(wtest_df[\"timestamp\"])\ntrain_df[\"timestamp\"]=pd.to_datetime(train_df[\"timestamp\"])\ntest_df[\"timestamp\"]=pd.to_datetime(test_df[\"timestamp\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df=train_df[[\"building_id\"]] ##Note this syntax creates dataframe from a single column of DataFrame instead of Series\ntemp_df=temp_df.merge(building_df,how=\"left\",on=\"building_id\")\ndel temp_df[\"building_id\"]\ntrain_df=pd.concat([train_df,temp_df],axis=1)\ntrain_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df=train_df[['building_id']]\ntemp_df=temp_df.merge(building_df,on=[\"building_id\"],how=\"left\")\ndel temp_df[\"building_id\"]\ntest_df=pd.concat([test_df,temp_df],axis=1)\ntest_df.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## <a id=\"9-2\">9.2 Merging wtrain_df and train_df</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merging wtest_df to test_df\ntemp_df=train_df[[\"site_id\",\"timestamp\"]]\ntemp_df=temp_df.merge(wtrain_df,on=[\"site_id\",\"timestamp\"],how=\"left\")\ndel temp_df[\"site_id\"], temp_df[\"timestamp\"]\ntrain_df=pd.concat([train_df,temp_df],axis=1)\ntrain_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merging wtest_df to test_df\ntemp_df=test_df[[\"site_id\",\"timestamp\"]]\ntemp_df=temp_df.merge(wtest_df,on=[\"site_id\",\"timestamp\"],how=\"left\")\ndel temp_df[\"site_id\"], temp_df[\"timestamp\"]\ntest_df=pd.concat([test_df,temp_df],axis=1)\ntest_df.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"10\">10. Pickling Of Train and Test Data</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Saving the train_df and test_df as their pickle file\ntrain_df.to_pickle(\"train_df.pickle\")\ntrain_df.to_pickle(\"test_df.pickle\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ndel train_df, test_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#reading train and test data from pickle file\ntrain_df=pd.read_pickle(\"train_df.pickle\")\ntest_df=pd.read_pickle(\"test_df.pickle\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"11\">11. Features</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"age\"]=train_df[\"year_built\"].max() - train_df[\"year_built\"] + 1\ntest_df[\"age\"]=test_df[\"year_built\"].max() - test_df[\"year_built\"] + 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"12\">12. Variables Encoding</a>\n#### <a href=\"#0\">Top</a>"},{"metadata":{"trusted":true},"cell_type":"code","source":"le=LabelEncoder()\ntrain_df[\"primary_use\"]=le.fit_transform(train_df[\"primary_use\"]).astype('int8')\ntest_df[\"primary_use\"]=le.fit_transform(test_df[\"primary_use\"]).astype('int8')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Number of unique values in primary_use for train_df:\\n {pd.unique(train_df[\"primary_use\"]).ravel()}\\n')\nprint(f'Number of unique values in primary_use for test_df:\\n {pd.unique(test_df[\"primary_use\"]).ravel()}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# <a id=\"13\">13. Handling Missing Values</a>\n#### <a href=\"#0\">Top</a>"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}