{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport dask\nimport dask.dataframe as dd\nfrom dask_ml.model_selection import train_test_split\nimport dask_xgboost\nimport matplotlib.pyplot as plt\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## This notebook will utilize Dask to perform dataframe operation to reduce RAM usage and possibly killing the kernel. Then apply seaborn to visualize some of the features."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"bld_meta=dd.read_csv('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\nweather_train=dd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_train.csv',parse_dates=['timestamp'])\nweather_test=dd.read_csv('/kaggle/input/ashrae-energy-prediction/weather_test.csv',parse_dates=['timestamp'])\ntrain_df=dd.read_csv('/kaggle/input/ashrae-energy-prediction/train.csv', parse_dates=['timestamp'])\ntest_df=dd.read_csv('/kaggle/input/ashrae-energy-prediction/test.csv',parse_dates=['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_timestamp(df, time_col):\n    df['year']=df[time_col].dt.year\n    df['day']=df[time_col].dt.day\n    df['weekday']=df[time_col].dt.weekday_name\n    df['month']=df[time_col].dt.month_name()\n    df['hour']=df[time_col].dt.hour\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=convert_timestamp(train_df, 'timestamp')\ntest_df=convert_timestamp(test_df, 'timestamp')\n\nweather_train=convert_timestamp(weather_train, 'timestamp')\nweather_test=convert_timestamp(weather_test, 'timestamp')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['meter']=train_df['meter'].mask(train_df['meter']==0, 'eletricity').mask(train_df['meter']==1, 'chilledwater').mask(train_df['meter']==2,'steam').mask(train_df['meter']==3, 'hotwater')\ntest_df['meter']=test_df['meter'].mask(test_df['meter']==0, 'eletricity').mask(test_df['meter']==1, 'chilledwater').mask(test_df['meter']==2,'steam').mask(test_df['meter']==3, 'hotwater')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(bld_meta.isnull().sum()/len(bld_meta)).compute()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"(weather_train.isnull().sum()/len(weather_train)).compute()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bld_meta=bld_meta.drop(columns=['year_built', 'floor_count'], axis=1)\nweather_train=weather_train.drop(columns=['cloud_coverage','precip_depth_1_hr'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## merge training data with meta data. Drop year_built and floor-count from meta data\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"bld_weather=dd.merge(weather_train, bld_meta, \n                     on='site_id')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bld_cols=['air_temperature',\n       'dew_temperature', 'sea_level_pressure',\n       'wind_direction', 'wind_speed', 'year','month', 'day', 'weekday',\n       'hour', 'building_id', 'primary_use', 'square_feet']\n\ntrain_cols=['building_id', 'meter', 'meter_reading','year','month','day','hour']\ntrain_merged_df=dd.merge(train_df[train_cols], bld_weather[bld_cols], how='left', \n                         on=['building_id','year','month','day','hour'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Using Dask scheduler overwelms the CPU because it writes to disk to keep a low-memory footprint. But if there's enough RAM then switch it to pandas DF."},{"metadata":{"trusted":true},"cell_type":"code","source":"# takes a while to run\ntrain_pd_df=train_merged_df.compute()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## select numerical variables to look at correlation and distribution"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_cols=['dew_temperature','air_temperature', 'sea_level_pressure','hour',\n       'wind_direction', 'wind_speed', 'square_feet','meter_reading']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corr_df=train_pd_df[num_cols].corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mask = np.zeros_like(corr_df, dtype=np.bool)\nmask[np.triu_indices_from(mask)] = True\n\n# Set up the matplotlib figure\nf, ax = plt.subplots(figsize=(11, 9))\n\n# Generate a custom diverging colormap\ncmap = sns.diverging_palette(220, 10, as_cmap=True)\n\n# Draw the heatmap with the mask and correct aspect ratio\nsns.heatmap(corr_df, mask=mask, cmap=cmap, vmin=-1, vmax=1, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5})\ndel corr_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_pd_df['log_meter']=np.log(train_pd_df['meter_reading']+1)\nsns.distplot(train_pd_df['log_meter'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"select_cols=['wind_speed', 'square_feet','log_meter']\ndf=train_pd_df.sample(frac=0.1, random_state=22) #sample 10% so the notebook can run quickly, especially the pairplot\ndf_num=df[select_cols]\nprint(df_num.shape)\nprint(df_num.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# takes a loong time to run\nsns.pairplot(df_num) \ndel df_num","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot the histograms for each primary use group\nselect_cols=['primary_use','wind_speed', 'square_feet','log_meter']\ndf2=train_pd_df[select_cols]\nbins = np.arange(0,18, 3)\ng = sns.FacetGrid(df2, col=\"primary_use\",col_wrap=4, height=4)\ng = g.map(plt.hist, \"log_meter\", bins=bins)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## look at the categorical variables. Aggregated statistic plots can take a long time so use the sampled DF."},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_cols=['primary_use','weekday','month','log_meter']\ndf_cat=df[cat_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.catplot(x=\"log_meter\",y=\"weekday\",kind='violin',data=df_cat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.catplot(x=\"log_meter\",y=\"primary_use\",kind='violin',data=df_cat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.catplot(x=\"log_meter\",y=\"month\",kind='violin',data=df_cat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}