{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import stats\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/ashrae-energy-prediction/test.csv',parse_dates=['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/ashrae-energy-prediction/train.csv',parse_dates=['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv',parse_dates=['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv',parse_dates=['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_metadata = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/ashrae-energy-prediction/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = reduce_mem_usage(test)\ntrain = reduce_mem_usage(train)\nweather_test = reduce_mem_usage(weather_test)\nweather_train = reduce_mem_usage(weather_train)\nbuilding_metadata = reduce_mem_usage(building_metadata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = weather_train.groupby('site_id').apply(lambda x: x.set_index('timestamp').interpolate(method='time',limit_direction='both').reset_index()).drop(columns='site_id').reset_index().drop(columns='level_1')\nweather_test = weather_test.groupby('site_id').apply(lambda x: x.set_index('timestamp').interpolate(method='time',limit_direction='both').reset_index()).drop(columns='site_id').reset_index().drop(columns='level_1')\n\nweather_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nweather_train['cloud_coverage'] = weather_train['cloud_coverage'].round()\nweather_train['wind_direction'] = weather_train['wind_direction'].round()\nweather_train['precip_depth_1_hr'] = weather_train['precip_depth_1_hr'].round()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.groupby('site_id').mean().sort_values(by='cloud_coverage')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(building_metadata, on='building_id', how='left')\ntest = test.merge(building_metadata, on='building_id', how='left')\n\ntrain = train.merge(weather_train, on=['site_id', 'timestamp'], how='left')\ntest = test.merge(weather_test, on=['site_id', 'timestamp'], how='left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del weather_train\ndel weather_test\ndel building_metadata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['aday']=train['timestamp'].dt.day\ntrain['amonth']=train['timestamp'].dt.month\ntrain['ayear']=train['timestamp'].dt.year\ntrain['aquarter']=train['timestamp'].dt.quarter\ntrain['adayofweek']=train['timestamp'].dt.dayofweek\ntrain['adayofyear']=train['timestamp'].dt.dayofyear\ntrain['adate']=train['timestamp'].dt.date\ntrain['ahour']=train['timestamp'].dt.hour\n\ntest['aday']=test['timestamp'].dt.day\ntest['amonth']=test['timestamp'].dt.month\ntest['ayear']=test['timestamp'].dt.year\ntest['aquarter']=test['timestamp'].dt.quarter\ntest['adayofweek']=test['timestamp'].dt.dayofweek\ntest['adayofyear']=test['timestamp'].dt.dayofyear\ntest['adate']=test['timestamp'].dt.date\ntest['ahour']=test['timestamp'].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test['source']='test'\ntrain['source']='train'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = reduce_mem_usage(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['ayear'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Univariate Analysis"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"sns.countplot(train['meter'])\nplt.show() # The meter count varies from meter to meter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['site_id'].value_counts().sort_index().plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,6))\ntrain['primary_use'].value_counts().plot(kind='bar')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,6))\nsns.distplot(train['square_feet'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(18,6))\ntrain['year_built'].value_counts().sort_index(ascending= False).plot(kind='bar')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['year_built'].value_counts().to_frame().iloc[:2013]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(12,6))\ntrain['floor_count'].value_counts().plot(kind='bar')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.boxplot(train['air_temperature'].dropna(), showfliers=True)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train['air_temperature'].dropna())\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d= train['cloud_coverage'].value_counts().sort_index().to_frame()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"(d).plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d.T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.stripplot(d.index,d.cloud_coverage)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"sns.boxplot(train['dew_temperature'].dropna(), showfliers=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['precip_depth_1_hr'].dropna().value_counts().sort_index().plot.density()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"sns.boxplot(train['precip_depth_1_hr'].dropna(), showfliers = False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m=train['precip_depth_1_hr'].dropna().value_counts().sort_index()\nm\n# Suspecting precip_depth_1_hr values under 0 to be incorrect entries","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nplt.figure(figsize=(24,7))\nsns.barplot(m.index,m.values)\nplt.xticks(rotation=85)\nplt.ylim((0,553352))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(m.to_frame()['precip_depth_1_hr'])\nplt.ylim((0,0.0000004))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\nsns.boxplot(train['sea_level_pressure'].dropna(), showfliers=True)\nplt.show() # varies from 995 to 1034","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['sea_level_pressure'].dropna().value_counts().sort_index().plot.hist(bins=50)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.boxplot(train['wind_direction'].dropna(), showfliers=True)\nplt.show() # varies from 0-360","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"sns.boxplot(train['wind_speed'].dropna(), showfliers=True)\nplt.show() # the plot is right skewed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Bivariate Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.boxplot(train.groupby('building_id')['meter_reading'].mean(), showfliers=False)\nplt.show() # the plot is right skewed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''plt.figure(figsize=(24,7))\n#sns.boxplot(train['building_id'], train['meter_reading'], showfliers=False)\nsns.barplot(train['building_id'], train['meter_reading'])\nplt.xticks(rotation=90)\nplt.show()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\nsns.boxplot(train['site_id'], train['meter_reading'], showfliers=False)\nplt.show() # site: 5,8, 12,3,11 are the most efficient sites","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['amonth'].value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\nsns.boxplot(train['site_id'], train['air_temperature'], showfliers=True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby(['site_id'])['air_temperature'].mean().sort_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m=train.groupby(['site_id','building_id'])['air_temperature'].mean()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m[0][0]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"d={}\nfor i in range(16):\n    n = np.array(m[i])\n    d.update({i:[i,np.mean(n),np.std(n)]})\n    print('For site_id = ', i,', mean value is: ',np.mean(n), 'standard dev value is: ', np.std(n))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train.loc[train.site_id == 5, 'air_temperature'].dropna())\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(d, index=['site_index','mean', 'std']).T.sort_values(by='mean').iloc[:,[0,1]].set_index('site_index').plot(kind='bar')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(d, index=['site_index','mean', 'std']).T.sort_values(by='mean').iloc[:,[0,1]].set_index('site_index').plot(kind='bar')\npd.DataFrame(d, index=['site_index','mean', 'std']).T.sort_values(by='std').iloc[:,[0,2]].set_index('site_index').plot(kind='bar')\nplt.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.boxplot(train['meter'], train['meter_reading'], showfliers=False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,6))\ntrain.groupby(['site_id','building_id'])['meter_reading'].mean().sort_values()[-50:].plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,6))\ntrain.groupby(['site_id','building_id'])['meter_reading'].mean().sort_values()[:50].plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\nsns.boxplot(train['primary_use'], train['meter_reading'], showfliers=False)\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\nsns.boxplot(train['primary_use'], train['meter_reading'], showfliers=True)\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m=train.groupby('square_feet')['meter_reading'].mean().to_frame()\nplt.figure(figsize=(18,7))\nsns.scatterplot(m.index,m['meter_reading']);\nplt.ylim((0,5000))\nm=train.groupby('square_feet')['meter_reading'].median().to_frame()\nsns.scatterplot(m.index,m['meter_reading']);\nplt.ylim((0,5000))\nplt.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\nsns.boxplot(train.groupby('year_built')['meter_reading'].mean(), showfliers=False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\ntrain.groupby('year_built')['meter_reading'].mean().plot.bar()\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m=train.groupby('year_built')['meter_reading'].mean().to_frame()\nplt.figure(figsize=(18,7))\nsns.scatterplot(m.index,m['meter_reading'])\n\nm=train.groupby('year_built')['meter_reading'].median().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\n\nplt.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(26,7))\nsns.barplot(train['year_built'], train['meter_reading'])\nplt.xticks(rotation=90)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(26,7))\nsns.boxplot(train['floor_count'], train['meter_reading'], showfliers=False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(26,7))\ntrain.groupby(['floor_count'])['meter_reading'].mean().sort_values().plot.bar()\nplt.show() # To analyze floor count & meter reading by grouping on the basis of site_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(26,7))\nsns.barplot(train['floor_count'], train['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m=train.groupby('air_temperature')['meter_reading'].mean().to_frame()\nplt.figure(figsize=(18,7))\nsns.scatterplot(m.index,m['meter_reading'])\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"m=train.groupby('air_temperature')['meter_reading'].mean().to_frame()\nplt.figure(figsize=(18,7))\nsns.scatterplot(m.index,m['meter_reading'])\nm=train.groupby('dew_temperature')['meter_reading'].mean().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"m=train.groupby('dew_temperature')['meter_reading'].mean().to_frame()\nplt.figure(figsize=(18,7))\nsns.scatterplot(m.index,m['meter_reading'])\nm=train.groupby('dew_temperature')['meter_reading'].median().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(24,7))\nsns.boxplot(train['cloud_coverage'], train['meter_reading'], showfliers=False)\nplt.show() # To analyze cloud_coverage & meter reading by grouping on the basis of site_id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m=train.groupby('precip_depth_1_hr')['meter_reading'].mean().to_frame()\nplt.figure(figsize=(18,7))\nsns.scatterplot(m.index,m['meter_reading'])\nm=train.groupby('precip_depth_1_hr')['meter_reading'].median().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(18,7))\nm=train.groupby('sea_level_pressure')['meter_reading'].mean().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nm=train.groupby('sea_level_pressure')['meter_reading'].median().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(18,7))\nm=train.groupby('wind_direction')['meter_reading'].mean().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nm=train.groupby('wind_direction')['meter_reading'].median().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(18,7))\nm=train.groupby('wind_speed')['meter_reading'].mean().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nm=train.groupby('wind_speed')['meter_reading'].median().to_frame()\nsns.scatterplot(m.index,m['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Multivariate Analysis"},{"metadata":{"trusted":true},"cell_type":"code","source":"m = train.groupby('site_id')['air_temperature', 'cloud_coverage'\n                         , 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure'\n                         , 'wind_direction','wind_speed'].median()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n=[i for i in m.columns]\nn\n# site_id, month, hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()[train.isnull().sum()!=0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop(columns=['year_built', 'floor_count'],inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()[train.isnull().sum()!=0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m.isnull().sum()[m.isnull().sum()!=0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in n:  \n    train[i]=train.groupby(['site_id','amonth', 'ahour'])[i].apply(lambda x: x.fillna(x.median()))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.isnull().sum()[train.isnull().sum()!=0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"m.isnull().sum()[m.isnull().sum()!=0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['wind_speed'].value_counts().sort_index()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1 & 5 are having similar weather conditions - sea_level_pressure, most probably KNN will be used for imputation"},{"metadata":{"trusted":true},"cell_type":"code","source":"d","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"train.groupby('site_id')['timestamp', 'air_temperature', 'cloud_coverage'\n                         , 'dew_temperature', 'precip_depth_1_hr', 'sea_level_pressure'\n                         , 'wind_direction','wind_speed',].mean().sort_values(by='air_temperature')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['air_temperature'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nsns.distplot(train['meter_reading'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,6))\ntrain.groupby('building_id')['square_feet'].mean().plot(kind='bar')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"train.iloc[:,[9,10,11,12,13,14,15]][train['air_temperature'].isnull()]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(14, 6))\ntrain['timestamp'].dt.floor('d').value_counts().sort_index().plot(ax=axes).set_xlabel('Date', fontsize=14);\ntest['timestamp'].dt.floor('d').value_counts().sort_index().plot(ax=axes).set_ylabel('Number of training examples', fontsize=14);\naxes.set_title('Number of training examples by day', fontsize=16);\naxes.legend(['Train', 'Test']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1, 1, figsize=(32, 4))\ntrain.iloc[:,[2,3]].set_index(['timestamp']).resample('H').mean()['meter_reading'].plot(ax=axes\n                                                                                        , label='By hour').set_ylabel('Meter reading', fontsize=14);\ntrain.iloc[:\n           ,[2,3]].set_index('timestamp').resample('D').mean()['meter_reading'].plot(ax=axes\n                                                                                      , label='By day'\n                                                                                     , alpha=1).set_ylabel('Meter reading', fontsize=14);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nsns.boxplot(x='meter', y='meter_reading', data=train, showfliers=False);\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(14, 6))\nsns.boxplot(x='meter', y='air_temperature', data=train);\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":1}