{"cells":[{"metadata":{},"cell_type":"markdown","source":"## <center> Import and Glimpse of Data"},{"metadata":{},"cell_type":"markdown","source":"### <center> Import"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib.pylab import rcParams\nrcParams['figure.figsize'] = 28, 18\n\n%matplotlib inline\nplt.style.use('ggplot')\nimport dask.dataframe as dd\nimport gc\n\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\ninit_notebook_mode(connected=True)    #THIS LINE IS MOST IMPORTANT AS THIS WILL DISPLAY PLOT ON \n#NOTEBOOK WHILE KERNEL IS RUNNING\nimport plotly.graph_objs as go              \nimport plotly.express as px                 \nimport plotly.offline as py     # 绘图的函数","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### <center> Read Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"building = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\nweather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')\ntrain_df = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\ntest_df = pd.read_csv('../input/ashrae-energy-prediction/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### <center> Data Merge"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(building, on='building_id', how='left')\ntest_df = test_df.merge(building, on='building_id', how='left')\n\ntrain_df = train_df.merge(weather_train, on=['site_id', 'timestamp'], how='left')\ntest_df = test_df.merge(weather_test, on=['site_id', 'timestamp'], how='left')\n\ntrain_df['timestamp'] = pd.to_datetime(train_df[\"timestamp\"], format='%Y-%m-%d %H:%M:%S')\ntest_df['timestamp'] = pd.to_datetime(test_df[\"timestamp\"], format='%Y-%m-%d %H:%M:%S')\n\n\ndel weather_train, weather_test,building\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### <center>Reduce Memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n\n    # 计算当前占用的内存 \n    start_mem = df.memory_usage(deep=True).sum() / 1024**2\n\n    # 循环每一列\n    for col in df.columns:\n\n        # 获取每一列的数据类型\n        col_type = df[col].dtypes\n\n        # 如果数据类型属于上面定义的类型之\n        if col_type in numerics:\n\n            # 计算该列数据的最小值和最大值 用于我们指定相应的数据类型 \n            c_min = df[col].min()\n            c_max = df[col].max()\n\n            # 如果 该列的数据类型属于 int 类型，然后进行判断\n            if str(col_type)[:3] == 'int':\n                # 如果 该列最小的值 大于int8类型的最小值，并且最大值小于int8类型的最大值，则采用int8类型 \n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n\n                # 同上\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n\n                # 同上\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n\n                # 同上\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n\n            # 否则 则采用 float 的处理方法       \n            else:\n\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    end_mem = df.memory_usage(deep=True).sum() / 1024**2\n\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['hour'] = train_df.timestamp.dt.hour\ntrain_df['day'] = train_df.timestamp.dt.day\ntrain_df['week'] = ((train_df.timestamp.dt.dayofweek) // 5 == 1).astype(float) \ntrain_df['month'] = train_df.timestamp.dt.month\n\ntest_df['hour'] = test_df.timestamp.dt.hour\ntest_df['day'] = test_df.timestamp.dt.day\ntest_df['week'] = ((test_df.timestamp.dt.dayofweek) // 5 == 1).astype(float) \ntest_df['month'] = test_df.timestamp.dt.month","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_bd = np.round(train_df.memory_usage().sum()/(1024*1024),1)\ntest_bd = np.round(test_df.memory_usage().sum()/(1024*1024),1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ntrain_df = reduce_mem_usage(train_df)\ntest_df = reduce_mem_usage(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ad = np.round(train_df.memory_usage().sum()/(1024*1024),1)\ntest_ad = np.round(test_df.memory_usage().sum()/(1024*1024),1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### <center> Plot Memory Reduce "},{"metadata":{"trusted":true},"cell_type":"code","source":"memory = pd.DataFrame({\n    'DataFrame':['train','test'],\n    'Before memory reducing':[train_bd,test_bd],\n    'After memory reducing':[train_ad,test_ad],\n\n})\n\nmemory = pd.melt(memory,id_vars='DataFrame',var_name='Status',value_name='Memory(MB)')\nmemory.sort_values('Memory(MB)',inplace=True)\nfig = px.bar(memory,x='DataFrame',y='Memory(MB)',color='Status',barmode='group',text='Memory(MB)')\nfig.update_traces(textposition='outside')\nfig.update_layout(template='seaborn',title='Effect of Downcasting')\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### <center> Data info"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nstats = []\nfor col in train_df.columns:\n    stats.append((col, train_df[col].nunique(), train_df[col].isnull().sum() * 100 / train_df.shape[0], train_df[col].value_counts(normalize=True, dropna=False).values[0] * 100, train_df[col].dtype))\n\n# 列分别为 特征名、该特征独立值数量、该特征缺失值比例、该特征下出现最多的值占所有值的比例、数据类型\nstats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])\nstats_df.sort_values('Percentage of missing values', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nstats = []\nfor col in test_df.columns:\n    stats.append((col, test_df[col].nunique(), test_df[col].isnull().sum() * 100 / test_df.shape[0], test_df[col].value_counts(normalize=True, dropna=False).values[0] * 100, test_df[col].dtype))\n\n# 列分别为 特征名、该特征独立值数量、该特征缺失值比例、该特征下出现最多的值占所有值的比例、数据类型\nstats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])\nstats_df.sort_values('Percentage of missing values', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`floor_count` have a lot of missing values for count ，that seem to say nothing"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.drop(columns='floor_count',inplace=True)\ntest_df.drop(columns='floor_count',inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## <center> Visualization and Data Processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"category_features=['hour','day','week','month','primary_use','meter','site_id'] \ntrain_df[category_features] = train_df[category_features].astype('category')\ntest_df[category_features] = test_df[category_features].astype('category')\n\ntrain_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nstats = []\nfor col in train_df.columns:\n    stats.append((col, train_df[col].nunique(), train_df[col].isnull().sum() * 100 / train_df.shape[0], train_df[col].value_counts(normalize=True, dropna=False).values[0] * 100, train_df[col].dtype))\n\n# 列分别为 特征名、该特征独立值数量、该特征缺失值比例、该特征下出现最多的值占所有值的比例、数据类型\nstats_df = pd.DataFrame(stats, columns=['Feature', 'Unique_values', 'Percentage of missing values', 'Percentage of values in the biggest category', 'type'])\nstats_df.sort_values('Percentage of missing values', ascending=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 筛选出 train 和 test 共同列"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('B Training Features shape: ', train_df.shape)\nprint('B Testing Features shape: ', test_df.shape)\n\ntrain_labels = train_df['meter_reading']\n\n# Align the training and testing data, keep only columns present in both dataframes \n# 选出两个数据中共同拥有的列，不共有的列过滤出去\ntrain_df, test_df = train_df.align(test_df, join = 'inner', axis = 1)\n\n# Add the target back in\ntrain_df['meter_reading'] = train_labels\n\nprint('A Training Features shape: ', train_df.shape)\nprint('A Testing Features shape: ', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### look at train and test dist"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Density plots of features function\ndef plot_feature_distribution(df1, df2, label1, label2, features):\n    i = 0\n    sns.set_style('whitegrid')\n    plt.figure()\n    fig, ax = plt.subplots(10,10,figsize=(18,22))\n    \n    failed_features = []\n    \n    for feature in features:\n        try:\n            i += 1\n            plt.subplot(5,2,i)\n            sns.distplot(df1[feature], hist=False,label=label1)\n            sns.distplot(df2[feature], hist=False,label=label2)\n            plt.xlabel(feature, fontsize=9)\n            locs, labels = plt.xticks()\n            plt.tick_params(axis='x', which='major', labelsize=6, pad=-6)\n            plt.tick_params(axis='y', which='major', labelsize=6)\n        except:\n            print(feature + 'KDE failed')\n            failed_features.append(feature)\n            continue\n    plt.show();\n    \n    return failed_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.set_style('darkgrid')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### year_built:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plotly func \n\n# def plot_categorical_feature(col,top_n=10000,train=train_df):\n    \n#     top_n = top_n if train[col].nunique() > top_n else train[col].nunique()\n#     print(f\"{col} has {train[col].nunique()} unique values and type: {train[col].dtype}.\")\n#     print(train[col].value_counts(normalize=True, dropna=False).head())\n    \n#     top_cat = list(train[col].value_counts(dropna=False).index[:top_n])\n    \n#     df = train.groupby([col]).agg({'meter_reading': ['count','mean']})\n#     df = df.sort_values(('meter_reading', 'count'), ascending=False).head(top_n).sort_index()\n    \n#     data = [go.Bar(x=df.index, y=df['meter_reading']['count'], name='counts'),\n#             go.Bar(x=df.index, y=df['meter_reading']['count'], name='counts'),\n#             go.Scatter(x=df.index, y=df['meter_reading']['mean'], name='meter_reading mean', yaxis='y2')]\n\n    \n    \n#     layout = go.Layout(dict(title = f\"Counts of {col} by top-{top_n} categories\",\n#                             xaxis = dict(title = f'{col}',\n#                                     showgrid=False,\n#                                     zeroline=False,\n#                                     showline=False,),\n#                             yaxis = dict(title = 'Counts',\n#                                     showgrid=False,\n#                                     zeroline=False,\n#                                     showline=False,),\n#                             yaxis2=dict(title='meter_reading mean', \n#                                         overlaying='y', \n#                                         side='right')\n#                             ),\n#                     legend=dict(orientation=\"v\"), barmode='group')\n\n#     py.iplot(dict(data=data, layout=layout))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[col for col in train_df.columns if col != 'meter_reading' and col not in category_features and col != 'timestamp']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features = [col for col in train_df.columns if col != 'meter_reading' and col not in category_features and col != 'timestamp']\nplot_feature_distribution(train_df, test_df, 'train', 'test', features)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we can see that the distribution of training data and test data is basically the same"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['precip_depth_1_hr'].hist(figsize=(12,8))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### meter_reading"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(figsize = (16, 6))\nplt.subplot(1, 2, 1)\nplt.hist(train_df['meter_reading']);\nplt.title('Distribution of meter_reading');\nplt.subplot(1, 2, 2)\nplt.hist(np.log1p(train_df['meter_reading']));\nplt.title('Distribution of log of meter_reading');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### meter:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_categorical_feature('meter')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### <center>Classified data："},{"metadata":{"trusted":true},"cell_type":"code","source":"def Classification_features_and_continuous_targets_func(col,target='meter_reading',rotation=0,loc=2):\n    \n    # 训练数据\n    train_data = train_df[col].value_counts(dropna=False, normalize=True).sort_index().values\n    ind = np.arange(len(train_data))\n    width = 0.35\n\n    fig, axes = plt.subplots(1,1,figsize=(14, 6), dpi=100)\n    tr = axes.bar(ind, train_data, width, color='royalblue')\n    \n    # 测试数据\n    test_data = test_df[col].value_counts(dropna=False, normalize=True).sort_index().values\n    tt = axes.bar(ind+width, test_data, width, color='seagreen')\n\n    axes.set_ylabel('Normalized number of observations');\n    axes.set_xlabel(col);\n    axes.set_xticks(ind + width / 2)\n    axes.set_xticklabels(train_df[col].value_counts().sort_index().index, rotation=rotation)\n    axes2 = axes.twinx()\n    mr = axes2.plot(ind, train_df[[col, target]].groupby(col)[target].mean().sort_index().values, 'D-', color='tab:orange', label='Mean {}'.format(target));\n    axes2.grid(False);\n    axes2.tick_params(axis='y', labelcolor='tab:orange');\n    axes2.set_ylabel('Mean {} by {}'.format(target,col), color='tab:orange');\n    axes.legend([tr, tt], ['Train', 'Test'], facecolor='white');\n    axes2.legend(loc=loc, facecolor='white');","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Classification_features_and_continuous_targets_func('meter','meter_reading')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(13,6)) #figure size\n\n#It's another way to plot our data. using a variable that contains the plot parameters\ng1 = sns.boxplot(x='meter', y='meter_reading', \n                   data=train_df[(train_df['meter'].isin((train_df['meter'].value_counts()[:10].index.values))) &\n                                  train_df['meter_reading'] > 0]\n                   ,showfliers=False)\ng1.set_title('Meter by meter_reading', fontsize=20) # title and fontsize\ng1.set_xticklabels(g1.get_xticklabels(),rotation=45) # It's the way to rotate the xticks when we use variable to our graphs\ng1.set_xlabel('Meter', fontsize=18) # Xlabel\ng1.set_ylabel('Trans meter_reading(log) Dist', fontsize=18) #Ylabel\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### site_id:"},{"metadata":{"trusted":true},"cell_type":"code","source":"Classification_features_and_continuous_targets_func('site_id',rotation=0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### cloud_coverage:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = train_df['cloud_coverage'].value_counts(normalize=True).sort_index().values\nind = np.arange(len(train_data))\nwidth = 0.35\n\nfig, axes = plt.subplots(1,1,figsize=(14, 6), dpi=100)\ntr = axes.bar(ind, train_data, width, color='royalblue')\n\n# 测试数据\ntest_data = test_df['cloud_coverage'].value_counts(normalize=True).sort_index().values\ntt = axes.bar(ind+width, test_data, width, color='seagreen')\n\naxes.set_ylabel('Normalized number of observations');\naxes.set_xlabel('cloud_coverage');\naxes.set_xticks(ind + width / 2)\naxes.set_xticklabels(train_df['cloud_coverage'].value_counts().sort_index().index, rotation=0)\naxes2 = axes.twinx()\nmr = axes2.plot(ind, train_df[['cloud_coverage', 'meter_reading']].groupby('cloud_coverage')['meter_reading'].mean().sort_index().values, 'D-', color='tab:orange', label='Mean {}'.format('meter_reading'));\naxes2.grid(False);\naxes2.tick_params(axis='y', labelcolor='tab:orange');\naxes2.set_ylabel('Mean {} by {}'.format('meter_reading','cloud_coverage'), color='tab:orange');\naxes.legend([tr, tt], ['Train', 'Test'], facecolor='white');\naxes2.legend(loc=2, facecolor='white');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### primary_use:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_categorical_feature('primary_use')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Classification_features_and_continuous_targets_func('primary_use',rotation=90,loc=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"education and services meter_reading is very big"},{"metadata":{},"cell_type":"markdown","source":"### timestamp:"},{"metadata":{"trusted":true},"cell_type":"code","source":"Classification_features_and_continuous_targets_func('week',rotation=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Classification_features_and_continuous_targets_func('month',rotation=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_categorical_feature('hour')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_categorical_feature('day')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_categorical_feature('week')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# plot_categorical_feature('month')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### <center>Continuous data："},{"metadata":{},"cell_type":"markdown","source":"### timestamp："},{"metadata":{"trusted":true},"cell_type":"code","source":"# Calling the function to transform the date column in datetime pandas object\n\n# 设置一些静态颜色选项\ncolor_op = ['#5527A0', '#BB93D7', '#834CF7', '#6C941E', '#93EAEA', '#7425FF', '#F2098A', '#7E87AC', \n        '#EBE36F', '#7FD394', '#49C35D', '#3058EE', '#44FDCF', '#A38F85', '#C4CEE0', '#B63A05', \n        '#4856BF', '#F0DB1B', '#9FDBD9', '#B123AC']\n\ndates_temp = train_df.groupby(train_df.timestamp.dt.date)['meter_reading'].mean().reset_index()\n\n# 使用第一个必要的 trace 参数，opacity 表示透明度\ntrace = go.Scatter(x=dates_temp['timestamp'], y=dates_temp.meter_reading,\n                opacity = 0.8, line = dict(color = color_op[7]), name= 'Meter_reading mean')\n\n# 下面我们将得到销售总额 (具体案例具体分析)\ndates_temp_sum = train_df.groupby(train_df.timestamp.dt.date)['meter_reading'].sum().reset_index()\n\n# 使用新的 dates_temp_sum ，我们将创建第二个 trace \ntrace1 = go.Scatter(x=dates_temp_sum.timestamp, line = dict(color = color_op[1]), name=\"Total Amount\",\n                        y=dates_temp_sum['meter_reading'], opacity = 0.8, yaxis='y2')\n\n# 创建布局将允许我们给出标题和给我们一些有趣的选项来处理图的输出\nlayout = dict(\n        title= \"Total meter_reading Informations by Date\",\n        xaxis=dict(\n                rangeselector=dict(\n                buttons=list([\n                        dict(count=1, label='1m', step='month', stepmode='backward'),\n                        dict(count=3, label='3m', step='month', stepmode='backward'),\n                        dict(count=6, label='6m', step='month', stepmode='backward'),\n                        dict(step='all')\n                ])\n                ),\n                rangeslider=dict(visible = True), # 控制最下面的图\n                type='date' ),\n        yaxis=dict(title='Total meter_reading'), # 左侧的y轴\n        yaxis2=dict(overlaying='y',             # 右侧的y轴 side ='right'\n                        anchor='x', side='right',\n                        zeroline=False, showgrid=False,\n                        title='Total meter_reading Amount')\n        )\n\n# 创建既有 trace 又有 layout 的图形\nfig = dict(data= [trace, trace1,], layout=layout)\n\n#绘制图形\niplot(fig) #it's an equivalent to plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meter_reading_sum = train_df[['timestamp','meter_reading']].groupby(['timestamp']).sum()\nmeter_reading_sum.head()\n\n\nfrom statsmodels.tsa.seasonal import seasonal_decompose\n\nweeks_per_year = 365\n\n\ntime_series = meter_reading_sum['meter_reading']\nsj_sc = seasonal_decompose(time_series, freq= weeks_per_year)\nsj_sc.plot()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fig, axes = plt.subplots(8,2,figsize=(14, 30), dpi=100)\n# for i in range(train_df['site_id'].nunique()):\n#     train_df[train_df['site_id'] == i][['timestamp', 'sea_level_pressure']].set_index('timestamp').resample('H').mean()['sea_level_pressure'].plot(ax=axes[i%8][i//8], alpha=0.8, label='By hour', color='tab:blue').set_ylabel('Mean sea_level_pressure', fontsize=13);\n#     test_df[test_df['site_id'] == i][['timestamp', 'sea_level_pressure']].set_index('timestamp').resample('H').mean()['sea_level_pressure'].plot(ax=axes[i%8][i//8], alpha=0.8, color='tab:blue', label='').set_xlabel('')\n#     train_df[train_df['site_id'] == i][['timestamp', 'sea_level_pressure']].set_index('timestamp').resample('D').mean()['sea_level_pressure'].plot(ax=axes[i%8][i//8], alpha=1, label='By day', color='tab:orange')\n#     test_df[test_df['site_id'] == i][['timestamp', 'sea_level_pressure']].set_index('timestamp').resample('D').mean()['sea_level_pressure'].plot(ax=axes[i%8][i//8], alpha=1, color='tab:orange', label='').set_xlabel('')\n#     axes[i%8][i//8].legend();\n#     axes[i%8][i//8].set_title('site_id {}'.format(i), fontsize=13);\n#     axes[i%8][i//8].axvspan(test_df['timestamp'].min(), test_df['timestamp'].max(), facecolor='green', alpha=0.2);\n#     plt.subplots_adjust(hspace=0.45)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### precip_depth_1_hr"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df[['timestamp', 'precip_depth_1_hr']].set_index('timestamp').resample('M').mean()['precip_depth_1_hr'].plot(ax=axes, alpha=0.8, label='By month', color='tab:blue').set_ylabel('Mean precip_depth_1_hr', fontsize=14);\ntest_df[['timestamp', 'precip_depth_1_hr']].set_index('timestamp').resample('M').mean()['precip_depth_1_hr'].plot(ax=axes, alpha=0.8, color='tab:blue', label='');\naxes.legend();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df.groupby('precip_depth_1_hr')['meter_reading'].mean().plot().set_ylabel('Mean meter reading');\naxes.set_title('Mean meter reading by precip_depth_1_hr of the building', fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### year_built："},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 2, figsize=(10, 8))\nsns.kdeplot(train_df['year_built'], ax=axes[0][0], label='Train');\nsns.kdeplot(test_df['year_built'], ax=axes[0][0], label='Test');\nsns.violinplot(x=train_df['year_built'], ax=axes[1][0]);\nsns.violinplot(x=test_df['year_built'], ax=axes[1][1]);\npd.DataFrame({'train': [train_df['year_built'].isnull().sum()], 'test': [test_df['year_built'].isnull().sum()]},index=['year_built']).plot(kind='bar', rot=0, ax=axes[0][1]);\naxes[0][0].legend();\naxes[0][0].set_title('Train/Test KDE distribution',fontsize=10);\naxes[0][1].set_title('Number of NaNs',fontsize=10);\naxes[1][0].set_title('violinplot for train',fontsize=10);\naxes[1][1].set_title('violinplot for test',fontsize=10);\ngc.collect();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df.groupby('year_built')['meter_reading'].mean().plot().set_ylabel('Mean meter reading');\naxes.set_title('Mean meter reading by year_built of the building', fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### building_id"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df.groupby('building_id')['meter_reading'].mean().plot().set_ylabel('Mean meter reading');\naxes.set_title('Mean meter reading by building_id of the building', fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"As shown in the figure ： may be have outliers in the data，Let's explore further"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[(train_df['meter_reading'] > 2500000)]['building_id'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df[(train_df['primary_use'] == 'Education')].groupby('building_id')['meter_reading'].mean().plot().set_ylabel('Mean meter reading');\naxes.set_title('Mean meter reading by primary_use == Education  of the building', fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is an obvious outlier in data. building id == 1099\n\n能看到在 primary_use == Education 的数据中，building_id = 1099 的数据好像存在异常"},{"metadata":{},"cell_type":"markdown","source":"Mean meter reading by building_id WITHOUT building_id 1099 \n\n这里排除  building_id 1099 的数据再来看一下："},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df[train_df['building_id'] != 1099].groupby('building_id')['meter_reading'].mean().plot();\naxes.set_title('Mean meter reading by building_id', fontsize=14);\naxes.set_ylabel('Mean meter reading', fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### square_feet:"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df[train_df['building_id'] != 1099].groupby('square_feet')['meter_reading'].mean().plot();\naxes.set_title('Mean meter reading by square_feet of the building', fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"在图中我们似乎比较难直接发现 square_feet 和 meter_reading之间的关系"},{"metadata":{},"cell_type":"markdown","source":"### wind_speed:"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df.groupby('wind_speed')['meter_reading'].mean().plot().set_ylabel('Mean meter reading');\naxes.set_title('Mean meter reading by wind_speed of the building', fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### air_temperature:"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(1,1,figsize=(14, 6))\ntrain_df.groupby('air_temperature')['meter_reading'].mean().plot().set_ylabel('Mean meter reading');\naxes.set_title('Mean meter reading by air_temperature of the building', fontsize=16);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}