{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Explaining this notebook\n\nI would like to explain my EDA for ASHARAE competition.\n\nI am bigginer of data science.  \nThis is my first notebook and I am not good at english.\n\nSo, Please forgive me my poor english."},{"metadata":{},"cell_type":"markdown","source":"# Load Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<pre>\n/kaggle/input/ashrae-energy-prediction/weather_train.csv\n/kaggle/input/ashrae-energy-prediction/test.csv\n/kaggle/input/ashrae-energy-prediction/weather_test.csv\n/kaggle/input/ashrae-energy-prediction/train.csv\n/kaggle/input/ashrae-energy-prediction/building_metadata.csv\n/kaggle/input/ashrae-energy-prediction/sample_submission.csv"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/train.csv\")\nbuilding_metadata = pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/building_metadata.csv\")\nweather_train = pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/weather_train.csv\")\n\nprint(\"train.shape:\", train.shape)\nprint(\"building_metadata.shape:\", building_metadata.shape)\nprint(\"weather_train.shape:\", weather_train.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merge train data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge = train.merge(building_metadata, on= \"building_id\",how=\"left\")\ntrain_merge = train_merge.merge(weather_train, on=[\"site_id\", \"timestamp\"], how=\"left\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# make features (day, months, weekdays)"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge[\"timestamp\"] = pd.to_datetime(train_merge[\"timestamp\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge[\"month\"] = train_merge[\"timestamp\"].dt.month\ntrain_merge[\"day\"] = train_merge[\"timestamp\"].dt.day\ntrain_merge[\"weekday\"] = train_merge[\"timestamp\"].dt.weekday\ntrain_merge[\"weekday_name\"] = train_merge[\"timestamp\"].dt.weekday_name","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# calculate outliers"},{"metadata":{"trusted":true},"cell_type":"code","source":"def CalcOutliers(df_num): \n\n    # calculating mean and std of the array\n    data_mean, data_std = np.mean(df_num), np.std(df_num)\n\n    # seting the cut line to both higher and lower values\n    # You can change this value\n    cut = data_std * 3\n\n    #Calculating the higher and lower cut values\n    lower, upper = data_mean - cut, data_mean + cut\n\n    # creating an array of lower, higher and total outlier values \n    outliers_lower = [x for x in df_num if x < lower]\n    outliers_higher = [x for x in df_num if x > upper]\n    outliers_total = [x for x in df_num if x < lower or x > upper]\n\n    # array without outlier values\n    outliers_removed = [x for x in df_num if x > lower and x < upper]\n    \n    print(\"upper:\", upper)\n    print(\"lower:\", lower)\n    print(\"-\"*30)\n    print('Identified lowest outliers: %d' % len(outliers_lower)) # printing total number of values in lower cut of outliers\n    print('Identified upper outliers: %d' % len(outliers_higher)) # printing total number of values in higher cut of outliers\n    print('Total outlier observations: %d' % len(outliers_total)) # printing total number of values outliers of both sides\n    print('Non-outlier observations: %d' % len(outliers_removed)) # printing total number of non outlier values\n    print(\"Total percentual of Outliers: \", round((len(outliers_total) / len(outliers_removed) )*100, 4)) # Percentual of outliers in points\n    \n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CalcOutliers(train_merge[\"meter_reading\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# detect outlier's building_id\n\nupper = 461823.9846903139\nnp.unique(train_merge.loc[train_merge[\"meter_reading\"] >= upper, \"building_id\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# analyzing 778\nid_778_outmeter = np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) & \n                                            (train_merge[\"building_id\"] == 778),\"meter\"].values)\nprint(\"id778 outlier / meter:\", id_778_outmeter)\n\nid_778_outmonth = np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) &\n                                            (train_merge[\"building_id\"] == 778), \"month\"].values)\nprint(\"id778 outlier / months:\", id_778_outmonth)\n\n\nfor i in id_778_outmonth:\n    print(\"months=\", i, sep=\"\")\n    print(\"primary_use\", np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) &\n                                        (train_merge[\"building_id\"] == 778) &\n                                        (train_merge[\"month\"] == i), \"primary_use\"].values))\n    \n    print(np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) &\n                                            (train_merge[\"building_id\"] == 778) &\n                                            (train_merge[\"month\"] == i), \"day\"].values))\n    print(\"-\" * 30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# analyzing 1099\nid_1099_outmeter = np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) & \n                                            (train_merge[\"building_id\"] == 1099),\"meter\"].values)\nprint(\"id778 outlier / meter:\", id_1099_outmeter)\n\nid_1099_outmonth = np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) &\n                                            (train_merge[\"building_id\"] == 1099), \"month\"].values)\nprint(\"id778 outlier / months:\", id_1099_outmonth)\n\nfor i in id_1099_outmonth:\n    print(\"month=\", i, sep=\"\")\n    print(\"primary_use\", np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) &\n                                            (train_merge[\"building_id\"] == 1099) &\n                                            (train_merge[\"month\"] == i), \"primary_use\"].values))\n    \n    print(\"days\", np.unique(train_merge.loc[(train_merge[\"meter_reading\"] >= upper) &\n                                            (train_merge[\"building_id\"] == 1099) &\n                                            (train_merge[\"month\"] == i), \"day\"].values))\n    print(\"-\" * 30)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Building_id 778 and 1099 have outliers.  \n\nOutliers of 778 is in september and october.  \nOutliers of 1099 is in Jan, Feb, Mar, Apl, May, Jun and Nov.\n\nI can not detect rule of days that have outliers."},{"metadata":{},"cell_type":"markdown","source":"# EDA"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge.groupby([\"timestamp\"])[\"meter_reading\"].mean().plot(figsize=(20, 5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I feel strange because this chart has plot that suddenly decrease."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge.groupby([\"month\"])[\"meter_reading\"].mean().plot(kind=\"bar\", figsize=(20, 5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Peak is Apl  \nAfter Jul, \"meter reading\" is low level."},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge.groupby([\"day\"])[\"meter_reading\"].mean().plot(kind=\"bar\", figsize=(20, 5))","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true},"cell_type":"code","source":"train_merge.groupby([\"air_temperature\"])[\"meter_reading\"].mean().plot(figsize=(20, 5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Low temperature has high \"meter reading\"  \n\nSo, I think that, cold has more \"meter reading\" than hot.  \nIn my opinion, high \"wind_speed\" has more \"meter_reading\""},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge.groupby([\"wind_speed\"])[\"meter_reading\"].mean().plot(figsize=(20, 5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"But, it is not related"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_merge.groupby([\"weekday_name\"])[\"meter_reading\"].mean().plot(kind=\"bar\", figsize=(20, 5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Weekends have low level \"meter reading\".  \n![](http://)So, \"Weekday\" has possible to important feature."},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"},"toc":{"base_numbering":1,"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":false,"toc_position":{},"toc_section_display":true,"toc_window_display":false}},"nbformat":4,"nbformat_minor":1}