{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **1.Imports**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport datetime\nimport warnings\nwarnings.filterwarnings('ignore')\nimport lightgbm as lgb\nimport gc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **2.Reading in Data**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"building = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\ntrain = pd.read_csv('../input/ashrae-energy-prediction/train.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **3.Glimpse of Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Size of building data', building.shape)\nprint('Size of weather_train data', weather_train.shape)\nprint('Size of train data', train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **4.Exploratory Data Analysis**"},{"metadata":{},"cell_type":"markdown","source":"## **4.1Building Primary Usage Distribution**"},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = building['primary_use'].unique()\ncount = sns.countplot(data = building,x = 'primary_use')\ncount.set(title = 'Number of Buildings primary use wise')\ncount.set_xticklabels(labels,rotation = 90)\ncount","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **4.2Feature Importance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"merge1 = building.merge(train, left_on = 'building_id', right_on = 'building_id')\nX = merge1.merge(weather_train, left_on = ['site_id','timestamp'], right_on = ['site_id','timestamp'])\ny = np.log1p(X.meter_reading)\nX.drop(\"timestamp\", axis = 1, inplace = True)\nX.drop(\"site_id\", axis = 1, inplace = True)\nX.drop(\"building_id\", axis = 1, inplace = True)\nX.drop(\"meter_reading\", axis = 1, inplace = True)\nX.drop(\"primary_use\", axis = 1, inplace = True)\nX_half_1 = X[:int(X.shape[0] / 2)]\nX_half_2 = X[int(X.shape[0] / 2):]\ny_half_1 = y[:int(X.shape[0] / 2)]\ny_half_2 = y[int(X.shape[0] / 2):]\nd_half_1 = lgb.Dataset(X_half_1, label = y_half_1,  free_raw_data = False)\nd_half_2 = lgb.Dataset(X_half_2, label = y_half_2,  free_raw_data = False)\nwatchlist_1 = [d_half_1, d_half_2]\nwatchlist_2 = [d_half_2, d_half_1]\nparams = {\n    \"objective\": \"regression\",\n    \"boosting\": \"gbdt\",\n    \"num_leaves\": 40,\n    \"learning_rate\": 0.05,\n    \"feature_fraction\": 0.85,\n    \"reg_lambda\": 2,\n    \"metric\": \"rmse\"\n}\nprint(\"Building model with first half and validating on second half:\")\nmodel_half_1 = lgb.train(params, train_set = d_half_1, num_boost_round = 1000, valid_sets = watchlist_1, verbose_eval = 200, early_stopping_rounds = 200)\nprint(\"Building model with second half and validating on first half:\")\nmodel_half_2 = lgb.train(params, train_set = d_half_2, num_boost_round = 1000, valid_sets = watchlist_2, verbose_eval = 200, early_stopping_rounds = 200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_fimp_1 = pd.DataFrame()\ndf_fimp_1[\"feature\"] = X.columns.values\ndf_fimp_1[\"importance\"] = model_half_1.feature_importance()\ndf_fimp_1[\"half\"] = 1\ndf_fimp_2 = pd.DataFrame()\ndf_fimp_2[\"feature\"] = X.columns.values\ndf_fimp_2[\"importance\"] = model_half_2.feature_importance()\ndf_fimp_2[\"half\"] = 2\ndf_fimp = pd.concat([df_fimp_1, df_fimp_2], axis=0)\nplt.figure(figsize=(14, 7))\nsns.barplot(x = \"importance\", y = \"feature\", data = df_fimp.sort_values(by = \"importance\", ascending = False))\nplt.title(\"LightGBM Feature Importance\")\nplt.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **4.3Meter Reading Missing Data Visualization**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['timestamp'] = pd.to_datetime(train.timestamp)\ntrain = train.set_index(['timestamp'])\nf,a = plt.subplots(1,4,figsize = (20,30))\nfor meter in np.arange(4):\n    df = train[train.meter==meter].copy().reset_index()\n    df['timestamp'] = pd.to_timedelta(df.timestamp).dt.total_seconds() / 3600\n    df['timestamp'] = df.timestamp.astype(int)\n    df.timestamp -= df.timestamp.min()\n    missmap = np.empty((1449, df.timestamp.max() + 1))\n    missmap.fill(np.nan)\n    for l in df.values:\n        if l[2] != meter:continue\n        missmap[int(l[1]), int(l[0])] = 0 if l[3] == 0 else 1\n    a[meter].set_title(f'meter {meter:d}')\n    sns.heatmap(missmap, cmap = 'Paired', ax = a[meter], cbar = False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Legend:\n* X axis: hours elapsed since Jan 1st 2016, for each of the 4 meter types\n* Y axis: building_id\n* Brown: meter reading available with non-zero value\n* Light blue: meter reading available with zero value\n* White: missing meter reading"},{"metadata":{},"cell_type":"markdown","source":"## **4.4Missing Weather Data Count**"},{"metadata":{},"cell_type":"markdown","source":"### 4.4.1 Adding Missing Hours"},{"metadata":{},"cell_type":"markdown","source":"This csv has hourly weather information for 16 sites in 2016. So this should have 140,544 records (16 x 24 x 366, 2016 is a leap year). But this csv has 139,773 records so 771 hours of data is missing."},{"metadata":{"trusted":true},"cell_type":"code","source":"time_format = \"%Y-%m-%d %H:%M:%S\"\nstart_date = datetime.datetime.strptime(weather_train['timestamp'].min(),time_format)\nend_date = datetime.datetime.strptime(weather_train['timestamp'].max(),time_format)\ntotal_hours = int(((end_date - start_date).total_seconds() + 3600) / 3600)\nhours_list = [(end_date - datetime.timedelta(hours = x)).strftime(time_format) for x in range(total_hours)]\nmissing_hours = []\nfor site_id in range(16):\n    site_hours = np.array(weather_train[weather_train['site_id'] == site_id]['timestamp'])\n    new_rows = pd.DataFrame(np.setdiff1d(hours_list,site_hours),columns = ['timestamp'])\n    new_rows['site_id'] = site_id\n    weather_train = pd.concat([weather_train,new_rows])\nweather_train = weather_train.reset_index(drop = True)    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4.4.2 Missing Values Count"},{"metadata":{"trusted":true},"cell_type":"code","source":"def missing_statistics(df):    \n    statistics = pd.DataFrame(df.isnull().sum()).reset_index()\n    statistics.columns=['COLUMN NAME',\"MISSING VALUES\"]\n    statistics['TOTAL ROWS'] = df.shape[0]\n    statistics['% MISSING'] = round((statistics['MISSING VALUES']/statistics['TOTAL ROWS']) * 100,2)\n    return statistics\nmissing_statistics_weather_train = missing_statistics(weather_train)\nmissing_statistics_weather_train.to_excel(\"missing_statistics_weather_train.xls\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **4.5Building Data Count**"},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_statistics_building = missing_statistics(building)\nmissing_statistics_building.to_excel(\"missing_statistics_building.xls\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_statistics(building)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}