{"cells":[{"metadata":{},"cell_type":"markdown","source":"# EDA for ASHRAE - Great Energy Predictor III Kaggle competition\nhttps://www.kaggle.com/c/ashrae-energy-prediction"},{"metadata":{},"cell_type":"markdown","source":"## Table of contents: <a class=\"anchor\" id=\"contents\"></a>\n* [Data Import](#DataImport)\n    * [Import](#Import)\n    * [Merge](#Merge)\n        \n* [EDA](#EDA)\n    * [Train](#Train)\n        * [Overview](#OverviewTrain)   \n        * [Missing](#MissingTrain) \n        * [meter_reading - target column](#meter_reading) \n        * [meter column](#meter) \n    * [Test](#Test)\n        * [Overview](#OverviewTest)   \n        * [Missing](#MissingTest) \n    * [Train VS Test](#TrainTestNum)\n    * [Weather](#Weather)\n        * [Overview](#OverviewWeather)   \n        * [Missing](#MissingWeather) \n        * [site_id key column](#site_idWeather) \n        * [air_temperature column](#air_temperature) \n        * [cloud_coverage column](#cloud_coverage) \n        * [dew_temperature column](#dew_temperature) \n        * [precip_depth_1_hr column](#precip_depth_1_hr)\n        * [sea_level_pressure column](#sea_level_pressure) \n        * [wind_direction column](#wind_direction) \n        * [wind_speed column](#wind_speed)\n    * [Buildings](#Buildings)\n        * [Overview](#OverviewBuildings)   \n        * [Missing](#MissingBuildings) \n        * [site_id key column](#site_idBuildings) \n        * [primary_use categorical column](#primary_use) \n        * [year_built column](#year_built) \n        * [floor_count column](#floor_count) "},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport time\nimport math\n\n# Visualiazation\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n# Import and suppress warnings\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def SecondsToStr(time_taken):\n    ''' Function return hours, minutes, seconds '''\n    ''' from the time in string format. '''\n\n    hours, rest = divmod(time_taken, 3600)\n    minutes, seconds = divmod(rest, 60)\n    h_ = str(math.trunc(hours))\n    m_ = str(math.trunc(minutes))\n    s_ = str(round(seconds, 2))\n    time_taken_str = ':'.join([h_, m_, s_])\n\n    # return hours, minutes, seconds from the time taken\n    return time_taken_str","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def df_eda(df_, with_stat_=False):\n    \n    # Columns of the DataFrame\n    print('columns:')\n    print(df_.columns.to_list())\n    # Shape (number of columns, rows)\n    print('\\nshape:')\n    print(df_.shape)\n    # Types of the columns\n    print('\\ntypes:')\n    print(df_.dtypes)\n    if with_stat_:\n        # Statistic for numerical columns\n        print('\\nstat:')\n        print(df_.describe())\n        \ndef column_info(df_, col_):\n    print(col_)\n    print('')\n    desc_ = df_[col_].describe()\n    print(round(desc_.drop(['count']), 2))\n    \ndef column_visualizatin(df_, col_, target_, koef_, with_target_=False):\n    fig, ax = plt.subplots() \n    ax.hist(df_[col_], color='g', alpha=0.5, normed=True, label=col_) \n    \n    if with_target_:\n        df_grouped_ = df_[[col_, target_]].groupby([col_]).mean().reset_index()\n        df_grouped_[target_] = df_grouped_[target_].astype('float')/koef_\n        ax.plot(df_grouped_[col_], df_grouped_[target_], color='r', label=target_)\n\n    ax.set(title=col_)\n    ax.legend(loc='best')\n    plt.show()\n    \ndef value_distribution(df_, col_, n_):\n    \n    ''' Check the share '''\n\n    print(round(df_[col_].value_counts(normalize=True)*100, 2)[:n_])\n    sns.countplot(x=col_, data=df_)\n    plt.xticks(rotation=90)\n    \ndef share_of_missing_per_column(df_, df_name_):\n    \n    print('')\n    print(df_name_)\n    print('Share of missing per column:\\n')\n    data = []\n\n    for col in df_.columns.to_list():\n        if (df_[col].isnull().sum() > 0):\n            data.append([col, '{}%'.format(round(100*df_[col].isnull().sum()/df_[col].shape[0], 2))])\n    return pd.DataFrame.from_records(data, columns=['Column', 'Missing_share'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Import <a class=\"anchor\" id=\"DataImport\"></a>\n[Table of contents](#contents)\n\nhttps://www.kaggle.com/c/ashrae-energy-prediction/data"},{"metadata":{},"cell_type":"markdown","source":"## Import <a class=\"anchor\" id=\"Import\"></a>\n[Table of contents](#contents)\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import train Dataset\nstart_time = time.time()\n\ntrain = pd.read_csv('../input/ashrae-energy-prediction/train.csv')\nbuilding_metadata = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nweather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\n\nprint('Total time: {}'.format(SecondsToStr(time.time() - start_time)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import test Dataset\nstart_time = time.time()\n\ntest = pd.read_csv('../input/ashrae-energy-prediction/test.csv')\nweather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')\nsample_submission = pd.read_csv('../input/ashrae-energy-prediction/sample_submission.csv')\n\nprint('Total time: {}'.format(SecondsToStr(time.time() - start_time)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge <a class=\"anchor\" id=\"Merge\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge Datasets\nstart_time = time.time()\n\n# Temporary merge for EDA\ntrain_df = (train.merge(building_metadata, on='building_id', how='left')).merge(weather_train, on=['site_id', 'timestamp'], how='left')\ntest_df = (test.merge(building_metadata, on='building_id', how='left')).merge(weather_test, on=['site_id', 'timestamp'], how='left')\n\n# Temporary merge for EDA\nweather_df = weather_train.append(weather_test, ignore_index=True)\nweather_columns = weather_df.columns.to_list()\n\nprint('Total time: {}'.format(SecondsToStr(time.time() - start_time)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA <a class=\"anchor\" id=\"EDA\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_eda(train_df, False)\n# df_eda(test_df, False)\n\ncat_cols, num_cols = [], []\n\nfor col in train_df.columns:\n    if train_df[col].dtype == object:\n        cat_cols.append(col)\n    else:\n        num_cols.append(col)\nprint('Numerical columns {}, categorical columns {}'.format(len(num_cols), len(cat_cols)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train <a class=\"anchor\" id=\"Train\"></a>\n[Table of contents](#contents)"},{"metadata":{},"cell_type":"markdown","source":"### Overview <a class=\"anchor\" id=\"OverviewTrain\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train shape:', train.shape)\nprint('Train TimeBorder:', train['timestamp'].min(), ':', train['timestamp'].max())\nprint('Train timestamp. Number of missing values:', train['timestamp'].isnull().sum())\ntrain.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Missing <a class=\"anchor\" id=\"MissingTrain\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"share_of_missing_per_column(train, 'Train')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### meter_reading target numerical column <a class=\"anchor\" id=\"meter_reading\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'meter_reading'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(round(train[target].value_counts(normalize=True)*100, 2))\ncolumn_info(train, target)\ntrain[target].plot()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Meter column <a class=\"anchor\" id=\"Meter\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the share of meter\nvalue_distribution(train, 'meter', 4)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test <a class=\"anchor\" id=\"Test\"></a>\n[Table of contents](#contents)"},{"metadata":{},"cell_type":"markdown","source":"### Overview <a class=\"anchor\" id=\"OverviewTest\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Test shape:', test.shape)\nprint('Test TimeBorder:', test['timestamp'].min(), ':', test['timestamp'].max())\nprint('Test timestamp. Number of missing values:', test['timestamp'].isnull().sum())\ntest.head(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Missing <a class=\"anchor\" id=\"MissingTest\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"share_of_missing_per_column(test, 'Test')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train VS Test (numerical statistic per column) <a class=\"anchor\" id=\"TrainTestNum\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"data = []\n\nlst = train_df.columns.to_list()\nlst.pop(lst.index(target))\n\nfor col in lst:\n    if col in num_cols:\n        eql_median = False\n\n        if train_df[col].median() == test_df[col].median():\n            eql_median = True\n\n        data.append([col, \n                     train_df[col].median(),\n                     test_df[col].median(),\n                     eql_median,\n                     train_df[col].mean(),\n                     test_df[col].mean(),\n                     train_df[col].var(),\n                     test_df[col].var()\n                     ])\npd.DataFrame.from_records(data, columns=['Column', \n                                         'Train_median', 'Test_median', \n                                         'Equal_medians',\n                                         'Train_mean', 'Test_mean', \n                                         'Train_var', 'Test_var', \n                                        ])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Weather <a class=\"anchor\" id=\"Weather\"></a>\n[Table of contents](#contents)"},{"metadata":{},"cell_type":"markdown","source":"### Overview <a class=\"anchor\" id=\"OverviewWeather\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Weather shape:', weather_df.shape)\nprint('Weather TimeBorder:', weather_df['timestamp'].min(), ':', weather_df['timestamp'].max())\nprint('Weather timestamp. Number of missing values:', weather_df['timestamp'].isnull().sum())\n\nweather_df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_df.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_df.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nsns.heatmap(weather_df.corr(), square=True, annot=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Missing <a class=\"anchor\" id=\"MissingWeather\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"share_of_missing_per_column(weather_df, 'Weather')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### site_id key column <a class=\"anchor\" id=\"site_idWeather\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the share of site_id\nvalue_distribution(weather_df, 'site_id', 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### air_temperature numerical column <a class=\"anchor\" id=\"air_temperature\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'air_temperature', target, 100000, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_df['air_temperature'].hist(color='salmon', alpha=0.5) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### cloud_coverage numerical column <a class=\"anchor\" id=\"cloud_coverage\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'cloud_coverage', target, 100000, True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### dew_temperature numerical column <a class=\"anchor\" id=\"dew_temperature\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'dew_temperature', target, 100000, True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_df['dew_temperature'].hist(color='salmon', alpha=0.5) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### precip_depth_1_hr numerical column <a class=\"anchor\" id=\"precip_depth_1_hr\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'precip_depth_1_hr', target, 1000000, True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### sea_level_pressure numerical column <a class=\"anchor\" id=\"sea_level_pressure\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'sea_level_pressure', target, 100000, True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### wind_direction numerical column <a class=\"anchor\" id=\"wind_direction\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'wind_direction', target, 1000000, True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### wind_speed numerical column <a class=\"anchor\" id=\"wind_speed\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'wind_speed', target, 100000, True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Buildings <a class=\"anchor\" id=\"Buildings\"></a>\n[Table of contents](#contents)"},{"metadata":{},"cell_type":"markdown","source":"### Overview <a class=\"anchor\" id=\"OverviewBuildings\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(building_metadata.shape)\nbuilding_metadata.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_metadata.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_metadata.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(8, 8))\nsns.heatmap(building_metadata.corr(), square=True, annot=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Missing <a class=\"anchor\" id=\"MissingBuildings\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"share_of_missing_per_column(building_metadata, 'Buildings')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### site_id numerical column <a class=\"anchor\" id=\"site_idBuildings\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the share of site_id\nvalue_distribution(building_metadata, 'site_id', 3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### primary_use categorical column <a class=\"anchor\" id=\"primary_use\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Check the share of primary_use\nvalue_distribution(building_metadata, 'primary_use', 10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### year_built numerical column <a class=\"anchor\" id=\"year_built\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'year_built', target, 100000, True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### floor_count numerical column <a class=\"anchor\" id=\"floor_count\"></a>\n[Table of contents](#contents)"},{"metadata":{"trusted":true},"cell_type":"code","source":"column_visualizatin(train_df, 'floor_count', target, 10000, True)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.3"}},"nbformat":4,"nbformat_minor":1}