{"cells":[{"metadata":{"id":"218rYG1v3lfN","colab_type":"text"},"cell_type":"markdown","source":"# Loading Libraries"},{"metadata":{"id":"acsjE3Vr3lfP","colab_type":"code","outputId":"29e8a156-3729-44f4-fe8c-9cbf6995bdef","colab":{"base_uri":"https://localhost:8080/","height":348},"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pandas import Series, DataFrame\nimport matplotlib as mpl\nimport pickle\nimport matplotlib.pyplot as plt\nimport seaborn as sns \n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"id":"Y47nFg4y3lfS","colab_type":"text"},"cell_type":"markdown","source":"# Loading dataset"},{"metadata":{"id":"WZn6MOX_3lfS","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"PATH = '../input/ashrae-energy-prediction/'\ntrain_df = pd.read_csv(PATH + 'train.csv')\nbuilding_meta_df=pd.read_csv(PATH + 'building_metadata.csv')\nweather_train_df=pd.read_csv(PATH + 'weather_train.csv')\nweather_test_df=pd.read_csv(PATH + 'weather_test.csv')\ntest_df = pd.read_csv(PATH + '/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"id":"oKuIWAH4pgyW","colab_type":"code","outputId":"fb6e7f69-4825-4b68-d064-38a125c7a397","colab":{"base_uri":"https://localhost:8080/","height":104},"trusted":true},"cell_type":"code","source":"print('Size of train data', train_df.shape)\nprint('Size of weather_train_df data', weather_train_df.shape)\nprint('Size of building_meta_df data', building_meta_df.shape)\n\nprint('Size of test data', test_df.shape)\nprint('Size of weather test data', weather_test_df.shape)\n\n\n\nprint('Dataset completo: ', train_df.shape)\ntrain_df= train_df.sample(frac=0.1, random_state=0)\nprint('Porzione ridotta per limiti computazionali:', train_df.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"isSmuBCYp8Yr","colab_type":"code","outputId":"aae55c61-32f2-4c50-99c7-3f8479b5acd1","colab":{"base_uri":"https://localhost:8080/","height":198},"trusted":true},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"id":"0YLbdOpopt8D","colab_type":"code","outputId":"03f2a80a-ed2f-4b34-f81f-ca9c4e66b660","colab":{"base_uri":"https://localhost:8080/","height":35},"trusted":true},"cell_type":"code","source":"train_df.columns.values","execution_count":null,"outputs":[]},{"metadata":{"id":"u03hqqCtpxh0","colab_type":"code","outputId":"1cac2b1e-0bc7-4d89-ee69-052ff7b8700a","colab":{"base_uri":"https://localhost:8080/","height":69},"trusted":true},"cell_type":"code","source":"weather_train_df.columns.values","execution_count":null,"outputs":[]},{"metadata":{"id":"VoydruZsp1dq","colab_type":"code","outputId":"bf8142df-1816-415d-9366-2286b764015b","colab":{"base_uri":"https://localhost:8080/","height":52},"trusted":true},"cell_type":"code","source":"building_meta_df.columns.values","execution_count":null,"outputs":[]},{"metadata":{"id":"HTCi7IYVqLtg","colab_type":"text"},"cell_type":"markdown","source":"# Data Description"},{"metadata":{"id":"b13nqn2l-6Ae","colab_type":"text"},"cell_type":"markdown","source":"**Metering data**\n*  building_id\n*  meter\n*  timestamp\n*  meter_reading\n\n**Buildings**\n*   primary_use\n*   square_feet\n*   year_built\n*   floor_count (may be too sparse to use)\n*   Weather\n\n\n**Weather**\n*   time of day\n*   holiday\n*   weekend\n*   cloud_coverage\n*   dew_temperature\n*   precip_depth\n*   sea_level_pressure\n*   wind_direction\n*   wind_speed\n\n**Meter_label**\n* 0: electricity\n* 1: chilledwater\n* 2: steam\n* 3: hotwater"},{"metadata":{"id":"R1i2GZ_SrBOK","colab_type":"text"},"cell_type":"markdown","source":"# Null values identification"},{"metadata":{},"cell_type":"markdown","source":"## Null value in training data"},{"metadata":{"id":"slA4nnbQrAqR","colab_type":"code","outputId":"e68c6ea6-fc39-4d10-ae7b-c0dc10f3f3d8","colab":{"base_uri":"https://localhost:8080/","height":168},"trusted":true},"cell_type":"code","source":"total = train_df.isnull().sum().sort_values(ascending = False)\npercent = (train_df.isnull().sum()/train_df.isnull().count()*100).sort_values(ascending = False)\nmissing__train_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing__train_data.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Null value in weather data"},{"metadata":{"id":"VciY_F60rKCo","colab_type":"code","outputId":"35c72256-18e8-40d7-c2e8-2ac95ce0b40a","colab":{"base_uri":"https://localhost:8080/","height":318},"trusted":true},"cell_type":"code","source":"total = weather_train_df.isnull().sum().sort_values(ascending = False)\npercent = (weather_train_df.isnull().sum()/weather_train_df.isnull().count()*100).sort_values(ascending = False)\nmissing_weather_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_weather_data.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Null value in building data"},{"metadata":{"id":"lFWE5gC-rQoY","colab_type":"code","outputId":"4a36814a-a6ce-45b5-89e2-d439468bc5cf","colab":{"base_uri":"https://localhost:8080/","height":228},"trusted":true},"cell_type":"code","source":"total = building_meta_df.isnull().sum().sort_values(ascending = False)\npercent = (building_meta_df.isnull().sum()/building_meta_df.isnull().count()*100).sort_values(ascending = False)\nmissing_building_meta_df  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_building_meta_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"id":"TwzL_GDGzZ6V","colab_type":"text"},"cell_type":"markdown","source":"# Features formatting and fixing"},{"metadata":{"id":"XDQPR2NltfIb","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"train_df[\"timestamp\"] = pd.to_datetime(train_df[\"timestamp\"])\ntrain_df[\"hour\"] = train_df[\"timestamp\"].dt.hour\ntrain_df[\"day\"] = train_df[\"timestamp\"].dt.day\ntrain_df[\"weekend\"] = train_df[\"timestamp\"].dt.weekday\ntrain_df[\"month\"] = train_df[\"timestamp\"].dt.month\ntrain_df[\"year\"] = train_df[\"timestamp\"].dt.year\n\nweather_train_df[\"timestamp\"] = pd.to_datetime(weather_train_df[\"timestamp\"])\n\ntest_df[\"timestamp\"] = pd.to_datetime(test_df[\"timestamp\"])\ntest_df[\"hour\"] = test_df[\"timestamp\"].dt.hour\ntest_df[\"day\"] = test_df[\"timestamp\"].dt.day\ntest_df[\"weekend\"] = test_df[\"timestamp\"].dt.weekday\ntest_df[\"month\"] = test_df[\"timestamp\"].dt.month\ntest_df[\"year\"] = train_df[\"timestamp\"].dt.year\nweather_test_df[\"timestamp\"] = pd.to_datetime(weather_test_df[\"timestamp\"])\nbuilding_meta_df['year_built'] = building_meta_df['year_built'].astype('Int64')\ntrain_df['meter_label']=train_df['meter'].apply(lambda x: {0: \"electricity\", 1: \"chilledwater\", 2: \"steam\", 3: \"hotwater\"}.get(x,x))","execution_count":null,"outputs":[]},{"metadata":{"id":"E0V92bT62QqF","colab_type":"text"},"cell_type":"markdown","source":"## Variables encoding"},{"metadata":{"id":"CgCBx3Ao2Dga","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\nle = LabelEncoder()\nbuilding_meta_df[\"primary_use_label\"] = le.fit_transform(building_meta_df[\"primary_use\"])","execution_count":null,"outputs":[]},{"metadata":{"id":"oh5X_wCJ1i85","colab_type":"text"},"cell_type":"markdown","source":"# Datasets merging"},{"metadata":{"id":"ixLVtaV8Moc-","colab_type":"code","outputId":"aa710b5b-4f8d-4885-fabc-bda552cb01df","colab":{"base_uri":"https://localhost:8080/","height":69},"trusted":true},"cell_type":"code","source":"train_df = train_df.merge(building_meta_df, on='building_id', how='left')\ntrain_df = train_df.merge(weather_train_df, on=['site_id', 'timestamp'], how='left')\ntest_df = test_df.merge(building_meta_df, on='building_id', how='left')\ntest_df = test_df.merge(weather_test_df, on=['site_id', 'timestamp'], how='left')\nprint('Size of train_df data', train_df.shape)\nprint('Size of weather_train_df data', weather_train_df.shape)\nprint('Size of building_meta_df data', building_meta_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"id":"_SimxW7LIZoY","colab_type":"text"},"cell_type":"markdown","source":"# Outliers removal (using IQR)"},{"metadata":{"id":"a2BFzyvyIB2S","colab_type":"code","colab":{},"trusted":true},"cell_type":"code","source":"## Using IQR\n\ndef outlier_treatment(datacolumn):\n    \n    \n    Q1,Q3 = np.percentile(sorted(datacolumn) , [25,75])\n    print (\"Q1:\",Q1)\n    print (\"Q3:\",Q3)\n    IQR = Q3 - Q1\n    print (\"IQR\",IQR)\n    lower_range = Q1 - (3 * IQR)\n    upper_range = Q3 + (3 * IQR)\n    \n    return lower_range,upper_range\n  \nl,u = outlier_treatment(train_df.meter_reading)\nprint(train_df [~(((train_df.meter_reading)> u) | ((train_df.meter_reading) < l)) ].shape)\ndf=train_df [~(((train_df.meter_reading)> u) | ((train_df.meter_reading) < l)) ]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploratory Data Analysis"},{"metadata":{"id":"jgaAb32ozsPv","colab_type":"text"},"cell_type":"markdown","source":"### Monthly energy usage in one building"},{"metadata":{"id":"dll8cQHwuP4U","colab_type":"code","outputId":"f4dfa452-70ed-4531-e951-84535de460c7","colab":{"base_uri":"https://localhost:8080/","height":366},"trusted":true},"cell_type":"code","source":"df[df[\"building_id\"] == 5].plot(\"timestamp\", \"meter_reading\",figsize=(5,5))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Visualization of monthly energy consumption "},{"metadata":{"trusted":true},"cell_type":"code","source":"df['timestamp'].value_counts().sort_index().plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Visualization of energy consumption by air temperature"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['air_temperature'].value_counts().sort_index().plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"###  Visualization of metering by building built year"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['year_built'].value_counts().sort_index().plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Building recorded metering"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['building_id'].value_counts().sort_index().plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Monthly metering by meter label"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 6))\ng=sns.countplot(x=\"month\", hue=\"meter_label\", data=df).set_title(\"Monthly metering by meter label\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Monthly energy consumption by meter label"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(16, 6))\ng=sns.barplot(x=\"month\", y=\"meter_reading\",hue=\"meter_label\", data=df).set_title(\"Monthly consumption by meter label\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Energy consumption by building primary usage "},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.FacetGrid(df, hue=\"meter_label\", col=\"primary_use\", col_wrap=4 )\ng.map(sns.barplot, \"month\",\"meter_reading\")\ng.add_legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Monthly metering recording by building primary usage "},{"metadata":{"trusted":true},"cell_type":"code","source":"def countplot(x, hue, **kwargs):\n    sns.countplot(x=x, hue=hue, **kwargs)\n\ngrid = sns.FacetGrid(data=df,col='primary_use',col_wrap=4,aspect=1)\nfig = grid.map(countplot,'month','meter_label',palette='Set1')\nfig.add_legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Monthly energy consumption by building primary usage "},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.FacetGrid(data=df, hue=\"meter_label\", col=\"primary_use\", col_wrap=4, )\n\ng.map(sns.lineplot, \"month\",\"meter_reading\")\ng.add_legend()\ng.set_xticklabels(['1','2','3','4','5','6','7','8','9','10','11','12'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Hourly energy consumption by building primary usage "},{"metadata":{"trusted":true},"cell_type":"code","source":"g = sns.FacetGrid(data=df, hue=\"meter_label\", col=\"primary_use\", col_wrap=4 )\n#g = g.map(plt.scatter, \"month\", \"meter_reading\", alpha=.7)\ng.map(sns.lineplot, \"hour\",\"meter_reading\")\ng.add_legend()","execution_count":null,"outputs":[]}],"metadata":{"colab":{"name":"Analisi","provenance":[],"toc_visible":true,"machine_shape":"hm"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"TPU"},"nbformat":4,"nbformat_minor":1}