{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport warnings\nimport datetime\nimport gc\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.model_selection import train_test_split,KFold\nfrom sklearn.metrics import mean_squared_error\n\n\nimport numpy as np\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\n\n%matplotlib inline\nfrom sklearn.feature_selection import RFE, f_regression\nfrom sklearn.linear_model import (LinearRegression, Ridge, Lasso,LogisticRegression)\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.impute import SimpleImputer\n\nfrom sklearn.ensemble import RandomForestRegressor\nfrom math import sqrt\nimport matplotlib.pyplot as plt\nimport seaborn as sns; sns.set()\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nSMALL_SIZE = 10\nMEDIUM_SIZE = 12\nBIGGER_SIZE = 16\n\nplt.rc('font', size=SMALL_SIZE)          # controls default text sizes\nplt.rc('axes', titlesize=BIGGER_SIZE)    # fontsize of the axes title\nplt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels\nplt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels\nplt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels\nplt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize\nplt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"** The following function is based on https://www.kaggle.com/gemartin/load-data-reduce-memory-usage?fbclid=IwAR2PdhpX6JywVbJ84gZTngvXORMhP0t2hMXJZlrzktANha4Xf2YDPQqI538"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n\n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df\n\n\ndef import_data(file):\n    \"\"\"create a dataframe and optimize its memory usage\"\"\"\n    df = pd.read_csv(file, parse_dates=True, keep_date_col=True)\n    df = reduce_mem_usage(df)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef missing_statistics(df):    \n    statitics = pd.DataFrame(df.isnull().sum()).reset_index()\n    statitics.columns=['COLUMN NAME',\"MISSING VALUES\"]\n    statitics['TOTAL ROWS'] = df.shape[0]\n    statitics['% MISSING'] = round((statitics['MISSING VALUES']/statitics['TOTAL ROWS'])*100,2)\n    return statitics","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Importation\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('-' * 80)\nprint('building')\nbuilding = import_data('/kaggle/input/ashrae-energy-prediction/building_metadata.csv')\n\nprint('-' * 80)\nprint('train')\ntrain = import_data('/kaggle/input/ashrae-energy-prediction/train.csv')\n\nprint('-' * 80)\nprint('test')\ntest = import_data('/kaggle/input/ashrae-energy-prediction/test.csv')\n\nprint('-' * 80)\nprint('weather train')\nweather_train = import_data('/kaggle/input/ashrae-energy-prediction/weather_train.csv')\n\nprint('-' * 80)\nprint('weather test')\nweather_test = import_data('/kaggle/input/ashrae-energy-prediction/weather_test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building.describe().transpose()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# check missing values in building data\nmissing_statistics(building)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxplot1 = building.boxplot(column=['floor_count'], by=['site_id'],figsize= (15,7))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxplot = building.boxplot(column=['year_built'], by=['site_id'],figsize= (10,7))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## removing columns with more than 50% NANs\nbuilding=building.drop(columns=['year_built','floor_count']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del boxplot,boxplot1\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Join tables \n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge dataset to create training dataframe\n\ntrain_buil = train.merge(building, on='building_id', how='left')\ndata_train = train_buil.merge(weather_train, on=['site_id','timestamp'], how='left')\ndata_train['timestamp'] = pd.to_datetime(data_train.timestamp, format='%Y-%m-%d %H:%M:%S')\n\ndel weather_train,train,train_buil\ndata_train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Merge dataset to create test dataframe\ntest_buil = test.merge(building, on='building_id', how='left')\ndata_test = test_buil.merge(weather_test, on=['site_id','timestamp'], how='left')\n\ndel weather_test,test_buil\n\ndata_test.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataframes description"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_statistics(data_train) \n# the cloud_coverage columns contains alot of missing values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_statistics(data_test) \n# the cloud_coverage columns contains alot of missing values ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"O, C = data_test.shape\nprint(f'Dans test, il y a {O} observations et {C} colonnes.')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"O, C = data_train.shape\nprint(f'Dans train, il y a {O} observations et {C} colonnes.') ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Deleting cloud_coverage column "},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train=data_train.drop(columns=['cloud_coverage']);\ndata_test=data_test.drop(columns=['cloud_coverage']);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Rows processing\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"liste = data_train.isnull().sum(axis=1).tolist()#liste with the number of nans in each row\n\nRowsEQUAL,RowsANDHIGH=[],[]\nfor n in range(0,8):\n    RowsEQUAL.append(round(sum( i == n for i in liste)/data_train.shape[0]*100,3))\n    RowsANDHIGH.append(round(sum(i > n for i in liste)/data_train.shape[0]*100,3))\n\nRows_Statistics = pd.DataFrame(RowsEQUAL, columns=[\"% Rows with NANs (=)\"])\nRows_Statistics[\" % Rows with NANs (>)\"] = RowsANDHIGH\nRows_Statistics.index.name = 'Number of NANs'\n#df.index += 1 \nprint(Rows_Statistics);\n\ndel RowsEQUAL,RowsANDHIGH\ndel Rows_Statistics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Keep only the rows with at least 8 non-NAN values # 6 NAN or plus\ndata_train = data_train.dropna(thresh = 8) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"o, c = data_train.shape\nprint(f'Dans test apres suppression , il y a {o} observations et {c} colonnes.') #20216100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Nombre de lignes supprimées : \" + str(O-o))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# number of rows deleted\nfeatures = data_train.columns\nprint(f'Les {C} colonnes sont: ', list(features))\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.timestamp.unique  #train data is for 1 year\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_test.timestamp.unique  #test data for 2 years","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.to_datetime(data_train.timestamp, format='%Y-%m-%d %H:%M:%S').dt.time.astype(str).unique()\n#meter reading for each hour","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# weather plotboxs\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train[\"datetime\"] = pd.to_datetime(data_train[\"timestamp\"])\ndata_train[\"hour\"] = data_train[\"datetime\"].dt.hour;\ndata_train[\"day\"] = data_train[\"datetime\"].dt.day;\ndata_train[\"week\"] = data_train[\"datetime\"].dt.week;\ndata_train[\"month\"] = data_train[\"datetime\"].dt.month;\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(6,1,figsize=(13,20)) \nfor i,col in enumerate(['air_temperature','dew_temperature','wind_direction','wind_speed','precip_depth_1_hr', 'sea_level_pressure']):\n\n    plot = data_train.boxplot(col, by=\"site_id\", ax=axes.flatten()[i]);\n\nplt.tight_layout() ;\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We notice a lack of data for some entire Sit, in terms of  'precip depth' and 'sea level pressure'. While for outliers we notice a large number of outliers especially in 'precip depth'"},{"metadata":{},"cell_type":"markdown","source":"# Filling in missing data using the mean\n### in terms of 'site_id', 'hour' and 'month'"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train = data_train.set_index(['site_id','hour','month'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling air temperature\nair_temperature_filler = pd.DataFrame(data_train.groupby(['site_id','hour','month'])['air_temperature'].mean(),columns=[\"air_temperature\"])\ndata_train.update(air_temperature_filler,overwrite=False)\ndel air_temperature_filler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling due temperature\ndue_temperature_filler = pd.DataFrame(data_train.groupby(['site_id','hour','month'])['dew_temperature'].mean(),columns=[\"dew_temperature\"])\ndata_train.update(due_temperature_filler,overwrite=False)\ndel due_temperature_filler\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# filling sea_level_pressure\nsea_level_filler = pd.DataFrame(data_train.groupby(['site_id','hour','month'])['sea_level_pressure'].mean(),columns=['sea_level_pressure'])\n\ndata_train.update(sea_level_filler,overwrite=False)\ndel sea_level_filler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling wind_direction\nwind_direction_filler =  pd.DataFrame(data_train.groupby(['site_id','hour','month'])['wind_direction'].mean(),columns=['wind_direction'])\ndata_train.update(wind_direction_filler,overwrite=False)\ndel wind_direction_filler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling  wind speed\nwind_speed_filler =  pd.DataFrame(data_train.groupby(['site_id','hour','month'])['wind_speed'].mean(),columns=['wind_speed'])\ndata_train.update(wind_speed_filler,overwrite=False)\ndel wind_speed_filler\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling precip depth\nprecip_depth_filler = pd.DataFrame(data_train.groupby(['site_id','hour','month'])['precip_depth_1_hr'].mean(),columns=['precip_depth_1_hr'])\ndata_train.update(precip_depth_filler,overwrite=False)\ndel precip_depth_filler\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.reset_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_statistics(data_train) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Consumption measures"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train['meter'].unique()\n#{0: 'electricity', 1: 'chilledwater', 2: 'steam', 3: 'hotwater'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train['meter'].value_counts().unique() #observations per feature [0,1,2,3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train[data_train['meter']== 3][\"meter\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"data_train['meter'].replace({0:\"electricité\",1:\"eau froide\",2:\"vapeur\",3:\"eau chaude\"},inplace=True)\nsns.countplot(x= \"meter\",data = data_train,palette =\"Set2\");","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test data description "},{"metadata":{"trusted":true},"cell_type":"code","source":"data_test.describe().transpose() #observations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"missing_statistics(data_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"    nullseries = data_test.isnull().sum()\n    onlynull= nullseries[nullseries > 0]\n    plt.figure(figsize=(10,5));\n    sns.barplot(x=onlynull.index , y=onlynull*100/len(data_test));\n    plt.ylabel(\"PERCENTAGE NAN DATA in Test\")\n    plt.xlabel(\"COLUMN NAME\");\n    plt.xticks(rotation=90);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualisations"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, figsize=(15, 10))\nlab1 = ['train data']\nlab2 = ['test data']\nn, bins, patches = ax.hist(data_test.site_id, bins=15,color='r', edgecolor='white',alpha=0.6,label = lab2)\nn1, bins1, patches1 = ax.hist(data_train.site_id, bins=15,color='b', edgecolor='white',alpha=0.5,label = lab1)\n\n# Annotate each bar with the no. of buildings in that site:\nfor number, b in zip(n, bins[:-1]):\n    ax.annotate(int(number), \n                 xy=(b+.5, number), xytext=(0, 1),#1 point vertical offset\n                 textcoords=\"offset points\",\n                 ha='center', va='bottom', fontsize=12)\n# Annotate each bar with the no. of buildings in that site:\nfor number1, b1 in zip(n1, bins1[:-1]):\n    ax.annotate(int(number1), \n                 xy=(b1 +.5, number1), xytext=(0, 1),#1 point vertical offset\n                 textcoords=\"offset points\",\n                 ha='center', va='bottom', fontsize=12)\n\nax.legend(prop ={'size': 10}) \nax.set_xlabel('site_id')\nax.set_ylabel('number of buildings')\nax.set_title('Occurrence of buildings on each site', fontsize=16);\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get a list of primary uses and its length\nprim_use_list = data_train['primary_use'].unique()\nlen(prim_use_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building.groupby(['primary_use','site_id']).size().unstack().fillna(0).astype(int).style.background_gradient(axis=None)\nbuilding.groupby(['primary_use']).size().to_frame('number_buildings').fillna(0).style.background_gradient(axis=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use = data_train.groupby(\"primary_use\").meter_reading.mean()\nsns.barplot(y=use.index,x=use)\ndel use","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Group by primary use and plot time series profiles\nfig, axes = plt.subplots(8, 2, figsize=(20, 35))\n\n\n# Daily energy use for each building\nedu_daily = data_train.groupby(['building_id', data_train['timestamp'].dt.date])['meter_reading'].sum()\nedu_daily = edu_daily.reset_index()\nedu_mean = edu_daily.groupby('timestamp')['meter_reading'].mean()\naxes[0, 0].plot(edu_mean.index, edu_mean)\n\n# For the rest of the building types we will write a loop for batch ploting:\nfor ax, use in zip(axes.flat[1:], prim_use_list[1:]): \n    prim_use_df = data_train[data_train['primary_use']==use]\n    prim_use_daily = prim_use_df.groupby(['building_id', prim_use_df['timestamp'].dt.date])['meter_reading'].sum()\n    prim_use_daily = prim_use_daily.reset_index()\n    mean = prim_use_daily.groupby('timestamp')['meter_reading'].mean()\n    \n    ax.plot(mean.index, mean)\n    ax.set_title(use)\n\nfig.add_subplot(111, frameon=False)\n# hide tick and tick label of the big axes\nplt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)\nplt.grid(False)\nplt.xlabel('Time')\nplt.ylabel('Meter Reading (Daily Sum)', labelpad=20)\n\nplt.title('Time series profiles for different building types', pad=30)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"boxplot = data_train.boxplot(column=['air_temperature','dew_temperature','wind_direction','wind_speed']);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del building , boxplot, fig, prim_use_daily \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_test[\"datetime\"] = pd.to_datetime(data_test[\"timestamp\"])\ndata_test[\"hour\"] = data_test[\"datetime\"].dt.hour;\ndata_test[\"week\"] = data_test[\"datetime\"].dt.week;\ndata_test[\"month\"] = data_test[\"datetime\"].dt.month;\ndata_test[\"day\"] = data_test[\"datetime\"].dt.day;","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"drop_features = [\"datetime\"] \n\ndata_train.drop(drop_features, axis=1, inplace=True)\ndata_test.drop(drop_features, axis=1, inplace=True)\ndel drop_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train['meter'].replace({\"electricité\":0,\"eau froide\":1,\"vapeur\":2,\"eau chaude\":3},inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_test.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Save processed data in the output "},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train.to_csv('data_training.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_test.to_csv('test_data.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}