{"cells":[{"metadata":{},"cell_type":"markdown","source":"<U><CENTER><H1>ASHRAE - EDA - LGBM  - BAGGING REGRESSION</H1></CENTER></U>"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import gc\nimport os\nimport random\nimport csv\nimport sys\n\nimport lightgbm as lgb\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\n\nfrom matplotlib import pyplot as plt\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.ensemble import BaggingRegressor\n\nfrom sklearn.experimental import enable_iterative_imputer\nfrom sklearn.impute import IterativeImputer\n\nimport time\nfrom datetime import datetime, timedelta\nimport pickle","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def preparationDonnees(data, building, weather, encoder=None, imputer=None, seuils=None):\n    t0 = time.time()\n    building = reduce_mem_usage(pd.read_csv(building))\n    #Traitement des outliers\n    seuil_min, seuil_max = seuil_min_max(building[\"square_feet\"], 2)\n    building[\"square_feet\"] = replaceOutliers(building[\"square_feet\"], seuil_min, seuil_max)\n    building.drop([\"year_built\", \"floor_count\"], axis=1, inplace=True)\n    #arrondi de la colonne square_feet\n    building[\"square_feet\"] = building[\"square_feet\"].apply(lambda x: int(x / 10) * 10)\n    building[\"square_feet\"] = np.log1p(building[\"square_feet\"])\n    \n    col_weather = [\"air_temperature\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\"]\n    weather = reduce_mem_usage(pd.read_csv(weather))\n    weather.drop([\"precip_depth_1_hr\", \"cloud_coverage\", \"dew_temperature\"], axis=1, inplace=True)\n    weather[\"timestamp\"] = weather[\"timestamp\"].apply(lambda x: datetime.strptime(x, \"%Y-%m-%d %H:%M:%S\"))\n    \n    if encoder is None and imputer is None:\n        encoder = LabelEncoder()\n        encoder.fit(building[\"primary_use\"])\n        building[\"primary_use\"] = encoder.transform(building[\"primary_use\"])\n        imputer = IterativeImputer()\n        imputer.fit(weather[col_weather])\n        seuils={}\n        seuil_min_air, seuil_max_air = seuil_min_max(weather[\"air_temperature\"])\n        seuil_min_pressure, seuil_max_pressure = seuil_min_max(weather[\"sea_level_pressure\"])\n        seuil_min_wind, seuil_max_wind = seuil_min_max(weather[\"wind_speed\"])\n        seuils[\"seuil_min_air\"] = seuil_min_air\n        seuils[\"seuil_max_air\"] = seuil_max_air\n        seuils[\"seuil_min_pressure\"] = seuil_min_pressure\n        seuils[\"seuil_max_pressure\"] = seuil_max_pressure\n        seuils[\"seuil_min_wind\"] = seuil_min_wind\n        seuils[\"seuil_max_wind\"] = seuil_max_wind      \n    else:\n        building[\"primary_use\"] = encoder.transform(building[\"primary_use\"])\n        seuil_min_air = seuils[\"seuil_min_air\"]\n        seuil_max_air = seuils[\"seuil_max_air\"]\n        seuil_min_pressure = seuils[\"seuil_min_pressure\"]\n        seuil_max_pressure = seuils[\"seuil_max_pressure\"]\n        seuil_min_wind = seuils[\"seuil_min_wind\"]\n        seuil_max_wind = seuils[\"seuil_max_wind\"]\n        \n    #### DATA ####\n    data = reduce_mem_usage(pd.read_csv(data))\n    data[\"timestamp\"] = data[\"timestamp\"].apply(lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))\n    ### FUSION TRAIN ET BUILDING --> TRAIN ####\n    print(\"Fusion de 'data' et 'building'\")\n    data = pd.merge(data, building, on=\"building_id\", how=\"left\")\n    #### FUSION TRAIN ET WEATHER --> TRAIN ####\n    print(\"Fusion de data et weather\")\n    data = pd.merge(data, weather, on=[\"timestamp\", \"site_id\"], how=\"left\")\n    \n    filtre = np.logical_or(data[\"air_temperature\"].isnull(), data[\"sea_level_pressure\"].isnull())\n    filtre2 = np.logical_or(data[\"wind_direction\"].isnull(), data[\"wind_speed\"].isnull())\n    filtre = np.logical_or(filtre, filtre2)\n    \n    d2 = data[filtre][col_weather]\n    d = imputer.transform(d2)\n    d2 = pd.DataFrame(data=d, index=d2.index, columns=col_weather)\n    data.update(d2)\n    \n    d2 = None\n    d = None\n    filtre = None\n    filtre2 = None\n    weather = None\n    nuilding = None\n    \n    #Ecrasement des outliers\n    print(\"Remplacements des outliers\")\n    data[\"air_temperature\"] = replaceOutliers(data[\"air_temperature\"], seuil_min_air, seuil_max_air)\n    data[\"sea_level_pressure\"] = replaceOutliers(data[\"sea_level_pressure\"], seuil_min_pressure, seuil_max_pressure)\n    data[\"wind_speed\"] = replaceOutliers(data[\"wind_speed\"], seuil_min_wind, seuil_max_wind)\n    #arrondi de la temperature\n    data[\"air_temperature\"] = np.round(data[\"air_temperature\"], decimals=1)\n    #Arrondir la direction du vent par dizaine\n    data[\"wind_direction\"] = data[\"wind_direction\"].apply(lambda x: int(round(x / 10, 0) * 10))\n    #remplacer 360 par 0\n    data[\"wind_direction\"] = data[\"wind_direction\"].apply(lambda x: 0 if x == 360 else x)\n\n    data = reduce_mem_usage(data)\n    \n    if \"meter_reading\" in list(data.columns):\n        data = data[['timestamp', 'site_id', 'building_id', 'meter', 'meter_reading',\n                 'primary_use', 'square_feet', 'air_temperature',\n                 'sea_level_pressure', 'wind_direction', 'wind_speed']]\n    else:\n        data = data[['timestamp', 'site_id', 'building_id', 'meter',\n                 'primary_use', 'square_feet', 'air_temperature',\n                 'sea_level_pressure', 'wind_direction', 'wind_speed']]\n    \n    \n    #Création des colonnes MONTH, DAY et HOUR\n    print(\"Création de la colonne Month\")\n    data[\"MONTH\"] = data[\"timestamp\"].apply(lambda x: x.month)\n    print(\"Création de la colonne Day\")\n    data[\"DAY\"] = data[\"timestamp\"].apply(lambda x: x.day)\n    print(\"Création de Hour\")\n    data[\"HOUR\"] = data[\"timestamp\"].apply(lambda x: x.hour)\n    print(\"Création du jour de la semaine\")\n    data[\"DAYOFWEEK\"] = data[\"timestamp\"].apply(lambda x: x.dayofweek)\n    \n    #Suppression de la colonne TIMESTAMP\n    #data.drop(\"timestamp\", axis=1, inplace=True)\n    \n    #Suppression de la colonne building_id\n    #data.drop(\"building_id\", axis=1, inplace=True)\n    \n    try:\n        data.drop(\"row_id\", axis=1, inplace=True)\n    except:\n        pass\n    \n    #Suppression des lignes avec 0 en meter_reading\n    #if \"meter_reading\" in data.columns:\n        #print(\"Nombre de lignes avec meter_reading à 0: {}\".format(len(data[data[\"meter_reading\"] <= 0])))\n        #data = data[data[\"meter_reading\"] > 0]\n    \n    print(\"Réduction de la place mémoire\")\n    data = reduce_mem_usage(data)\n    \n    #suppression des données en double (data leakage)\n    if \"meter_reading\" in data.columns:\n        target = data[\"meter_reading\"]\n        data.drop(\"meter_reading\", axis=1, inplace=True)\n        data.drop_duplicates(inplace=True)\n        filtre = data.duplicated()\n        print(\"Nombre de ligne en double dans le train: {}\".format(filtre.sum()))\n        data = data[np.logical_not(filtre)]\n        target = target[data.index]\n        target = np.log1p(target)\n    else:\n        target = None\n        \n    print(\"Durée exécution: {:0.2f} secondes\".format(time.time() - t0))\n        \n    return data, target, encoder, imputer, seuils","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def seuil_min_max(data_series, coef=1.5):\n    seuil_max = data_series.quantile(0.75) + (data_series.quantile(0.75) - data_series.quantile(0.25)) * coef\n    seuil_min = data_series.quantile(0.25) - (data_series.quantile(0.75) - data_series.quantile(0.25)) * coef\n    print(\"Seuil min {}: {}\".format(data_series.name, seuil_min))\n    print(\"Seuil max {}: {}\".format(data_series.name, seuil_max))\n    #Proportion des outliers\n    filtre = np.logical_or(data_series >= seuil_max, data_series <= seuil_min)\n    print(\"Proportion outliers: {:0.2f}%\".format(len(data_series[filtre]) / len(data_series) * 100))\n    return seuil_min, seuil_max\n\n\ndef replaceOutliers(data_series, seuil_min, seuil_max):\n    #Remplace des extremes par les seuils\n    data_series = data_series.apply(lambda x: seuil_min if x <= seuil_min else x)\n    data_series = data_series.apply(lambda x: seuil_max if x >= seuil_max else x)\n    return data_series","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"def prepare_data(X, building_data, weather_data, test=False):\n    \"\"\"\n    Preparing final dataset with all features.\n    \"\"\"\n    \n    X = X.merge(building_data, on=\"building_id\", how=\"left\")\n    X = X.merge(weather_data, on=[\"site_id\", \"timestamp\"], how=\"left\")\n    \n    X.timestamp = pd.to_datetime(X.timestamp, format=\"%Y-%m-%d %H:%M:%S\")\n    X.square_feet = np.log1p(X.square_feet)\n    \n    if not test:\n        X.sort_values(\"timestamp\", inplace=True)\n        X.reset_index(drop=True, inplace=True)\n    \n    gc.collect()\n    \n    holidays = [\"2016-01-01\", \"2016-01-18\", \"2016-02-15\", \"2016-05-30\", \"2016-07-04\",\n                \"2016-09-05\", \"2016-10-10\", \"2016-11-11\", \"2016-11-24\", \"2016-12-26\",\n                \"2017-01-01\", \"2017-01-16\", \"2017-02-20\", \"2017-05-29\", \"2017-07-04\",\n                \"2017-09-04\", \"2017-10-09\", \"2017-11-10\", \"2017-11-23\", \"2017-12-25\",\n                \"2018-01-01\", \"2018-01-15\", \"2018-02-19\", \"2018-05-28\", \"2018-07-04\",\n                \"2018-09-03\", \"2018-10-08\", \"2018-11-12\", \"2018-11-22\", \"2018-12-25\",\n                \"2019-01-01\"]\n    \n    X[\"month\"] = X.timestamp.dt.month #Rajout\n    X[\"day\"] = X.timestamp.dt.day #Rajout\n    X[\"hour\"] = X.timestamp.dt.hour\n    X[\"weekday\"] = X.timestamp.dt.weekday\n    #X[\"is_holiday\"] = (X.timestamp.dt.date.astype(\"str\").isin(holidays)).astype(int)\n    \n    #suppression en plus de la caratéristique \"dew_temperature\"\n    drop_features = [ \"dew_temperature\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\",\"year_built\",\"floor_count\",\"cloud_coverage\", \"precip_depth_1_hr\"]\n\n    X.drop(drop_features, axis=1, inplace=True)\n\n    if test:\n        row_ids = X.row_id\n        X.drop(\"row_id\", axis=1, inplace=True)\n        return X, row_ids\n    else:\n        y = np.log1p(X.meter_reading)\n        X.drop(\"meter_reading\", axis=1, inplace=True)\n        return X, y","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"plt.style.use(\"seaborn\")\nsns.set(font_scale=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"path_data = \"/kaggle/input/ashrae-energy-prediction/\"\npath_train = path_data + \"train.csv\"\npath_test = path_data + \"test.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"\npath_weather_test = path_data + \"weather_test.csv\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H2>1 - EXPLORATORY DATA ANALYSIS</H2>"},{"metadata":{},"cell_type":"markdown","source":"<H3>1.1 ANALYSIS OF THE \"building_metadata.csv\" FILE </H3>"},{"metadata":{"trusted":false},"cell_type":"code","source":"building = reduce_mem_usage(pd.read_csv(path_building))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"6 features:\n- site id\n- building id\n- site type\n- year built\n- number of floors"},{"metadata":{"trusted":false},"cell_type":"code","source":"building.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It is a database describing 1500 separate buildings."},{"metadata":{},"cell_type":"markdown","source":"We observe missing data for the columns:\n- \"year_built\"\n- \"flour_count\""},{"metadata":{},"cell_type":"markdown","source":"<H4>1.1.1 ANALYSIS OF MISSING DATA</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"col = list(building.columns)\npct_missing = [building[c].isnull().sum() / len(building) * 100 for c in building.columns]\ndf_missing = pd.DataFrame({\"Name feature\": col, \"pct_missing\": pct_missing})\ndf_missing.set_index(\"Name feature\", inplace=True, drop=True)\nplt.figure(figsize=(10, 6))\nplt.bar(df_missing.index, df_missing.pct_missing)\nfor i, n in enumerate(list(df_missing.index)):\n    plt.text(i, df_missing.loc[n]['pct_missing'], s=\"{:0.2f}%\".format(df_missing.loc[n]['pct_missing']), horizontalalignment=\"center\")\nplt.xticks(rotation=45, horizontalalignment=\"right\")\nplt.ylabel(\"Percentage of missing data\")\nplt.title(\"Proportion of missing data by feature\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"labelEncoder = LabelEncoder()\nlabelEncoder.fit(building[\"primary_use\"])\nbuilding[\"primary_use\"] = labelEncoder.transform(building[\"primary_use\"])\nbuilding.set_index(\"building_id\", drop=True, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(10, 10))\nax = sns.heatmap(building.corr(), fmt=\".2f\", annot=True, ax=ax, cmap=\"RdBu_r\", vmin=-1, vmax=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"There is no significant correlation between the criteria in this table except between the characteristics \"floor_count\" and \"square_feet\"."},{"metadata":{},"cell_type":"markdown","source":"In addition we note that several \"site_id\" do not have the data \"year_built\" and / or \"floor_count\"."},{"metadata":{"trusted":false},"cell_type":"code","source":"df = building[[\"site_id\", \"primary_use\", \"year_built\", \"floor_count\"]].groupby(\"site_id\").count()\ndf[\"%_missing_data_year_built\"] = df[[\"primary_use\", \"year_built\"]].apply(lambda x: round((x[\"primary_use\"] - x[\"year_built\"]) / x[\"primary_use\"] * 100, 2), axis=1)\ndf[\"%_missing_data_floor_count\"] = df[[\"primary_use\", \"floor_count\"]].apply(lambda x: round((x[\"primary_use\"] - x[\"floor_count\"]) / x[\"primary_use\"] * 100, 2), axis=1)\n\n\nplt.figure(figsize=(12, 6))\nplt.bar(df.index - 0.2, df[\"%_missing_data_year_built\"], width=0.4, label=\"year_built\")\nplt.bar(df.index +  0.2, df[\"%_missing_data_floor_count\"], width=0.4, label=\"floor_count\")\nplt.xticks(range(len(df)))\nplt.xlabel(\"site_id\")\nplt.ylabel(\"%\")\nplt.legend()\nplt.title(\"Proportion of missing data for the year_built and floor_count columns for each site\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We note that some sites have no data on the year of construction or the number of floors.<br>\nBut generally, the base is unbalanced concerning these 2 features."},{"metadata":{},"cell_type":"markdown","source":"<H4>1.1.2 REPARTITION OF TYPES OF BUILDINGS BY SITE</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"\"Number of types of building: {}\".format(len(set(building[\"primary_use\"])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"The different types of building:\")\nfor t in set(building[\"primary_use\"]):\n    print(\"- \" + labelEncoder.classes_[t])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df = building[[\"site_id\", \"primary_use\"]].groupby(\"site_id\").agg({\"primary_use\":[\"nunique\", \"count\"]})\n\nfig, ax1 = plt.subplots(figsize=(15, 8))\n\nax1.bar(df.index - 0.2, df[\"primary_use\"][\"nunique\"], width=0.4, color=\"orange\", label=\"Number of building types\")\nfor i, row in df.iterrows():\n    ax1.text(i - 0.2, row[\"primary_use\"][\"nunique\"] + 0.2, s=row[\"primary_use\"][\"nunique\"], horizontalalignment=\"center\")\n\nax2 = ax1.twinx()\n\nax2.bar(df.index + 0.2, df[\"primary_use\"][\"count\"], width=0.4, label=\"Number of buildings\")\nfor i, row in df.iterrows():\n    plt.text(i + 0.2, row[\"primary_use\"][\"count\"], s=row[\"primary_use\"][\"count\"], horizontalalignment=\"center\")\n\nax1.set_xlabel(\"site_id\")\nax1.set_ylabel(\"Number of building types\")\nax2.set_ylabel(\"Number of buildings per site\")\nplt.xticks(range(16))\nax1.legend()\nax2.legend()\nax1.grid(visible=False)\nax2.grid(visible=False)\nplt.title(\"Number of building types per site\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Once again we note an unbalanced database on the distribution of observations according to the site_id: the number of building types between sites is very variable.<br>\nWe must ask ourselves the question whether or not to keep the criteria of site_id in the rest of this project?"},{"metadata":{"trusted":false},"cell_type":"code","source":"df = building[[\"primary_use\", \"site_id\"]].groupby(\"primary_use\").count()\ndf.sort_values(\"site_id\", ascending=False, inplace=True)\ndf.reset_index(inplace=True, drop=False)\ndf[\"percentage\"] = df[\"site_id\"].apply(lambda x: round((x / df[\"site_id\"].sum()) * 100, 2))\nplt.figure(figsize=(14, 5))\nplt.bar(df.index, df[\"site_id\"])\nfor i, n in enumerate(list(df.index)):\n    plt.text(i, df.iloc[i][\"site_id\"], s=\"{}%\".format(df.iloc[i][\"percentage\"]), horizontalalignment=\"center\")\nplt.xticks(range(len(df)), df[\"primary_use\"])\nplt.ylabel(\"Number of sites\")\nplt.title(\"Number of buildings per site\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The site_id 0 contains more than a third of the observations !!!"},{"metadata":{},"cell_type":"markdown","source":"<H4>1.1.3 ANALYSIS SQUARE FEET AND THE OUTLIERS</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"building[[\"square_feet\"]].boxplot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are going to replace the outliers with the value corresponding to the maximum threshold defined by the boxplot."},{"metadata":{"trusted":false},"cell_type":"code","source":"#Remplacement des outliers par le seuil\nseuil_min, seuil_max = seuil_min_max(building[\"square_feet\"], 2)\nbuilding[\"square_feet\"] = replaceOutliers(building[\"square_feet\"], seuil_min, seuil_max)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"building[[\"square_feet\"]].describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>1.1.5 RECOMMENDATION FOR FEATURES DELETION</H4>"},{"metadata":{},"cell_type":"markdown","source":"In view of the analyzes carried out so far, I propose to eliminate the features below because the available data are insufficient:\n+ year_built\n+ floor_count"},{"metadata":{"trusted":false},"cell_type":"code","source":"building.drop([\"year_built\", \"floor_count\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H3>1.2 WEATHER TRAIN ANALYSIS </H3></H3>"},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train = reduce_mem_usage(pd.read_csv(path_weather_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This database displays hourly weather data for 2016.<br>\nBy performing a quick calculation, we note that we are missing observations.<br>\nIn this database we have 139 773 observations.<br>\nIf we multiply 366 days (in 2016) by 24 hours by 16 site_id, we should have 140 544 observations.<br>\nhe is therefore missing 771 observations."},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.1 PERIOD ANALYSIS</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train[\"timestamp\"] = pd.to_datetime(weather_train[\"timestamp\"], format=\"%Y-%m-%d %H:%M:%S\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(\"First Date: {}\".format(weather_train[\"timestamp\"].min()))\nprint(\"Last Date: {}\".format(weather_train[\"timestamp\"].max()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.2 ANALYSIS OF MISSING DATA</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"col = []\npct = []\nfor c in weather_train.columns:\n    col.append(c)\n    pct.append(weather_train[c].isnull().sum() / len(weather_train) * 100)\n\na = pd.DataFrame({\"colonne\": col, \"pct\": pct}).set_index(\"colonne\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nplt.bar(a.index, a[\"pct\"])\nfor i, c, in enumerate(a.index):\n    y = a.iloc[i][\"pct\"]\n    plt.text(i, y=y, s=\"{:0.2f}%\".format(y), horizontalalignment=\"center\")\n\nplt.xticks(rotation =45, horizontalalignment=\"right\")\n\nplt.ylabel(\"Percentage\")\nplt.title(\"Percentage of missing data by data type\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We find that the data is missing very significantly for the features:\n+ Cloud_coverage\n+ precip_depth_1_hr"},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.2.1 ANALYSIS OF MISSING DATA FOR precip_depth_1_hr</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train[[\"precip_depth_1_hr\"]].dropna().describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Replacement -1 by 0\nweather_train['precip_depth_1_hr'] = np.where(weather_train[\"precip_depth_1_hr\"] == -1, 0, weather_train[\"precip_depth_1_hr\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#I want to know how many observation with rain in the entire database Weather_train\nweather_train[\"rain\"] = weather_train[\"precip_depth_1_hr\"].dropna().apply(lambda x: 1 if x > 0 else 0)\nplt.figure(figsize=(6, 6))\nplt.pie(weather_train[[\"rain\", \"site_id\"]].dropna().groupby(\"rain\").count(), \n        explode=(0, 0.2), \n        labels=[\"without rain\", \"rain\"], \n        shadow=True,\n        autopct='%1.1f%%')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In view of the results, we eliminate:\n+ precip_depth_1_hr\n+ cloud_coverage"},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train.drop([\"precip_depth_1_hr\", \"cloud_coverage\", \"rain\"], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.2 CROSS-VARIABLE ANALYSIS</H4>"},{"metadata":{},"cell_type":"markdown","source":"For the other features, we are going to replace the missing data with help of iterativeImputer from sklearn."},{"metadata":{"trusted":false},"cell_type":"code","source":"iterativeImputer = IterativeImputer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"iterativeImputer.fit(weather_train[['site_id', 'dew_temperature', 'air_temperature', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']])\nweather_train[['site_id', 'dew_temperature', 'air_temperature', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']] = iterativeImputer.transform(weather_train[['site_id', 'dew_temperature', 'air_temperature', 'sea_level_pressure',\n       'wind_direction', 'wind_speed']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(8, 8))\nsns.heatmap(weather_train.corr(), fmt=\".2f\", annot=True, ax=ax, cmap=\"RdBu_r\", vmin=-1, vmax=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Strong correlation between dew_temperature and air_temperature We deviate from dew_temperature.\n\nWe eliminate \"dew_temperature\""},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train.drop(\"dew_temperature\", axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.3 ANALYZE AIR TEMPERATURE</H4>"},{"metadata":{},"cell_type":"markdown","source":"In th same way as the outliers of the feature square_feet, we are going to replace the outliers with the value corresponding to the maximum threshold defined by the boxplot."},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train[[\"air_temperature\"]].boxplot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Remplace des extremes par les seuils\nseuil_min_air_temp, seuil_max_air_temp = seuil_min_max(weather_train[\"air_temperature\"], 1)\nweather_train[\"air_temperature\"] = replaceOutliers(weather_train[\"air_temperature\"], seuil_min_air_temp, seuil_max_air_temp)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.4 ANALYZE SEA LEVEL PRESSURE</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train[[\"sea_level_pressure\"]].boxplot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"seuil_min_sea_level_pressure, seuil_max_sea_level_pressure = seuil_min_max(weather_train[\"sea_level_pressure\"])\nweather_train[\"sea_level_pressure\"] = replaceOutliers(weather_train[\"sea_level_pressure\"], seuil_min_sea_level_pressure, seuil_max_sea_level_pressure)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.5 ANALYZE WIND SPEED</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"weather_train[[\"wind_speed\"]].boxplot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Replacement of outliers by the maximum value in the interquartile range * 1.5 * deviation type"},{"metadata":{"trusted":false},"cell_type":"code","source":"seuil_min_wind_speed, seuil_max_wind_speed = seuil_min_max(weather_train[\"wind_speed\"])\nweather_train[\"wind_speed\"] = replaceOutliers(weather_train[\"wind_speed\"], seuil_min_wind_speed, seuil_max_wind_speed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>1.2.6 ANALYZE WIND DIRECTION</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"#Arrondir la direction du vent par dizaine\nweather_train[\"wind_direction\"] = weather_train[\"wind_direction\"].apply(lambda x: int(round(x / 10, 0) * 10))\n#remplacer 360 par 0\nweather_train[\"wind_direction\"] = weather_train[\"wind_direction\"].apply(lambda x: 0 if x == 360 else x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"plt.hist(weather_train[\"wind_direction\"], bins=35)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Curiously, we see that the sale blows at 0 degrees in a large part of the observations."},{"metadata":{},"cell_type":"markdown","source":"<H3>1.3 FILE TRAIN ANALYSIS</H3>"},{"metadata":{},"cell_type":"markdown","source":"We collect the train, weather_train and building tables. We perform data cleaning thanks to the analyzes previously carried out"},{"metadata":{"trusted":false},"cell_type":"code","source":"train, target, encoder, imputer, seuils = preparationDonnees(path_train, path_building, path_weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train[\"meter_reading\"] = target","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"features = [\"building\", \"meter\", \"count\", \"mean\", \"std\", \"min\", \"25%\", \"50%\", \"75%\", \"maxi\"]\ndf_building = pd.DataFrame(columns=features)\n\ncompteur = 0\nfor i in range(1500):\n    for j in range(4):\n        train_building = train[np.logical_and(train[\"building_id\"] == i,  train[\"meter\"] == j)]\n        df = train_building[[\"meter_reading\", \"MONTH\"]].groupby(\"MONTH\").sum()\n        df_building.loc[compteur] = [i] + [j] + list(df.describe()[\"meter_reading\"])\n        compteur += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_building.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We observe a large number of missing data"},{"metadata":{},"cell_type":"markdown","source":"Feature creation to detect anormal data"},{"metadata":{"trusted":false},"cell_type":"code","source":"df_building[\"std / mean\"] = df_building[\"std\"] / df_building[\"mean\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Building_id without data\nbuilding_id_without_data = []\nfor i in range(1500):\n    df_b = df_building[df_building[\"building\"] == i]\n    if df_b[\"std\"].isnull().sum() == 4:\n        building_id_without_data.append(i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\"Number buiding without data: {} ({:0.2f}%)\".format(len(building_id_without_data), len(building_id_without_data) / 1500 * 100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Elimination of the lines for the building without data"},{"metadata":{"trusted":false},"cell_type":"code","source":"index_to_eliminate = []\nfor b in building_id_without_data:\n    df = train[train[\"building_id\"] == b]\n    index_to_eliminate.extend(list(df.index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"index_to_keep = set(train.index).difference(set(index_to_eliminate))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train = train[train.index.isin(index_to_keep)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Data by meter\ndf_building[[\"meter\", \"std\"]].groupby(\"meter\").count().plot(kind=\"bar\")\nplt.title(\"Number building with data by meter\")\nmeter = {0: \"electricity\", 1: \"chilledwater\", 2: \"steam\", 3: \"hotwater\"}\nplt.xticks(range(4), [\"electricity\", \"chilledwater\", \"steam\", \"hotwater\"], rotation=45, horizontalalignment=\"right\")\nplt.ylabel(\"Number of building\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#outliers \ndf_building[[\"std / mean\"]].boxplot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"threshold_building = (df_building[[\"std / mean\"]].quantile(0.75) - df_building[[\"std / mean\"]].quantile(0.25)) + df_building[[\"std / mean\"]].quantile(0.75)\nthreshold_building = threshold_building[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_outliers = df_building[np.logical_or(df_building[\"std\"].isnull(), df_building[\"std / mean\"] > threshold_building)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Building id outliers\nnumber_graph = len(df_outliers[df_outliers[\"std / mean\"].notnull()])\nfig = plt.figure(figsize=(17, 100))\nj = 0\nfor k in range(4):\n    for i, row in df_outliers[np.logical_and(df_outliers[\"std / mean\"].notnull(), df_outliers[\"meter\"] == k)].iterrows():\n        train_building_meter = train[np.logical_and(train[\"building_id\"] == row[\"building\"], train[\"meter\"] == row[\"meter\"])]\n        fig.add_subplot(int(number_graph / 3) + 1, 3, j + 1)\n        plt.plot(train_building_meter[\"timestamp\"], train_building_meter[\"meter_reading\"])\n        plt.title(\"Building: {} - Meter: {}\".format(int(row[\"building\"]), int(row[\"meter\"])))\n        j += 1\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that a number of buildings have curious measure:\n+ For example, the first graphs present measurements which does not start until mid-May\n+ other graphs with measurements at 0 mid-year\n+ measurements with very large variations during the year"},{"metadata":{},"cell_type":"markdown","source":"We will assume that the energy consumption of buildings must be \"consistent\" throughout the year and should be above 0:\n+ if we find successive measurements at 0 during a too \"long\" period, we eliminate this observations."},{"metadata":{},"cell_type":"markdown","source":"For each building detection of periods at 0:"},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"df_traces = pd.DataFrame(columns=[\"building\", \"meter\", \"start date\", \"end date\", \"duration\"])\nfor i, row in df_outliers[df_outliers[\"std / mean\"].notnull()].iterrows():\n    df = train[np.logical_and(train[\"building_id\"] == row[\"building\"], train[\"meter\"] == row[\"meter\"])]\n    df_0 = df[df[\"meter_reading\"] == 0][[\"timestamp\"]]\n    if len(df_0) > 0:\n        df_0[\"delta\"] = df_0[\"timestamp\"].diff()\n        periode = 0\n        date1 = df_0.iloc[0][\"timestamp\"]\n        for i in range(1, len(df_0)):\n            if df_0.iloc[i][\"delta\"] == timedelta(hours=1):\n                periode += 1\n            elif periode > 0:\n                date2 = df_0.iloc[i-1][\"timestamp\"]\n                len_df_traces = len(df_traces)\n                df_traces.loc[len_df_traces + 1] = [int(row[\"building\"]), int(row[\"meter\"]), date1, date2, periode]\n                date1 = df_0.iloc[i][\"timestamp\"]\n                periode = 0\n            else:\n                date1 = df_0.iloc[i][\"timestamp\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pickle.dump(df_traces, open(\"df_traces.pickle\", \"wb\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df_traces = df_traces.sort_values(\"duration\", ascending=False)\n\nplt.plot(range(len(df_traces)), df_traces[\"duration\"])\nplt.xlim(-10, 1000)\nplt.ylabel(\"number of consecutive hours with zero energy consumption\")\nplt.xlabel(\"Observation\")\nplt.title(\"Decreasing ranking of periods with zero energy consumption on Outliers buildings\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will define the threshold from which a period with energy consumption at 0 is abnormal.<br>\nThe choice will be made with the help of the graphic above.<br>\nArbitrarily the choice of the threshold is made at the place where the curve begins to flatten: it located approximately at the abscissa 600 which corresponds to a period of 117 hours or 5 days."},{"metadata":{},"cell_type":"markdown","source":"We will therefore delete the observations corresponding to the previously specified threshold."},{"metadata":{"trusted":false},"cell_type":"code","source":"df_traces_to_eliminate = df_traces[df_traces[\"duration\"] >= 117]\ndf_traces_to_eliminate","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Identification of the lines to be deleted in the TRAIN."},{"metadata":{"trusted":false},"cell_type":"code","source":"index_to_eliminate = []\nfor i, row in df_traces_to_eliminate.iterrows():\n    date1 = row[\"start date\"]\n    date2 = row[\"end date\"]\n    df_temp = train[np.logical_and(train[\"building_id\"] == row[\"building\"], train[\"meter\"] == row[\"meter\"])]\n    df_temp = df_temp[np.logical_and(df_temp[\"timestamp\"] >= date1, df_temp[\"timestamp\"] <= date2)]\n    index_to_eliminate.extend(list(df_temp.index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"index_to_keep = set(train.index).difference(set(index_to_eliminate))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train = train[train.index.isin(index_to_keep)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<B><H3>1.4 CONCLUSION OF THE EXPLORATORY ANALYSIS</H3><B>"},{"metadata":{},"cell_type":"markdown","source":"<H4>1.4.1 BUILDING Table</H4>"},{"metadata":{},"cell_type":"markdown","source":"1 - In view of the analyzes carried out so far, I propose to eliminate the features below because the available data are insufficient:\n+ <b><u>year_built</u></b>\n+ <b><u>floor_count</u></b>"},{"metadata":{},"cell_type":"markdown","source":"2 - We replace the outliers of the <b><u>square_feet</u></b> column"},{"metadata":{},"cell_type":"markdown","source":"3 - I propose to eliminate the <u><b>building</b></u> id and <u><b>site_id</b></u> features due to the unbalanced dataset"},{"metadata":{},"cell_type":"markdown","source":"<H4>1.4.2 WEATHER_TRAIN Table</H4>"},{"metadata":{},"cell_type":"markdown","source":"1 - In view of the results, we eliminate:\n+ precip_depth_1_hr\n+ cloud_coverage"},{"metadata":{},"cell_type":"markdown","source":"2 - Strong correlation between dew_temperature and air_temperature We deviate from dew_temperature.\n\nWe eliminate \"dew_temperature\""},{"metadata":{},"cell_type":"markdown","source":"3 - We replace the outliers for the features:\n+ air_temperature\n+ sea_level_pressure\n+ wind_speed"},{"metadata":{},"cell_type":"markdown","source":"<H4>1.4.3 TRAIN Table</H4>"},{"metadata":{},"cell_type":"markdown","source":"1 - We eliminate 52 building without data<br>\n2 - We eliminate observations for periods longer than 5 days when the meter_reading value is 0"},{"metadata":{},"cell_type":"markdown","source":"<H2>2 - CONSTRUCTION OF THE LGBM MODEL</H2>"},{"metadata":{"trusted":false},"cell_type":"code","source":"def dataPreparation(data, building, weather, encoder=None, imputer=None, seuils=None):\n    t0 = time.time()\n    \n    #Preparation of the BUILDING table\n    print(\"Preparation of the BUILDING table\")\n    building = reduce_mem_usage(pd.read_csv(building))\n    #Treatment of the outliers\n    seuil_min, seuil_max = seuil_min_max(building[\"square_feet\"], 2)\n    building[\"square_feet\"] = replaceOutliers(building[\"square_feet\"], seuil_min, seuil_max)\n    building.drop([\"year_built\", \"floor_count\"], axis=1, inplace=True)\n    #value rounding of the column square_feet\n    building[\"square_feet\"] = building[\"square_feet\"].apply(lambda x: int(x / 10) * 10)\n    building[\"square_feet\"] = np.log1p(building[\"square_feet\"])\n    \n    #Preparation of the Weather table\n    print(\"Preparation of the Weather table\")\n    col_weather = [\"air_temperature\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\"]\n    weather = reduce_mem_usage(pd.read_csv(weather))\n    weather.drop([\"precip_depth_1_hr\", \"cloud_coverage\", \"dew_temperature\"], axis=1, inplace=True)\n    weather[\"timestamp\"] = weather[\"timestamp\"].apply(lambda x: datetime.strptime(x, \"%Y-%m-%d %H:%M:%S\"))\n    if encoder is None and imputer is None:\n        encoder = LabelEncoder()\n        encoder.fit(building[\"primary_use\"])\n        building[\"primary_use\"] = encoder.transform(building[\"primary_use\"])\n        imputer = IterativeImputer()\n        imputer.fit(weather[col_weather])\n        seuils={}\n        seuil_min_air, seuil_max_air = seuil_min_max(weather[\"air_temperature\"])\n        seuil_min_pressure, seuil_max_pressure = seuil_min_max(weather[\"sea_level_pressure\"])\n        seuil_min_wind, seuil_max_wind = seuil_min_max(weather[\"wind_speed\"])\n        seuils[\"seuil_min_air\"] = seuil_min_air\n        seuils[\"seuil_max_air\"] = seuil_max_air\n        seuils[\"seuil_min_pressure\"] = seuil_min_pressure\n        seuils[\"seuil_max_pressure\"] = seuil_max_pressure\n        seuils[\"seuil_min_wind\"] = seuil_min_wind\n        seuils[\"seuil_max_wind\"] = seuil_max_wind    \n    else:\n        building[\"primary_use\"] = encoder.transform(building[\"primary_use\"])\n        seuil_min_air = seuils[\"seuil_min_air\"]\n        seuil_max_air = seuils[\"seuil_max_air\"]\n        seuil_min_pressure = seuils[\"seuil_min_pressure\"]\n        seuil_max_pressure = seuils[\"seuil_max_pressure\"]\n        seuil_min_wind = seuils[\"seuil_min_wind\"]\n        seuil_max_wind = seuils[\"seuil_max_wind\"]\n    \n    #### DATA ####\n    print(\"MERGE TABLES DATA, BUILDING AND WEATHER\")\n    data = reduce_mem_usage(pd.read_csv(data))\n    data[\"timestamp\"] = data[\"timestamp\"].apply(lambda x: datetime.strptime(x, '%Y-%m-%d %H:%M:%S'))\n    ### FUSION TRAIN ET BUILDING --> TRAIN ####\n    print(\"Fusion de 'data' et 'building'\")\n    data = pd.merge(data, building, on=\"building_id\", how=\"left\")\n    #### FUSION TRAIN ET WEATHER --> TRAIN ####\n    print(\"Fusion de data et weather\")\n    data = pd.merge(data, weather, on=[\"timestamp\", \"site_id\"], how=\"left\")\n    \n    #Création des colonnes MONTH, DAY et HOUR\n    print(\"Création de la colonne Month\")\n    data[\"MONTH\"] = data[\"timestamp\"].apply(lambda x: x.month)\n    print(\"Création de la colonne Day\")\n    data[\"DAY\"] = data[\"timestamp\"].apply(lambda x: x.day)\n    print(\"Création de Hour\")\n    data[\"HOUR\"] = data[\"timestamp\"].apply(lambda x: x.hour)\n    print(\"Création du jour de la semaine\")\n    data[\"DAYOFWEEK\"] = data[\"timestamp\"].apply(lambda x: x.dayofweek)\n    \n    #### detection of buildings with abnormal data ####\n    \n    if \"meter_reading\" in list(data.columns):\n        print(\"detection of buildings with abnormal data\")\n        features = [\"building\", \"meter\", \"count\", \"mean\", \"std\", \"min\", \"25%\", \"50%\", \"75%\", \"maxi\"]\n        df_building = pd.DataFrame(columns=features)\n\n        compteur = 0\n        for i in range(1500):\n            for j in range(4):\n                train_building = data[np.logical_and(data[\"building_id\"] == i,  data[\"meter\"] == j)]\n                df = train_building[[\"meter_reading\", \"MONTH\"]].groupby(\"MONTH\").sum()\n                df_building.loc[compteur] = [i] + [j] + list(df.describe()[\"meter_reading\"])\n                compteur += 1\n    \n        #Building_id without data\n        print(\"Building_id without data\")\n        building_id_without_data = []\n        for i in range(1500):\n            df_b = df_building[df_building[\"building\"] == i]\n            if df_b[\"std\"].isnull().sum() == 4:\n                building_id_without_data.append(i)\n        \n        print(\"Elimination of the Building without data\")\n        index_to_eliminate = []\n        for b in building_id_without_data:\n            df = data[data[\"building_id\"] == b]\n            index_to_eliminate.extend(list(df.index))\n    \n        index_to_keep = set(data.index).difference(set(index_to_eliminate))\n        data = data[data.index.isin(index_to_keep)]\n        \n        \n        print(\"Detection abnormal meter_reading at 0\")\n        df_building[\"std / mean\"] = df_building[\"std\"] / df_building[\"mean\"]\n        threshold_building = (df_building[[\"std / mean\"]].quantile(0.75) - df_building[[\"std / mean\"]].quantile(0.25)) + df_building[[\"std / mean\"]].quantile(0.75)\n        df_outliers = df_building[np.logical_or(df_building[\"std\"].isnull(), df_building[\"std / mean\"] > threshold_building[0])]\n    \n        df_traces = pd.DataFrame(columns=[\"building\", \"meter\", \"start date\", \"end date\", \"duration\"])\n        for i, row in df_outliers[df_outliers[\"std / mean\"].notnull()].iterrows():\n            df = data[np.logical_and(data[\"building_id\"] == row[\"building\"], data[\"meter\"] == row[\"meter\"])]\n            df_0 = df[df[\"meter_reading\"] == 0][[\"timestamp\"]]\n            if len(df_0) > 0:\n                df_0[\"delta\"] = df_0[\"timestamp\"].diff()\n                periode = 0\n                date1 = df_0.iloc[0][\"timestamp\"]\n                for i in range(1, len(df_0)):\n                    if df_0.iloc[i][\"delta\"] == timedelta(hours=1):\n                        periode += 1\n                    elif periode > 0:\n                        date2 = df_0.iloc[i-1][\"timestamp\"]\n                        len_df_traces = len(df_traces)\n                        df_traces.loc[len_df_traces + 1] = [int(row[\"building\"]), int(row[\"meter\"]), date1, date2, periode]\n                        date1 = df_0.iloc[i][\"timestamp\"]\n                        periode = 0\n                    else:\n                        date1 = df_0.iloc[i][\"timestamp\"]\n        \n        print(\"Elimination of observations whose meter_reading is at 0 for more than 5 days\")\n        df_traces_to_eliminate = df_traces[df_traces[\"duration\"] >= 117]\n        \n        index_to_eliminate = []\n        for i, row in df_traces_to_eliminate.iterrows():\n            date1 = row[\"start date\"]\n            date2 = row[\"end date\"]\n            df_temp = data[np.logical_and(data[\"building_id\"] == row[\"building\"], data[\"meter\"] == row[\"meter\"])]\n            df_temp = df_temp[np.logical_and(df_temp[\"timestamp\"] >= date1, df_temp[\"timestamp\"] <= date2)]\n            index_to_eliminate.extend(list(df_temp.index))\n    \n        index_to_keep = set(data.index).difference(set(index_to_eliminate))\n        data = data[data.index.isin(index_to_keep)]\n\n    print('filling cells without data')\n    filtre = np.logical_or(data[\"air_temperature\"].isnull(), data[\"sea_level_pressure\"].isnull())\n    filtre2 = np.logical_or(data[\"wind_direction\"].isnull(), data[\"wind_speed\"].isnull())\n    filtre = np.logical_or(filtre, filtre2)\n    \n    d2 = data[filtre][col_weather]\n    d = imputer.transform(d2)\n    d2 = pd.DataFrame(data=d, index=d2.index, columns=col_weather)\n    data.update(d2)\n    \n    d2 = None\n    d = None\n    filtre = None\n    filtre2 = None\n    weather = None\n    building = None\n    \n    \n    #Ecrasement des outliers\n    print(\"Replacements of the outliers\")\n    data[\"air_temperature\"] = replaceOutliers(data[\"air_temperature\"], seuil_min_air, seuil_max_air)\n    data[\"sea_level_pressure\"] = replaceOutliers(data[\"sea_level_pressure\"], seuil_min_pressure, seuil_max_pressure)\n    data[\"wind_speed\"] = replaceOutliers(data[\"wind_speed\"], seuil_min_wind, seuil_max_wind)\n    #arrondi de la temperature\n    data[\"air_temperature\"] = np.round(data[\"air_temperature\"], decimals=1)\n    #Arrondir la direction du vent par dizaine\n    data[\"wind_direction\"] = data[\"wind_direction\"].apply(lambda x: int(round(x / 10, 0) * 10))\n    #remplacer 360 par 0\n    data[\"wind_direction\"] = data[\"wind_direction\"].apply(lambda x: 0 if x == 360 else x)\n\n    data = reduce_mem_usage(data)\n    \n    if \"meter_reading\" in list(data.columns):\n        data = data[['timestamp', 'site_id', 'building_id', 'meter', 'meter_reading',\n                 'primary_use', 'square_feet', 'air_temperature',\n                 'sea_level_pressure', 'wind_direction', 'wind_speed']]\n    else:\n        data = data[['timestamp', 'site_id', 'building_id', 'meter',\n                 'primary_use', 'square_feet', 'air_temperature',\n                 'sea_level_pressure', 'wind_direction', 'wind_speed']]\n    \n    #Suppression de la colonne TIMESTAMP\n    data.drop(\"timestamp\", axis=1, inplace=True)\n    \n    #Suppression de la colonne building_id\n    data.drop(\"building_id\", axis=1, inplace=True)\n    data.drop(\"site_id\", axis=1, inplace=True)\n    \n    try:\n        data.drop(\"row_id\", axis=1, inplace=True)\n    except:\n        pass\n    \n    print(\"Réduction de la place mémoire\")\n    data = reduce_mem_usage(data)\n    \n    #suppression des données en double (data leakage)\n    if \"meter_reading\" in data.columns:\n        target = data[\"meter_reading\"]\n        data.drop(\"meter_reading\", axis=1, inplace=True)\n        data.drop_duplicates(inplace=True)\n        filtre = data.duplicated()\n        print(\"Nombre de ligne en double dans le train: {}\".format(filtre.sum()))\n        data = data[np.logical_not(filtre)]\n        target = target[data.index]\n        target = np.log1p(target)\n    else:\n        target = None\n    \n    print(\"Durée exécution: {:0.2f} secondes\".format(time.time() - t0))\n    return data, target, encoder, imputer, seuils","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H3>2.1 LGBM</H3>"},{"metadata":{"trusted":false},"cell_type":"code","source":"train, target, encoder, imputer, seuils = dataPreparation(path_train, path_building, path_weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pickle.dump(train, open(\"train.pickle\", \"wb\"))\npickle.dump(target, open(\"target.pickle\", \"wb\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pickle.dump(encoder, open(\"encoder.pickle\", \"wb\"))\npickle.dump(imputer, open(\"imputer.pickle\", \"wb\"))\npickle.dump(seuils, open(\"seuils.pickle\", \"wb\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(train, target, test_size=0.3, shuffle=True, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"lgr = lgb.LGBMRegressor(n_estimators=500, learning_rate=0.1, num_leaves=512, random_state=45)\nt0 = time.time()\nlgr.fit(X_train, \n        y_train, \n        eval_set=[(X_train, y_train), (X_test, y_test)], \n        eval_metric=\"rmse\",\n        categorical_feature=[0,1,2],\n        verbose=True)\nprint(\"Durée: {:0.2f} secondes\".format(time.time() - t0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_l2, val_l2 = lgr.evals_result_[\"training\"][\"rmse\"], lgr.evals_result_[\"valid_1\"][\"rmse\"]\nplt.plot(range(len(train_l2)), train_l2, label=\"train\")\nplt.plot(range(len(val_l2)), val_l2, label=\"val\")\nplt.legend()\nplt.ylabel(\"score rmse\")\nplt.xlabel(\"iteration\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H4>2.2.2 UTILISATION DE BAGGING REGRESSOR</H4>"},{"metadata":{"trusted":false},"cell_type":"code","source":"regr = BaggingRegressor(base_estimator=lgr, n_estimators=10, verbose=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"regr.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pickle.dump(regr, open(\"regr.pickle\", \"wb\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"regr.score(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"regr.score(X_test, y_test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<H3>2.2.3 PREDICTION</H3>"},{"metadata":{"trusted":false},"cell_type":"code","source":"test, target, encoder, imputer, seuils = dataPreparation(path_test, path_building, path_weather_test, encoder, imputer, seuils)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"pickle.dump(test, open(\"test.pickle\", \"wb\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"seuils = np.linspace(start=0, stop=len(test), num=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"target_predict_regr = []\ncount=0\nfor i in range(len(seuils) - 1):\n    t0 = time.time()\n    count += 1\n    print(\"Count: \" + str(i))\n    print(int(seuils[i]), \"-\", int(seuils[i + 1] - 1))\n    seuil_1 = int(seuils[i])\n    seuil_2 = int(seuils[i + 1] - 1)\n    res = regr.predict(test.loc[seuil_1:seuil_2])\n    target_predict_regr.extend(list(res))\n    print(\"Durée execution: {:0.2f} secondes\".format(time.time() - t0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"submission = pd.DataFrame()\ntarget_predict_regr_exp = np.round(np.expm1(target_predict_regr), decimals=4)\nsubmission[\"meter_reading\"] = target_predict_regr_exp\nsubmission.index.name = \"row_id\"\nsubmission.to_csv(\"submission_regr.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":1}