{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Challenge 01 - AIRPORT FLIGHT DELAY**\n","metadata":{"id":"vqZ7nQj_SiCZ"}},{"cell_type":"markdown","source":"## Initial Setup","metadata":{"id":"OJBiMcKFS8Hh"}},{"cell_type":"code","source":"! pip install scikit-learn==0.21\n! pip install hypopt","metadata":{"id":"fiNwwuzWdbMo","executionInfo":{"status":"ok","timestamp":1619434676695,"user_tz":-120,"elapsed":6318,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"38dc7cc4-5248-4f23-f93f-d65817c6f4d2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport json\nimport warnings\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.dates as mdates\nimport seaborn as sns\nimport tensorflow as tf\nimport numpy as np\nimport tensorflow_probability as tfp\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler\nfrom hypopt import GridSearch\nfrom matplotlib import rc \n\nif not sys.warnoptions:\n    warnings.simplefilter(\"ignore\")\n\nrc('figure', **{'dpi': 200})","metadata":{"id":"ydHq0-6FddsN","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_dtype = {\n    'ID'                   : int,\n    'YEAR'                 : int,  \n    'MONTH'                : int,  \n    'DAY'                  : int,  \n    'DAY_OF_WEEK'          : int,  \n    'AIRLINE'              : str, \n    'FLIGHT_NUMBER'        : str,  \n    'TAIL_NUMBER'          : str, \n    'ORIGIN_AIRPORT'       : str, \n    'DESTINATION_AIRPORT'  : str, \n    'SCHEDULED_DEPARTURE'  : str,  \n    'DEPARTURE_TIME'       : str, \n    'DEPARTURE_DELAY'      : float,\n    'TAXI_OUT'             : str, \n    'WHEELS_OFF'           : str,\n    'SCHEDULED_TIME'       : float,\n    'AIR_TIME'             : float,\n    'DISTANCE'             : int,\n    'SCHEDULED_ARRIVAL'    : str,\n    'DIVERTED'             : int,  \n    'CANCELLED'            : int,  \n    'CANCELLATION_REASON'  : str\n}\n\ny_dtype = {\n    'ID'                   : int,\n    \"ARRIVAL_DELAY\"        : float\n}","metadata":{"id":"ekNsIcaK09PB","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset Loading","metadata":{"id":"McGd5ybAlGak"}},{"cell_type":"markdown","source":"In this section we will parse the original dataset and fix the format of some datum to have easier manageability.","metadata":{"id":"U501w3HGUQMY"}},{"cell_type":"code","source":"path = '../input/eurecom-aml-2021-challenge-1/data/'\nX_train_df = pd.read_csv(path + \"train_features.csv\", dtype=X_dtype)\ny_train_df = pd.read_csv(path + \"train_targets.csv\", dtype=y_dtype)","metadata":{"id":"9SvCUNzJdpiP","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Transform in a common time format (military format): **DEPARTURE_TIME, SCHEDULED_ARRIVAL, SCHEDULED_DEPARTURE** ","metadata":{"id":"kHH-PHjQTRhQ"}},{"cell_type":"code","source":"# Convert date-time features from str to datetime format\ndef parse_hhmm(x):\n    try: return pd.datetime.strptime(x, '%H%M')#.time()\n    except: return pd.NaT\n\nX_train_df.DEPARTURE_TIME = X_train_df.DEPARTURE_TIME.apply(parse_hhmm)\nX_train_df.SCHEDULED_ARRIVAL = X_train_df.SCHEDULED_ARRIVAL.apply(parse_hhmm)\nX_train_df.SCHEDULED_DEPARTURE = X_train_df.SCHEDULED_DEPARTURE.apply(parse_hhmm)","metadata":{"id":"fY2zotPod-KQ","executionInfo":{"status":"ok","timestamp":1619448931801,"user_tz":-120,"elapsed":175613,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"9e341546-3b78-4efb-9b3b-ec5187d9c30d","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the arrival time and departure time into minutes since the midnight\ndef minutes_since_midnight(dt):\n    return dt.hour * 60 + dt.minute\n\nX_train_df.SCHEDULED_DEPARTURE = X_train_df.SCHEDULED_DEPARTURE.apply(minutes_since_midnight)\nX_train_df.SCHEDULED_ARRIVAL = X_train_df.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\nX_train_df.DEPARTURE_TIME = X_train_df.DEPARTURE_TIME.apply(minutes_since_midnight)","metadata":{"id":"6lum20w_kFnP","executionInfo":{"status":"error","timestamp":1619448945390,"user_tz":-120,"elapsed":189200,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"1b409281-a7cd-4840-fc0b-68230fda85cc"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Create a new feature **TIME_SINCE_BEGIN_YEAR** in order to have a common time reference for all flights (usefull for sliding window metrics) ","metadata":{"id":"IdQ-dwSvTwgX"}},{"cell_type":"code","source":"m_since_b = (((X_train_df['MONTH'] - 1)*30 + (X_train_df['DAY']-1))*24 )*60 + X_train_df['DEPARTURE_TIME'] - X_train_df['DEPARTURE_DELAY']\nX_train_df.insert(1, \"TIME_SINCE_BEGIN_YEAR\", m_since_b)","metadata":{"id":"_WYrYU8rfGDs"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge feature dataframe and target dataframe for data exploration\ndf = pd.merge(X_train_df, y_train_df, on='ID')\n#df.to_pickle(\"airports_features_SCHEDULE_TIME_CHANGED.pkl\")","metadata":{"id":"74Ljic_Kd_4N"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Processing","metadata":{"id":"7QtEXzcglN6U"}},{"cell_type":"markdown","source":"In this section we will add some new features derived from other datasets or that summarize some interesting behaviours","metadata":{"id":"JWkPekJrUlkZ"}},{"cell_type":"code","source":"path = '../input/group19datasets/'\nairports_nn = pd.read_pickle(path + \"airports_nn.pkl\")\nairports = pd.read_pickle(path + \"airport_custom.pkl\")\ndf = pd.read_pickle(path +\"airports_features_SCHEDULE_TIME_CHANGED.pkl\")","metadata":{"id":"L8v9rfnodpNp","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Class Distribution","metadata":{"id":"BJpuM5QgBq15"}},{"cell_type":"markdown","source":"### Samples Distribution","metadata":{"id":"NxVk_0epDdiu"}},{"cell_type":"code","source":"n_samples = df.shape[0]\n\n# function that extract statistical parameters from a grouby objet:\ndef get_stats(group):\n    return {'min': group.min(), 'max': group.max(), 'mean': group.mean(),\n            '%samples': group.count()/n_samples * 100}\n\n# Creation of a dataframe with statitical infos on each airline:\nglobal_stats = df['DEPARTURE_DELAY'].groupby(df['AIRLINE']).apply(get_stats).unstack()\nglobal_stats = global_stats.sort_values('%samples')\nglobal_stats","metadata":{"id":"UR0EHmHUEm3G","executionInfo":{"status":"ok","timestamp":1619455684864,"user_tz":-120,"elapsed":867,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"5a4d2289-7007-4d84-a313-0cb4e1b1a81c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.patches as mpatches\n\n#__________________________________________________________________\n# I extract a subset of columns and redefine the airlines labeling \ndf2 = df.loc[:, ['AIRLINE', 'DEPARTURE_DELAY']]\nabbr_companies = df2['AIRLINE'].unique()\n#________________________________________________________________________\ncolors = ['royalblue', 'grey', 'wheat', 'c', 'firebrick', 'seagreen', 'lightskyblue',\n          'lightcoral', 'yellowgreen', 'gold', 'tomato', 'violet', 'aquamarine', 'chartreuse']\n#___________________________________\nfig, ax1= plt.subplots(figsize=(5,5))\n\n#------------------------------\n# Pie chart nº1: nb of flights\n#------------------------------\nlabels = [s for s in  global_stats.index]\nsizes  = global_stats['%samples'].values\nexplode = [0.3 if sizes[i] < 0.3 else 0.0 for i in range(len(abbr_companies))]\npatches, texts, autotexts = ax1.pie(sizes, explode = explode,\n                                labels=labels, colors = colors,  autopct='%1.0f%%',\n                                shadow=False, startangle=0)\nfor i in range(len(abbr_companies)): \n    texts[i].set_fontsize(14)\nax1.axis('equal')","metadata":{"id":"pP2BxbOvFaUN","executionInfo":{"status":"ok","timestamp":1619457801692,"user_tz":-120,"elapsed":2648,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"56c31049-32d5-4991-8bbc-e416fae5fe29","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax3 = plt.subplots(figsize=(15,10))\n\n# striplot with all the values reported for the delays\n\ncolors = ['firebrick', 'gold', 'lightcoral', 'aquamarine', 'c', 'yellowgreen', 'grey',\n          'seagreen', 'tomato', 'violet', 'wheat', 'chartreuse', 'lightskyblue', 'royalblue']\n#___________________________________________________________________\nax3 = sns.stripplot(y=\"AIRLINE\", x=\"DEPARTURE_DELAY\", size = 4, palette = colors,\n                    data=df2, linewidth = 0.5,  jitter=True)\nplt.setp(ax3.get_xticklabels(), fontsize=14)\nplt.setp(ax3.get_yticklabels(), fontsize=14)\nax3.set_xticklabels(['{:2.0f}h{:2.0f}m'.format(*[int(y) for y in divmod(x,60)])\n                         for x in ax3.get_xticks()])\nplt.xlabel('Departure delay', fontsize=18, bbox={'facecolor':'midnightblue', 'pad':5},\n           color='w', labelpad=20)\nax3.yaxis.label.set_visible(False)\n#________________________\nplt.tight_layout(w_pad=3) ","metadata":{"id":"8zCiygH_KWcx","executionInfo":{"status":"ok","timestamp":1619457798955,"user_tz":-120,"elapsed":237820,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"6893eb09-4e8b-4790-ef0e-004ab9f9f863","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that the **different companies** have a much **different tail distribution of the delays**.","metadata":{}},{"cell_type":"markdown","source":"### Delay Distribution","metadata":{"id":"iujfUYJ_DGcb"}},{"cell_type":"markdown","source":"The Delay of each company seems to have different mean and variance and can be fit to a Asymmetrical Double Exponential (Laplace Distr.)","metadata":{"id":"2B2Zv5qlDKC9"}},{"cell_type":"code","source":"points = [] ; label_company = []\nfig = plt.figure(1, figsize=(11,11))\ni = 0\ndf2 = df.copy()\ndf2 = df2.drop(df2[df2['DEPARTURE_DELAY'] > 60].index)\nairlines = df2['AIRLINE'].unique()\nfor carrier_name in airlines:\n    i += 1\n    ax = fig.add_subplot(5,3,i)    \n    #_________________________\n    # Fit of the distribution\n    n, bins, patches = plt.hist(x = df2[df2['AIRLINE']==carrier_name]['DEPARTURE_DELAY'],\n                                range = (-20,180), bins= 60)\n    \n    # define tick labels for each subplot\n    if i < 10:\n        ax.set_xticklabels(['' for x in ax.get_xticks()])\n    else:\n        ax.set_xticklabels(['{:2.0f}h{:2.0f}m'.format(*[int(y) for y in divmod(x,60)])\n                            for x in ax.get_xticks()])\n    #______________\n    # subplot title\n    plt.title(carrier_name, fontsize = 14, fontweight = 'bold', color = 'darkblue')\n    #____________\n    # axes labels \n    if i == 4:\n        ax.text(-0.3,0.9,'Normalized count of flights', fontsize=16, rotation=90,\n            color='k', horizontalalignment='center', transform = ax.transAxes)\n    if i == 14:\n        ax.text( 0.5, -0.5 ,'Delay at origin', fontsize=16, rotation=0,\n            color='k', horizontalalignment='center', transform = ax.transAxes)\n    \nplt.tight_layout()","metadata":{"id":"okzzgekzCQG3","executionInfo":{"status":"ok","timestamp":1619455033154,"user_tz":-120,"elapsed":11186,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"d518cd88-bc85-4586-8ebe-8e39c1ea0989","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **TRAFFIC_ORIGIN** and **TRAFFIC_DESTINATION**","metadata":{"id":"wgbhEc9KVJKn"}},{"cell_type":"markdown","source":"Add some new features called **TRAFFIC_ORIGIN** and **TRAFFIC_DESTINATION** that measure the fan_in of the source and destination airports. ","metadata":{"id":"X-ZdhR0-Uz_Y"}},{"cell_type":"code","source":"traffic_origin = airports[['IATA_CODE','TRAFFIC']].rename(columns={'TRAFFIC':'TRAFFIC_ORIGIN'})\ntraffic_dest = airports[['IATA_CODE','TRAFFIC']].rename(columns={'TRAFFIC':'TRAFFIC_DESTINATION'})","metadata":{"id":"7k0vZGs8mlA1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(df,traffic_origin,left_on='ORIGIN_AIRPORT' ,right_on='IATA_CODE')\ndf = pd.merge(df,traffic_dest,left_on='DESTINATION_AIRPORT' ,right_on='IATA_CODE')\ndf = df.drop(['IATA_CODE_x','IATA_CODE_y'],axis=1)","metadata":{"id":"N9aAf-Tlmw6D"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **DELAY distribution** (mean, std) for each day for each company","metadata":{"id":"qRKuE7AWVUu9"}},{"cell_type":"markdown","source":"Add information regarding the average behaviour of a company in a specific day in a specific airport. This information could be improved by observing smaller windows since, as observed in the data exploration part, the correlation between delays is strong only within close hours.","metadata":{"id":"eb4F5RjbVuFw"}},{"cell_type":"code","source":"mean_origin_delay = df.groupby(['AIRLINE','ORIGIN_AIRPORT','MONTH','DAY'], as_index=False)['ARRIVAL_DELAY'].mean().rename(columns={ 'ARRIVAL_DELAY':'MEAN_ARRIVAL_ORIGIN'})\nstd_origin_delay = df.groupby(['AIRLINE','ORIGIN_AIRPORT','MONTH','DAY'], as_index=False)['ARRIVAL_DELAY'].std().rename(columns={'ARRIVAL_DELAY':'STD_ARRIVAL_ORIGIN'})\norigin_delay_dist = pd.merge(mean_origin_delay,std_origin_delay, on=['AIRLINE','ORIGIN_AIRPORT','MONTH','DAY'])","metadata":{"id":"KkMhPBHyAsBA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_destination_delay = df.groupby(['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'], as_index=False)['ARRIVAL_DELAY'].mean().rename(columns={'ARRIVAL_DELAY':'MEAN_ARRIVAL_DEST'})\nstd_destination_delay = df.groupby(['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'], as_index=False)['ARRIVAL_DELAY'].std().rename(columns={'ARRIVAL_DELAY':'STD_ARRIVAL_DEST'})\ndestination_delay_dist = pd.merge(mean_destination_delay,std_destination_delay, on=['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'])","metadata":{"id":"WJRYm6VGCh12"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(df, origin_delay_dist, how='left', on=['AIRLINE','ORIGIN_AIRPORT','MONTH','DAY'])\ndf = pd.merge(df, destination_delay_dist, how='left', on=['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'])","metadata":{"id":"jeUw4v2SWr8E"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Airports **Longitude** and **Latitude** ","metadata":{"id":"sOuPN2-dWa-d"}},{"cell_type":"code","source":"coor_src = airports[['IATA_CODE','LATITUDE','LONGITUDE']].rename(columns={'IATA_CODE':'ORIGIN_AIRPORT','LATITUDE':'LAT_ORIGIN','LONGITUDE':'LONG_ORIGIN'})\ncoor_dest = airports[['IATA_CODE','LATITUDE','LONGITUDE']].rename(columns={'IATA_CODE':'DESTINATION_AIRPORT','LATITUDE':'LAT_DEST','LONGITUDE':'LONG_DEST'})","metadata":{"id":"sz1Sea0cWnKh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.merge(df, coor_src, how='left', on=['ORIGIN_AIRPORT'])\ndf = pd.merge(df, coor_dest, how='left', on=['DESTINATION_AIRPORT'])","metadata":{"id":"Bk9LCP61ArHH"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Critical Periods**","metadata":{"id":"QsAK1rbkX9YE"}},{"cell_type":"markdown","source":"The idea that prompted us to do this is essentially linked to the impossibility of injecting too many one-hot encoded columns into the dataset used to build the regression model. For this reason it was necessary to find a suitable heuristic to describe the information related to the **ORIGIN_AIRPORT** critical periods without affecting negatively the performance of the model.","metadata":{"id":"PLz2ZGpXY5CO"}},{"cell_type":"markdown","source":"### Utilities","metadata":{"id":"8HGW2Lu8RCqN"}},{"cell_type":"code","source":"def pad_zero(x):\n\n    y = \"\"\n    c = 0\n\n    for step in x.split(\"-\"):\n        if len(step) == 1:\n            step = \"0\" + step\n        y += step + (\"-\" if c < 2 else \"\")\n        c += 1\n\n    return y","metadata":{"id":"AuJA32eIQ8Ht","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def from_military_time(x):\n\n    y = list(str(x))\n    if len(y) == 3:\n        y = [\"0\"] + y\n\n    return \"\".join(y[:2]) + \":00:00\"","metadata":{"id":"Lbyp7YVHRAdl","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Time series analysis","metadata":{"id":"Gs_42oSERHVx"}},{"cell_type":"markdown","source":"The main step of the iterative analysis are the followings:\n\n1.   Extract the date-time information from the train dataset.\n2.   Select at each iteration a different origin airport on which perform the time-series analysis.\n3.   Apply Prophet algorithm to describe the seasonal trends of that series.\n\nIn this way it was possible to obtain a description of the different trends that characterizes the delay variation over time of the different airports.","metadata":{"id":"M-OEpWsTTPRI"}},{"cell_type":"code","source":"!pip install prophet","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from prophet import Prophet\nimport os\nimport pickle as pkl","metadata":{"id":"IEp8lQMXQAhO","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delays_with_airports = pd.merge(X_train_df, y_train_df, on=\"ID\")","metadata":{"id":"OSiyiO-eRQDR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"delays_with_airports[\"SCHEDULED_DEPARTURE\"] = delays_with_airports[\"SCHEDULED_DEPARTURE\"].astype(int)\ndelays_with_airports[\"SCHEDULED_DEPARTURE_BIN\"] = (delays_with_airports[\"SCHEDULED_DEPARTURE\"] // 100) * 100","metadata":{"id":"G0_7BNHfRQd_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for airport in delays_with_airports[\"ORIGIN_AIRPORT\"].unique():\n\n    # Extract historical data for the current airport\n    airport_historical = delays_with_airports[delays_with_airports[\"ORIGIN_AIRPORT\"] == airport].groupby([\"MONTH\", \"DAY\", \"SCHEDULED_DEPARTURE_BIN\"]).agg({'DEPARTURE_DELAY': 'mean'})\n\n    if len(airport_historical) > 1000:\n\n        # Do some preprocessing\n        airport_historical = airport_historical.reset_index()\n        airport_historical[\"YEAR\"] = \"2021\"\n        airport_historical[\"DATE\"] = airport_historical[[\"YEAR\", \"MONTH\", \"DAY\"]].astype(str).agg('-'.join, axis=1).apply(pad_zero)\n        airport_historical[\"SCHEDULED_DEPARTURE_BIN_HOUR\"] = airport_historical[\"SCHEDULED_DEPARTURE_BIN\"].apply(from_military_time)\n        airport_historical[\"DATETIME\"] = airport_historical[[\"DATE\", \"SCHEDULED_DEPARTURE_BIN_HOUR\"]].astype(str).agg(' '.join, axis=1)\n\n        # Extract time-series data\n        train_series = airport_historical[[\"DATETIME\", \"DEPARTURE_DELAY\"]]\n        train_series.columns = [\"ds\", \"y\"]\n\n        # Let's do some magic <3\n        m = Prophet()\n        m.fit(train_series)\n        future = m.make_future_dataframe(periods=0)\n        forecast = m.predict(future)\n\n        # Extract time references from forecast dataset\n        forecast[\"MONTH\"] = forecast['ds'].dt.month\n        forecast[\"DAYOFWEEK\"] = forecast['ds'].dt.dayofweek\n        forecast[\"HOUR\"] = (forecast['ds'].dt.hour * 100)\n\n        # Extract seasonal trends [MONTH, WEEK, HOUR]\n        monthly_trend = forecast.groupby(\"MONTH\").agg({\"trend\": \"mean\"})\n        weekly_trend = forecast.groupby([\"DAYOFWEEK\", \"HOUR\"]).agg({\"weekly\": \"mean\"})\n        hourly_trend = forecast.groupby(\"HOUR\").agg({\"daily\": \"mean\"})\n\n        monthly_trend.to_pickle(f\"{airport}_monthly.pkl\")\n        weekly_trend.to_pickle(f\"{airport}_weekly.pkl\")\n        hourly_trend.to_pickle(f\"{airport}_hourly.pkl\")","metadata":{"id":"sj-xNX5_QCUF"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract lookup table","metadata":{"id":"kWRV_Yp4RbAh"}},{"cell_type":"code","source":"lookup_table = {}\nfor file in os.listdir(\"timeseries\"):\n\n    airport, step = file.split(\".\")[0].split(\"_\")\n\n    if airport not in lookup_table:\n        lookup_table[airport] = {}\n    if step not in lookup_table[airport]:\n        lookup_table[airport][step] = {}\n\n    df = pd.read_pickle(f\"timeseries/{file}\")\n    df_int = (df > np.mean(df)).astype(int)\n    df_int.to_dict()\n\n    lookup_table[airport][step] = list(df_int.to_dict().values())[0]","metadata":{"id":"vRrNZivxRdn4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"series_lookup.pkl\", \"wb\") as file:\n    pkl.dump(lookup_table, file)","metadata":{"id":"BJIuyuYHRgwQ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Critical DataFrame","metadata":{"id":"aDfQRrtRRycp"}},{"cell_type":"markdown","source":"Thanks to this approach it was possible to trace the most critical seasonal periods for each of the airports considered and use this information to produce a lookup table to add three new features to the train set:\n\n*   CRITICAL_WEEK\n*   CRITICAL_MONTH\n*   CRITICAL_DAY\n\nWhich represent a  0/1 boolean variable that describes the criticality of the departure period of a given flight.\n\nMore specifically, we described as critical periods the ones characterized by an inferred delay greater than the mean delay related to the considered seasonality (daily, weekly, monthly).","metadata":{"id":"PVoXd2LJTV9t"}},{"cell_type":"code","source":"def is_critical_month(row):\n\n    if row[\"ORIGIN_AIRPORT\"] in lookup_table:\n        critical = lookup_table[row[\"ORIGIN_AIRPORT\"]][\"monthly\"].get(row[\"MONTH\"], 0)\n    else:\n        critical = 0\n\n    return critical","metadata":{"id":"jXjrnuABR3It"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def is_critical_day(row):\n\n    if row[\"ORIGIN_AIRPORT\"] in lookup_table:\n        critical = lookup_table[row[\"ORIGIN_AIRPORT\"]][\"weekly\"].get((row[\"DAY_OF_WEEK\"], row[\"HOURBIN\"]), 0)\n    else:\n        critical = 0\n\n    return critical","metadata":{"id":"p-W2HmYmR_HG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def is_critical_day(row):\n\n    if row[\"ORIGIN_AIRPORT\"] in lookup_table:\n        critical = lookup_table[row[\"ORIGIN_AIRPORT\"]][\"weekly\"].get((row[\"DAY_OF_WEEK\"], row[\"HOURBIN\"]), 0)\n    else:\n        critical = 0\n\n    return critical","metadata":{"id":"RTyRKciVSBMZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_df[\"CRITICAL_MONTH\"] = X_train_df.apply(is_critical_month, axis=1)\nX_train_df[\"HOURBIN\"] = (X_train_df[\"DEPARTURE_TIME\"].astype(int) // 100) * 100\nX_train_df[\"CRITICAL_WEEK\"] = X_train_df.apply(is_critical_day, axis=1)\nX_train_df[\"CRITICAL_HOUR\"] = X_train_df.apply(is_critical_hour, axis=1)\ncritical = X_train_df[[\"ID\", \"CRITICAL_MONTH\", \"CRITICAL_WEEK\", \"CRITICAL_HOUR\"]]\ncritical.to_pickle(\"critical.pkl\")","metadata":{"id":"Xgw0wu8bSERS"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Load Critical DataFrame","metadata":{"id":"8qJdBNIlSJJO"}},{"cell_type":"code","source":"critical = pd.read_pickle(path + \"critical.pkl\")\ndf = pd.merge(df, critical, how='left', on=['ID'])","metadata":{"id":"TjuiUQUxYDev"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df.to_pickle(\"flights_features.pkl\")","metadata":{"id":"6tVrXYY9XDyR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Utility Functions","metadata":{"id":"2LdFDok4dLGB"}},{"cell_type":"code","source":"#X_test_df = pd.read_csv(\"test_features.csv\", dtype=X_dtype)\n\ndef prepare_samples(X_df):\n\n  #Delay Dist\n  X_df = pd.merge(X_df, origin_delay_dist, how='left', on=['AIRLINE','ORIGIN_AIRPORT', 'MONTH', 'DAY'])\n  X_df = pd.merge(X_df, destination_delay_dist,how='left', on=['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'])\n\n  #Critical\n  X_df = pd.merge(X_df, critical, how='left', on=['ID'])\n\n  #Fan_In\n  X_df = pd.merge(X_df,traffic_origin,how='left',left_on='ORIGIN_AIRPORT' ,right_on='IATA_CODE')\n  X_df = pd.merge(X_df,traffic_dest,how='left',left_on='DESTINATION_AIRPORT' ,right_on='IATA_CODE')\n  X_df = X_df.drop(['IATA_CODE_x','IATA_CODE_y'],axis=1)\n\n  #Coordinates\n  X_df = pd.merge(X_df, coor_src, how='left', on=['ORIGIN_AIRPORT'])\n  X_df = pd.merge(X_df, coor_dest, how='left', on=['DESTINATION_AIRPORT'])\n\n  return X_df\n\n#X_test_df = prepare_samples(X_test_df)\n#X_test_df.to_pickle(\"test_features_modified.pkl\")","metadata":{"id":"sUUX9jRtlqyj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_data(df, feature_names, imputer, scaler):\n    \"\"\"Preprocess data.\n\n    Parameters\n    ----------\n    df: pandas DataFrame.\n        The input data.\n    feature_names: list of strings.\n        The names of selected features.\n    imputer: sklearn.impute.SimpleImputer\n        The imputation transformer for completing missing values.\n    scaler: sklearn.preprocessing.StandardScaler.\n        The scaler used to normalize the features.\n\n    Returns\n    -------\n    X: numpy array.\n        The preprocessed data.\n    \"\"\"\n    # Select features\n    X_df = df[feature_names]\n\n    # Impute missing values\n    X = imputer.transform(X_df)\n    \n    # Normalize features\n    X = scaler.transform(X)\n\n    return X","metadata":{"id":"JH7eyXZOdNdI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_prediction(X, model, scaler):\n    \"\"\"Makes predictions given a preprocessed dataset.\n\n    Parameters\n    ----------\n    X: numpy array.\n        The input data, which already is pre-processed.\n    model: an hypopt or sklearn model.\n        The trained model used for making predictions.\n    scaler: sklearn.preprocessing.StandardScaler.\n        The scaler used to normalize the targets.\n\n    Returns\n    -------\n    y_pred: numpy array.\n        The unnormalized predictions.\n    \"\"\"\n    y_pred = scaler.inverse_transform(model.predict(X))\n    return y_pred","metadata":{"id":"nTDNyLkLdQ5G"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models","metadata":{"id":"riaWk5xmXXc9"}},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder\nfrom sklearn.preprocessing import PolynomialFeatures","metadata":{"id":"qydTzrPcXgZ-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_pickle(path + \"flights_features.pkl\")","metadata":{"id":"TMR2SPpJXd5r"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Base Linear Model**","metadata":{"id":"B60igPZcw-KV"}},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                'CRITICAL_WEEK','CRITICAL_HOUR']\ncategorical = ['AIRLINE']\ntarget_name = ['ARRIVAL_DELAY']\n\ndf3 = df","metadata":{"id":"2ZVpKbap0p0E"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = pd.DataFrame(df3['ARRIVAL_DELAY'])\nX = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\nX_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n    X, y, random_state=1, test_size=0.2)\n\nX_train_df = X_train_df[y_train_df['ARRIVAL_DELAY']<=300]\ny_train_df = y_train_df[y_train_df['ARRIVAL_DELAY']<=300]\n\nencoder = OneHotEncoder(sparse=False)\nencoder.fit(df3[categorical])\nc_train = X_train_df[categorical]\nc_val = X_val_df[categorical]\nc_train = encoder.transform(c_train)\nc_val = encoder.transform(c_val)\n\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(X_train_df[feature_names])\n\nX_scaler = StandardScaler()\nX_scaler.fit(X_train_df[feature_names])\n\ny_scaler = StandardScaler()\ny_scaler.fit(y_train_df[target_name])\n\nX_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\ny_train = y_scaler.transform(y_train_df[target_name])\n\nX_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\ny_val = y_scaler.transform(y_val_df[target_name])\n\npolinomial = PolynomialFeatures(degree=1, interaction_only=True, include_bias=True, order='C')\nX_train = polinomial.fit_transform(X_train)\nX_val = polinomial.fit_transform(X_val)\n\n#concat categorical features\nX_train = np.concatenate((X_train, c_train),axis=1)\nX_val = np.concatenate((X_val,c_val),axis=1)\n\n# Grid-search using the validation set.\nparam_grid = [\n  {'alpha': [0.001, 0.01, 0.1, 0., 1.0, 10.0, 100.]},\n]\n\nopt = GridSearch(model=Ridge(tol=0.0001), param_grid=param_grid)\nopt.fit(X_train, y_train, X_val, y_val, scoring='neg_mean_squared_error')","metadata":{"id":"C1L-J1YLxA9c","executionInfo":{"status":"ok","timestamp":1619451315232,"user_tz":-120,"elapsed":57317,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"2f7bcd9a-f02d-4c20-c829-4b37976d3b4e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred = make_prediction(X_train, opt, y_scaler)\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\ny_val_pred = make_prediction(X_val, opt, y_scaler)\nval_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))","metadata":{"id":"sq0_DcBZZZO2","executionInfo":{"status":"ok","timestamp":1619451319541,"user_tz":-120,"elapsed":758,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"2cfede46-5c71-4c59-cb97-14fcd23e57f7"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature Selection","metadata":{"id":"stIUe1jm3Sky"}},{"cell_type":"markdown","source":"#### Feature Importance","metadata":{"id":"xYquCfg15qDH"}},{"cell_type":"code","source":"from sklearn.feature_selection import RFE","metadata":{"id":"kb4uLA5f3SQ_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model=Ridge(tol=0.0001, alpha=0)\nrfe = RFE(model)\nrfe.fit(X_train, y_train)","metadata":{"id":"H4OHPA1X3YQm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['MONTH', 'DAY', 'DAY OF WEEK', 'DISTANCE', 'SCHEDULED ARRIVAL', \n  'DEPARTURE TIME','WHEELS OFF', 'AIR TIME','DEPARTURE DELAY', 'TAXI OUT',\n  'SCHEDULED TIME','LAT DEST','LONG DEST','LAT ORIGIN','LONG ORIGIN',\n  'MEAN ARRIVAL ORIGIN', 'STD ARRIVAL ORIGIN',\n  'TRAFFIC ORIGIN','TRAFFIC DESTINATION',\n  'CRITICAL WEEK','CRITICAL HOUR']\n\nlabels.extend(encoder.categories_[0].tolist())\n\nranking = rfe.ranking_[1:]\nfeat = [label for rank, label in sorted(zip(ranking, labels))]\nrank = [rank for rank, label in sorted(zip(ranking, labels))]\n\nx = np.arange(len(feat))  # the label locations\nfig, ax = plt.subplots()\n\nrects1 = ax.bar( x, 1/np.array(rank), label='Rank')\n\n# Add some text for labels, title and custom x-axis tick labels, etc.\nax.set_ylabel('Scores')\nax.set_title('Feature Ranking')\nax.set_xticks(x)\nax.set_xticklabels(feat)\nplt.xticks(rotation=90)\nax.legend()\n\n\nfig.tight_layout()\n\nplt.show()\n","metadata":{"id":"xImbMvnE5R2M","executionInfo":{"status":"ok","timestamp":1619453944629,"user_tz":-120,"elapsed":1212,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"9167867c-44ee-4ab9-bb67-1a3f79230d70"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = ['MONTH', 'DAY', 'DAY OF WEEK', 'DISTANCE', 'SCHEDULED ARRIVAL', \n  'DEPARTURE TIME','WHEELS OFF', 'AIR TIME','DEPARTURE DELAY', 'TAXI OUT',\n  'SCHEDULED TIME','LAT DEST','LONG DEST','LAT ORIGIN','LONG ORIGIN',\n  'MEAN ARRIVAL ORIGIN', 'STD ARRIVAL ORIGIN',\n  'TRAFFIC ORIGIN','TRAFFIC DESTINATION',\n  'CRITICAL WEEK','CRITICAL HOUR']\n\nlabels.extend(encoder.categories_[0].tolist())\n\nlabels = labels[:]\ncoeff = np.log(np.abs(opt.best_estimator_.coef_.flatten()[1:]))\n\nx = np.arange(len(labels))  # the label locations\nfig, ax = plt.subplots()\n\nrects1 = ax.bar( x, coeff, label='LR coeff')\n\n\n# Add some text for labels, title and custom x-axis tick labels, etc.\nax.set_ylabel('log(w)')\nax.set_title('Linear Regression Coefficients')\nax.set_xticks(x)\nax.set_xticklabels(labels)\nplt.xticks(rotation=90)\nax.legend()\n\n\nfig.tight_layout()\n\nplt.show()\n","metadata":{"id":"c2vnjWYxr56t","executionInfo":{"status":"ok","timestamp":1619453916447,"user_tz":-120,"elapsed":1179,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"41054648-62cc-479a-a4b0-80253bceea5f"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Proposal Creation","metadata":{}},{"cell_type":"code","source":"X_test_df = pd.read_pickle(\"test_features_modified.pkl\")\n\nc_test = X_test_df[categorical]\nc_test = encoder.transform(c_test)\n\nX_test = preprocess_data(X_test_df, feature_names, imputer, X_scaler)\n\npolinomial = PolynomialFeatures(degree=1, interaction_only=True, include_bias=True, order='C')\nX_test = polinomial.fit_transform(X_test)\n\n#concat categorical features\nX_test = np.concatenate((X_test, c_test),axis=1)\n\ny_pred = make_prediction(X_test, opt, y_scaler)\ndata = {'ID':X_test_df['ID'],'ARRIVAL_DELAY':y_pred.flatten().tolist()}\nresults = pd.DataFrame.from_dict(data)\n\nresults.set_index('ID').to_csv('SUB4.csv')","metadata":{"id":"SoMVhbtTpn8d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Bayesian Model**","metadata":{"id":"B6fwfSFubm1H"}},{"cell_type":"code","source":"from sklearn.linear_model import BayesianRidge","metadata":{"id":"gaTsP2h0c5DA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                'CRITICAL_WEEK','CRITICAL_HOUR']\ncategorical = ['AIRLINE']\ntarget_name = ['ARRIVAL_DELAY']\n\ndf3 = df","metadata":{"id":"y0ASfy_3bvZZ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = pd.DataFrame(df3['ARRIVAL_DELAY'])\nX = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\nX_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n    X, y, random_state=1, test_size=0.2)\n\nX_train_df = X_train_df[y_train_df['ARRIVAL_DELAY']<=300]\ny_train_df = y_train_df[y_train_df['ARRIVAL_DELAY']<=300]\n\nencoder = OneHotEncoder(sparse=False)\nencoder.fit(df3[categorical])\nc_train = X_train_df[categorical]\nc_val = X_val_df[categorical]\nc_train = encoder.transform(c_train)\nc_val = encoder.transform(c_val)\n\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(X_train_df[feature_names])\n\nX_scaler = StandardScaler()\nX_scaler.fit(X_train_df[feature_names])\n\ny_scaler = StandardScaler()\ny_scaler.fit(y_train_df[target_name])\n\nX_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\ny_train = y_scaler.transform(y_train_df[target_name])\n\nX_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\ny_val = y_scaler.transform(y_val_df[target_name])\n\npolinomial = PolynomialFeatures(degree=1, interaction_only=True, include_bias=True, order='C')\nX_train = polinomial.fit_transform(X_train)\nX_val = polinomial.fit_transform(X_val)\n\n#concat categorical features\nX_train = np.concatenate((X_train, c_train),axis=1)\nX_val = np.concatenate((X_val,c_val),axis=1)\n\n# Grid-search using the validation set.\nopt=BayesianRidge()\nopt.fit(X_train, y_train.ravel())","metadata":{"id":"V2GlQYbSbmVl","executionInfo":{"status":"ok","timestamp":1619454762423,"user_tz":-120,"elapsed":50762,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"e4c8a178-a2cb-49fb-cc04-5a8605425885"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred = make_prediction(X_train, opt, y_scaler)\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\ny_val_pred = make_prediction(X_val, opt, y_scaler)\nval_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))","metadata":{"id":"Jp8bWzI4dg6t","executionInfo":{"status":"ok","timestamp":1619454763071,"user_tz":-120,"elapsed":30632,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"aacbd703-aec5-4a98-d54c-e45dd2c9b856"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Different Approach: Build a model for each Airline","metadata":{"id":"hNyiCOcqwU45"}},{"cell_type":"markdown","source":"The Idea is to create smaller datasets taylored for each airline company.","metadata":{"id":"pOzg-pP1eY22"}},{"cell_type":"code","source":"airlines_list = df['AIRLINE'].unique()","metadata":{"id":"LMFuFOLPxT1g"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Linear Regression Model","metadata":{"id":"ewzsOxVW7i6O"}},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                 'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                 'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                 'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                 'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                  'CRITICAL_WEEK','CRITICAL_HOUR']\ncategorical = []\ntarget_name = ['ARRIVAL_DELAY']\n\nmodels_dict = {}\nscalers_X = {}\nscalers_y ={}\nimputers = {}\n\nPol_Degree = {'AA': 1, 'NK':2, 'B6':1, 'OO':1, 'UA':1, 'DL':1, 'MQ':1, 'EV':1, \n            'AS': 2, 'F9':1, 'US':2, 'WN':2, 'VX':2, 'HA':1}","metadata":{"id":"x_9rNb8FelB4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for airline in airlines_list:\n  print(f\"\\n Airline: {airline}\")\n  df3 = df[(df['AIRLINE']==airline)]\n  \n  y = pd.DataFrame(df3['ARRIVAL_DELAY'])\n  X = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\n  X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n      X, y, random_state=1, test_size=0.2)\n  \n  X_train_df = X_train_df[y_train_df['ARRIVAL_DELAY']<=300]\n  y_train_df = y_train_df[y_train_df['ARRIVAL_DELAY']<=300]\n\n  encoder = OneHotEncoder(sparse=False)\n  encoder.fit(df3[categorical])\n  c_train = X_train_df[categorical]\n  c_val = X_val_df[categorical]\n  c_train = encoder.transform(c_train)\n  c_val = encoder.transform(c_val)\n\n  imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n  imputer.fit(X_train_df[feature_names])\n\n  X_scaler = StandardScaler()\n  X_scaler.fit(X_train_df[feature_names])\n\n  y_scaler = StandardScaler()\n  y_scaler.fit(y_train_df[target_name])\n\n  X_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\n  y_train = y_scaler.transform(y_train_df[target_name])\n\n  X_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\n  y_val = y_scaler.transform(y_val_df[target_name])\n\n  polinomial = PolynomialFeatures(degree= Pol_Degree[airline], \n                                  interaction_only=False, \n                                  include_bias=True, order='C')\n  X_train = polinomial.fit_transform(X_train)\n  X_val = polinomial.fit_transform(X_val)\n\n  #concat categorical features\n  X_train = np.concatenate((X_train, c_train),axis=1)\n  X_val = np.concatenate((X_val,c_val),axis=1)\n\n    # Grid-search using the validation set.\n  param_grid = [\n    {'alpha': [0.001, 0.01, 0.1, 0., 1.0, 10.0, 100.]},\n  ]\n\n  opt = GridSearch(model=Ridge(tol=0.0001), param_grid=param_grid)\n  opt.fit(X_train, y_train, X_val, y_val, scoring='neg_mean_squared_error')\n\n  y_train_pred = make_prediction(X_train, opt, y_scaler)\n  train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\n  print(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\n  y_val_pred = make_prediction(X_val, opt, y_scaler)\n  val_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\n  print(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))\n\n  models_dict[airline] = opt\n  scalers_X[airline] = X_scaler\n  scalers_y[airline] = y_scaler\n  imputers[airline] = imputer","metadata":{"id":"UC7D8U82xDH3","executionInfo":{"status":"ok","timestamp":1619433144263,"user_tz":-120,"elapsed":120985,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"edc11626-b370-4ba6-bcc1-31226edeb290"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_df = pd.read_pickle(\"test_features_modified.pkl\")","metadata":{"id":"3zlp0lzh2GdP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = pd.DataFrame()\n\nfor airline in airlines_list:\n  X_df = X_test_df[X_test_df['AIRLINE']==airline]\n  ID = X_df['ID'].values.reshape(-1,1)\n  X_df = X_df[feature_names]\n\n  X_scaler = scalers_X[airline]\n  y_scaler = scalers_y[airline]\n  imputer = imputers[airline]\n  model = models_dict[airline]\n\n  X = preprocess_data(X_df, feature_names, imputer, X_scaler)\n  \n  polinomial = PolynomialFeatures(degree=Pol_Degree[airline],\n                                  interaction_only=False,\n                                  include_bias=True, order='C')\n  X = polinomial.fit_transform(X)\n\n  y = make_prediction(X, model, y_scaler)\n  data = {'ID':ID.flatten().tolist(),'ARRIVAL_DELAY':y.flatten().tolist()}\n  tmp = pd.DataFrame.from_dict(data)\n  results = results.append(tmp, ignore_index=True)\n\nresults = results.set_index('ID').sort_values(by=['ID'])\nresults.to_csv('SUB5.csv')\nresults","metadata":{"id":"aKrLI8Ae3gNa"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bayesian Models","metadata":{"id":"SOlJ7BvL5QcH"}},{"cell_type":"code","source":"from sklearn.linear_model import BayesianRidge","metadata":{"id":"4SoGXLVg5Tib"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                 'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                 'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                 'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                 'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                  'CRITICAL_WEEK','CRITICAL_HOUR']\ncategorical = ['ORIGIN_AIRPORT']\ntarget_name = ['ARRIVAL_DELAY']\n\nmodels_dict = {}\nscalers_X = {}\nscalers_y ={}\nimputers = {}\n\nfor airline in airlines_list:\n  print(f\"\\n Airline: {airline}\")\n  df3 = df2[(df2['AIRLINE']==airline)]\n  df3 = pd.merge(df3, origin_delay_dist, how='left', on=['AIRLINE','ORIGIN_AIRPORT','MONTH','DAY'])\n  df3 = pd.merge(df3, destination_delay_dist, how='left', on=['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'])\n  \n  y = pd.DataFrame(df3['ARRIVAL_DELAY'])\n  X = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\n  X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n      X, y, random_state=1, test_size=0.2)\n  \n  X_train_df = X_train_df[y_train_df['ARRIVAL_DELAY']<=180]\n  y_train_df = y_train_df[y_train_df['ARRIVAL_DELAY']<=180]\n\n\n  encoder = OneHotEncoder(sparse=False)\n  encoder.fit(df3[categorical])\n  c_train = X_train_df[categorical]\n  c_val = X_val_df[categorical]\n  c_train = encoder.transform(c_train)\n  c_val = encoder.transform(c_val)\n\n  imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n  imputer.fit(X_train_df[feature_names])\n\n  X_scaler = StandardScaler()\n  X_scaler.fit(X_train_df[feature_names])\n\n  y_scaler = StandardScaler()\n  y_scaler.fit(y_train_df[target_name])\n\n  X_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\n  y_train = y_scaler.transform(y_train_df[target_name])\n\n  X_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\n  y_val = y_scaler.transform(y_val_df[target_name])\n\n  polinomial = PolynomialFeatures(degree=2, interaction_only=True, include_bias=True, order='C')\n  X_train = polinomial.fit_transform(X_train)\n  X_val = polinomial.fit_transform(X_val)\n\n  #concat categorical features\n  X_train = np.concatenate((X_train, c_train),axis=1)\n  X_val = np.concatenate((X_val,c_val),axis=1)\n\n  # Grid-search using the validation set.\n  opt=BayesianRidge()\n  opt.fit(X_train, y_train)\n\n  y_train_pred = make_prediction(X_train, opt, y_scaler)\n  train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\n  print(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\n  y_val_pred = make_prediction(X_val, opt, y_scaler)\n  val_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\n  print(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))\n\n  models_dict[airline] = opt\n  scalers_X[airline] = X_scaler\n  scalers_y[airline] = y_scaler\n  imputers[airline] = imputer","metadata":{"id":"PuNHm5jnkpfs"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Other Models Trial (Older Versions)","metadata":{"id":"xJb85gqJzuLy"}},{"cell_type":"markdown","source":"## REGRESSION TREE","metadata":{"id":"qYk7Q4jpP8Bc"}},{"cell_type":"markdown","source":"This model has been taken into account but the performances did not significantly improve with respect to a common linear model, so we preferred to remain on simpler mode since the model complexity increaase did not justify the performances.","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor","metadata":{"id":"TVKCmKDKQAGS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                 'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                 'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                 'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                 'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                  'CRITICAL_WEEK','CRITICAL_HOUR']\ncategorical = ['AIRLINE']\ntarget_name = ['ARRIVAL_DELAY']\n\ndf3 = df","metadata":{"id":"H64HBfVrBVwd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = pd.DataFrame(df3['ARRIVAL_DELAY'])\nX = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\nX_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n    X, y, random_state=1, test_size=0.2)\n\nX_train_df = X_train_df[y_train_df['ARRIVAL_DELAY']<=180]\ny_train_df = y_train_df[y_train_df['ARRIVAL_DELAY']<=180]\n\nencoder = OneHotEncoder(sparse=False)\nencoder.fit(df3[categorical])\nc_train = X_train_df[categorical]\nc_val = X_val_df[categorical]\nc_train = encoder.transform(c_train)\nc_val = encoder.transform(c_val)\n\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(X_train_df[feature_names])\n\nX_scaler = StandardScaler()\nX_scaler.fit(X_train_df[feature_names])\n\ny_scaler = StandardScaler()\ny_scaler.fit(y_train_df[target_name])\n\nX_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\ny_train = y_scaler.transform(y_train_df[target_name])\n\nX_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\ny_val = y_scaler.transform(y_val_df[target_name])\n\n#concat categorical features\nX_train = np.concatenate((X_train, c_train),axis=1)\nX_val = np.concatenate((X_val,c_val),axis=1)","metadata":{"id":"glPlgfyBBEiX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt=RandomForestRegressor(n_estimators=10, max_depth=50, min_samples_split=100, n_jobs=-1, verbose=10)\nopt.fit(X_train, y_train.ravel())","metadata":{"id":"Hol_Iaz3C_2c","executionInfo":{"status":"ok","timestamp":1619442287348,"user_tz":-120,"elapsed":249210,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"4109295f-66c8-47ec-f4b0-247839c49ca1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred = make_prediction(X_train, opt, y_scaler)\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\ny_val_pred = make_prediction(X_val, opt, y_scaler)\nval_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))","metadata":{"id":"mZp7w-LbBIsh","executionInfo":{"status":"ok","timestamp":1619442323367,"user_tz":-120,"elapsed":3769,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"43e08154-a7b4-4c9e-dff5-f5632c901990"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## GAUSSIAN PROCESSES\n\nThe Idea is to select very few samples close to the sample to evaluate both in term of space and time and to create a model on the fly by using a Gaussian Process. Although the results are reasonable, **creating a model for every sample is computationally intensive and does not scale well in practice**.","metadata":{"id":"Gkei2RgM1sy9"}},{"cell_type":"code","source":"from sklearn.gaussian_process import GaussianProcessRegressor\nfrom sklearn.gaussian_process.kernels import RBF, ConstantKernel as C","metadata":{"id":"cMhv1Rnz1xo3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def locate_dataset(X_train, y_train, sample):\n  window_upper = sample.TIME_SINCE_BEGIN_YEAR + 24*60\n  window_lower = sample.TIME_SINCE_BEGIN_YEAR - 24*60\n  origin_airport = sample.ORIGIN_AIRPORT\n  mask = (X_train['TIME_SINCE_BEGIN_YEAR'] <= window_upper) &\\\n   (X_train['TIME_SINCE_BEGIN_YEAR'] >= window_lower) &\\\n    (X_train['ORIGIN_AIRPORT'] == origin_airport)\n  return X_train[mask], y_train[mask]","metadata":{"id":"INvZ1nFe4Q_W"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                'CRITICAL_WEEK','CRITICAL_HOUR']\ntarget_name = ['ARRIVAL_DELAY']\n\nmodels_dict = {}\n\nval_rmse = 0\nval_rmse_airline = []\nfor airline in airlines_list:\n  print(f\"\\n Airline: {airline}\")\n  df3 = df[df['AIRLINE']==airline]\n  df3 = df3[df3['ARRIVAL_DELAY'] < 120]\n  df3 = pd.merge(df3, origin_delay_dist, on=['AIRLINE','ORIGIN_AIRPORT'])\n  df3 = pd.merge(df3, destination_delay_dist, on=['AIRLINE','DESTINATION_AIRPORT'])\n  \n  y = pd.DataFrame(df3['ARRIVAL_DELAY'])\n  X = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\n  X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n      X, y, random_state=1, test_size=0.2)\n  \n  cnt = 0\n  \n  for (ID, sample),(_,target) in zip(X_val_df.iterrows(), y_val_df.iterrows()):\n\n    # Build reduced dataset starting from current sample\n\n    X_train_mini_df, y_train_mini_df = locate_dataset(X_train_df, y_train_df, sample)\n\n    imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n    imputer.fit(X_train_mini_df[feature_names])\n\n    X_scaler = StandardScaler()\n    X_scaler.fit(X_train_mini_df[feature_names])\n\n    y_scaler = StandardScaler()\n    y_scaler.fit(y_train_mini_df[target_name])\n\n    X_train = preprocess_data(X_train_mini_df, feature_names, imputer, X_scaler)\n    y_train = y_scaler.transform(y_train_mini_df)\n\n    X_val = preprocess_data(sample, feature_names, imputer, X_scaler)\n    target_values = target.values\n    if (len(target.values) < 2):\n      target_values = np.reshape(target.values, [-1, 1])\n        \n    y_val = y_scaler.transform(target_values)\n\n    kernel = C(1.0, (1e-3, 1e3)) * RBF(10, (1e-2, 1e2))\n    gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=9)\n\n    gp.fit(X_train, y_train)\n\n    # will always be zero in case of Gaussian process\n    #y_train_pred = make_prediction(X_train, opt, y_scaler)\n    #train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_mini_df.values))\n    #print(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\n    y_val_pred = make_prediction(X_val, gp, y_scaler)\n    val_rmse = val_rmse + mean_squared_error(y_val_pred, target)\n    cnt = cnt + 1\n    if ( cnt % 100 ):\n      print(\"Validation RMSE is at:\", np.sqrt(val_rmse/cnt))\n  val_rmse_airline.append(np.sqrt(val_rmse/X_val_df.shape[0]))\n  print(\"Validation RMSE is:\", val_rmse_airline[-1])","metadata":{"id":"M5z8cB-614qi"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Neural Network","metadata":{"id":"eVkN2k_zdi_r"}},{"cell_type":"markdown","source":"**Discarded since it did not improve with respect to a polinomial model and was not explainable**","metadata":{}},{"cell_type":"code","source":"import keras\nfrom keras.layers import Input, Dense, GaussianNoise, Lambda, ActivityRegularization, BatchNormalization\nfrom keras.models import Model  \nfrom keras import Sequential\n#from keras.layers import ActivityRegularization\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.optimizers import Adam,SGD  \n\nfrom keras import backend as K\n\nfrom sklearn.preprocessing import OneHotEncoder","metadata":{"id":"tbA1WhhtZLSf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### One for airline","metadata":{"id":"HSix4fwS1UK9"}},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                 'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                 'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                 'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                 'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                  'CRITICAL_WEEK','CRITICAL_HOUR']\ncategorical = ['ORIGIN_AIRPORT']\ntarget_name = ['ARRIVAL_DELAY']\n\nmodels_dict = {}\nscalers_X = {}\nscalers_y ={}\nimputers = {}\n\nfor airline in airlines_list:\n  print(f\"\\n Airline: {airline}\")\n  df3 = df2[(df2['AIRLINE']==airline)]\n  df3 = pd.merge(df3, origin_delay_dist, how='left', on=['AIRLINE','ORIGIN_AIRPORT','MONTH','DAY'])\n  df3 = pd.merge(df3, destination_delay_dist, how='left', on=['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'])\n  \n  y = pd.DataFrame(df3['ARRIVAL_DELAY'])\n  X = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\n  X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n      X, y, random_state=1, test_size=0.2)\n  \n  X_train_df = X_train_df[y_train_df['ARRIVAL_DELAY']<=300]\n  y_train_df = y_train_df[y_train_df['ARRIVAL_DELAY']<=300]\n\n  encoder = OneHotEncoder(sparse=False)\n  encoder.fit(df3[categorical])\n  c_train = X_train_df[categorical]\n  c_val = X_val_df[categorical]\n  c_train = encoder.transform(c_train)\n  c_val = encoder.transform(c_val)\n\n  imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n  imputer.fit(X_train_df[feature_names])\n\n  X_scaler = StandardScaler()\n  X_scaler.fit(X_train_df[feature_names])\n\n  y_scaler = StandardScaler()\n  y_scaler.fit(y_train_df[target_name])\n\n  X_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\n  y_train = y_train_df[target_name] #y_scaler.transform(y_train_df[target_name])\n\n  X_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\n  y_val = y_val_df[target_name]# y_scaler.transform(y_val_df[target_name])\n\n  #concat categorical features\n  X_train = np.concatenate((X_train, c_train),axis=1)\n  X_val = np.concatenate((X_val,c_val),axis=1)\n\n  layer1_size = 30\n  layer2_size = 40\n  layer3_size = 20\n\n  epochs = 20\n  batch_size = 100\n\n  model = Sequential(\n              [\n                Dense(layer1_size, activation='relu'),\n                Dense(layer2_size, activation='relu'),\n                Dense(layer3_size, activation='relu'),\n                Dense(1, activation='linear'),\n              ]\n          )\n\n  model.compile(optimizer='adam', loss='mse')  \n\n  hist = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,validation_data=(X_val, y_val))\n\n  y_train_pred = model.predict(X_train) # make_prediction(X_train, model, y_scaler)\n  train_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\n  print(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\n  y_val_pred = model.predict(X_val) #make_prediction(X_val, model, y_scaler)\n  val_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\n  print(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))\n\n  models_dict[airline] = opt\n  scalers_X[airline] = X_scaler\n  scalers_y[airline] = y_scaler\n  imputers[airline] = imputer","metadata":{"id":"5DIMAKBEY4oz"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Single Model","metadata":{"id":"LPoEeL2p1dUz"}},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                 'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                 'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                 'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                 'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                  'CRITICAL_WEEK','CRITICAL_HOUR']\ncategorical = ['AIRLINE']\ntarget_name = ['ARRIVAL_DELAY']\n\n\ndf3 = df","metadata":{"id":"xpdGqZmLyHyC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = pd.DataFrame(df3['ARRIVAL_DELAY'])\nX = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\nX_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n    X, y, random_state=1, test_size=0.2)\n\nX_train_df = X_train_df[y_train_df['ARRIVAL_DELAY']<=300]\ny_train_df = y_train_df[y_train_df['ARRIVAL_DELAY']<=300]\n\nencoder = OneHotEncoder(sparse=False)\nencoder.fit(df3[categorical])\nc_train = X_train_df[categorical]\nc_val = X_val_df[categorical]\nc_train = encoder.transform(c_train)\nc_val = encoder.transform(c_val)\n\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(X_train_df[feature_names])\n\nX_scaler = StandardScaler()\nX_scaler.fit(X_train_df[feature_names])\n\ny_scaler = StandardScaler()\ny_scaler.fit(y_train_df[target_name])\n\nX_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\ny_train = y_train_df[target_name] #y_scaler.transform(y_train_df[target_name])\n\nX_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\ny_val = y_val_df[target_name]# y_scaler.transform(y_val_df[target_name])\n\n#concat categorical features\nX_train = np.concatenate((X_train, c_train),axis=1)\nX_val = np.concatenate((X_val,c_val),axis=1)\n\nlayer1_size = 20\nlayer2_size = 30\nlayer3_size = 20\n\nepochs = 40\nbatch_size = 1000\n\nmodel = Sequential(\n            [\n              Dense(layer1_size, activation='relu'),\n              BatchNormalization(),\n              Dense(layer2_size, activation='relu'),\n              BatchNormalization(),\n              Dense(layer3_size, activation='relu'),\n              Dense(1, activation='linear'),\n            ]\n        )\n\nmodel.compile(optimizer='adam', loss='mse')  \n\nhist = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,validation_data=(X_val, y_val))\n\ny_train_pred = model.predict(X_train) # make_prediction(X_train, model, y_scaler)\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\ny_val_pred = model.predict(X_val) #make_prediction(X_val, model, y_scaler)\nval_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))","metadata":{"id":"zzTk3seJpHq1","executionInfo":{"status":"ok","timestamp":1619435482497,"user_tz":-120,"elapsed":587219,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"c0a9957d-2049-4221-c22c-a4e89c61c29d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_df = pd.read_csv(\"test_features.csv\", dtype=X_dtype)\nX_test_df = pd.merge(X_test_df, origin_delay_dist, how='left', on=['AIRLINE','ORIGIN_AIRPORT', 'MONTH', 'DAY'])\nX_test_df = pd.merge(X_test_df, destination_delay_dist,how='left', on=['AIRLINE','DESTINATION_AIRPORT','MONTH','DAY'])\nX_test_df = pd.merge(X_test_df, coor_src, how='left', on=['ORIGIN_AIRPORT'])\nX_test_df = pd.merge(X_test_df, coor_dest, how='left', on=['DESTINATION_AIRPORT'])\nX_test_df","metadata":{"id":"BbBdFBK8tQ-r"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = OneHotEncoder(sparse=False)\nencoder.fit(df3[categorical])\nc_test = X_test_df[categorical]\nc_test = encoder.transform(c_test)\n\nX_test = preprocess_data(X_test_df, feature_names, imputer, X_scaler)\n\n#concat categorical features\nX_test = np.concatenate((X_test, c_test),axis=1)\n\n\ny_pred = model.predict(X_test) #make_prediction(X_val, model, y_scaler)","metadata":{"id":"YU5SfpVS27S_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred","metadata":{"id":"kmgONdrk3nEn","executionInfo":{"status":"ok","timestamp":1619351471594,"user_tz":-120,"elapsed":1255,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"14db4e4e-2068-420d-e149-d475875f8415"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = {'ID':X_test_df['ID'],'ARRIVAL_DELAY':y_pred.flatten().tolist()}\nresults = pd.DataFrame.from_dict(data).set_index('ID').to_csv('SUB3.csv')","metadata":{"id":"jr7Phvh83o-p"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CLASSIFIER MAGNITUDE DELAY - RANDOM FOREST CLASSIFIER\n\nThe Idea is to generate a new feature that classify a first approximation of the delay in order to give a hint to the magnitude and generate some categorical features.","metadata":{"id":"JG_1LeIjN9gn"}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nimport seaborn as sn\n\nfrom sklearn.metrics import confusion_matrix","metadata":{"id":"Rnt7WCUlAffw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"intervals = np.array([ -60, -30, -10, 10, 30, 60 ])\ndelay_class = np.digitize( df['ARRIVAL_DELAY'],intervals)","metadata":{"id":"anXfqCWkSVFq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'SCHEDULED_ARRIVAL', \n                 'DEPARTURE_TIME','WHEELS_OFF', 'AIR_TIME','DEPARTURE_DELAY', 'TAXI_OUT',\n                 'SCHEDULED_TIME','LAT_DEST','LONG_DEST','LAT_ORIGIN','LONG_ORIGIN',\n                 'MEAN_ARRIVAL_ORIGIN', 'STD_ARRIVAL_ORIGIN',\n                 'TRAFFIC_ORIGIN','TRAFFIC_DESTINATION',\n                  'CRITICAL_WEEK','CRITICAL_HOUR']\ntarget_name = ['DELAY_CLASS']\n\nmodels_dict = {}\n\nfor airline in airlines_list[:1]:\n  print(f\"\\n Airline: {airline}\")\n  df3 = df[df['AIRLINE']==airline]\n  df3 = pd.merge(df3, origin_delay_dist, on=['AIRLINE','ORIGIN_AIRPORT'])\n  df3 = pd.merge(df3, destination_delay_dist, on=['AIRLINE','DESTINATION_AIRPORT'])\n  \n  intervals = np.array([ -60, -30, 0, 30, 60 ])\n  y = pd.DataFrame(np.digitize(df3['ARRIVAL_DELAY'],intervals), columns=target_name)\n  X = df3.drop(['ARRIVAL_DELAY'],axis=1)\n\n  X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n      X, y, random_state=1, test_size=0.2)\n\n  imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n  imputer.fit(X_train_df[feature_names])\n\n  X_scaler = StandardScaler()\n  X_scaler.fit(X_train_df[feature_names])\n\n  X_train = preprocess_data(X_train_df, feature_names, imputer, X_scaler)\n  y_train = y_train_df[target_name]\n\n  X_val = preprocess_data(X_val_df, feature_names, imputer, X_scaler)\n  y_val = y_val_df[target_name]\n\n  opt=RandomForestClassifier(min_samples_split=100, n_jobs = 4)\n  opt.fit(X_train, y_train)\n\n  y_train_pred = opt.predict(X_train)\n  err_train = accuracy_score(y_train, y_train_pred)\n  print(\"Training Err: {:.5f}\".format(float(err_train)))\n\n  y_val_pred = opt.predict(X_val)\n  err_val = accuracy_score(y_val, y_val_pred)\n  print(\"Validation Err: {:.5f}\".format(float(err_val)))\n\n  conf_mat = confusion_matrix(y_val, y_val_pred)\n  sn.heatmap(conf_mat)\n\n  #models_dict[airline] = opt","metadata":{"id":"otks3ngyN9E_","executionInfo":{"status":"error","timestamp":1619437575320,"user_tz":-120,"elapsed":502,"user":{"displayName":"DNA Project","photoUrl":"","userId":"07880967405216852507"}},"outputId":"a7112c00-0d21-4317-96ee-9de9f96092c5"},"execution_count":null,"outputs":[]}]}