{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Challenge 1 : Dense Neural Network model\n\nTo properly run the visualization cells, this presentation notebook reads only part of the data for the sake of memory and running time. Consequently, the performance of this run does not represent the performance achieved during our various tests.","metadata":{}},{"cell_type":"markdown","source":"## Preparing the environment\nFirst, we install all required packages.","metadata":{}},{"cell_type":"code","source":"#!pip install keras\n#!pip install numpy\n#!pip install pandas\n#!pip install seaborn\n#!pip install tensorflow","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Imports\nThen we import them.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport datetime\nfrom sklearn import datasets\nimport tensorflow as tf\nimport itertools\nimport matplotlib.pyplot as plt\nimport math\nimport os\n\nfrom math import sqrt\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer, PowerTransformer\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.pipeline import make_pipeline\nfrom tensorflow import keras\nfrom keras.models import Sequential,load_model\nfrom keras.regularizers import l2\nfrom keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization\nfrom keras.callbacks import ModelCheckpoint\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.linear_model import Ridge\nfrom keras import regularizers\nfrom os.path import join, isdir, isfile\n\nimport seaborn as sns\ncolor = sns.color_palette()\nsns.set_style('darkgrid')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Files checking\nWe verify that the environment is properly set up.","metadata":{}},{"cell_type":"code","source":"data_directory = '../input/eurecom-aml-2021-challenge-1/data'\nworking_directory = './'\n\nassert isdir (data_directory)\nassert isdir (working_directory)\nassert isfile(join(data_directory, 'airlines.csv'          ))\nassert isfile(join(data_directory, 'airports.csv'          ))\nassert isfile(join(data_directory, 'example_submission.csv'))\nassert isfile(join(data_directory, 'test_features.csv'     ))\nassert isfile(join(data_directory, 'train_features.csv'    ))\nassert isfile(join(data_directory, 'train_targets.csv'     ))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'AIR_TIME', 'SCHEDULED_ARRIVAL', 'DEPARTURE_TIME', 'TAXI_OUT', 'AIRLINE_FAILURE_MEAN', 'AIRLINE_FAILURE_STD', 'TAIL_FAILURE_MEAN', 'TAIL_FAILURE_STD', 'DESTINATION_AIRPORT_FAILURE_MEAN', 'DESTINATION_AIRPORT_FAILURE_STD', 'TRAVEL_FAILURE_MEAN', 'TRAVEL_FAILURE_STD', 'ORIGIN_LAT', 'ORIGIN_LONG', 'DESTINATION_LAT', 'DESTINATION_LONG']\ntarget_names = ['ARRIVAL_DELAY']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_dtype = {\n    'ID'                   : int,\n    'YEAR'                 : int,  \n    'MONTH'                : int,  \n    'DAY'                  : int,  \n    'DAY_OF_WEEK'          : int,  \n    'AIRLINE'              : str, \n    'FLIGHT_NUMBER'        : str,  \n    'TAIL_NUMBER'          : str, \n    'ORIGIN_AIRPORT'       : str, \n    'DESTINATION_AIRPORT'  : str, \n    'SCHEDULED_DEPARTURE'  : str,  \n    'DEPARTURE_TIME'       : str, \n    'DEPARTURE_DELAY'      : float,\n    'TAXI_OUT'             : str, \n    'WHEELS_OFF'           : str,\n    'SCHEDULED_TIME'       : float,\n    'AIR_TIME'             : float,\n    'DISTANCE'             : int,\n    'SCHEDULED_ARRIVAL'    : str,\n    'DIVERTED'             : int,  \n    'CANCELLED'            : int,  \n    'CANCELLATION_REASON'  : str\n}\n\ny_dtype = {\n    'ID'                   : int,\n    \"ARRIVAL_DELAY\"        : float\n}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Pre-processing\n\nWe read the .csv files:","metadata":{}},{"cell_type":"code","source":"prediction_df = pd.read_csv('../input/eurecom-aml-2021-challenge-1/data/test_features.csv',nrows=100000)\nX_train_df = pd.read_csv('../input/eurecom-aml-2021-challenge-1/data/train_features.csv', dtype=x_dtype,nrows=100000)\nY_train_df = pd.read_csv('../input/eurecom-aml-2021-challenge-1/data/train_targets.csv', dtype=y_dtype,nrows=100000)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Depending on our models, we sometimes had to reduce memory usage by reducing the precision of the features.","metadata":{}},{"cell_type":"code","source":"#memory optimization used in other solutions\n\n#for c in list(df.columns):\n    \n#    df[c] = pd.to_numeric(df[c])\n    \n#    m = df[c].max()\n#    n = df[c].min()\n    \n#    norm_dict[c] = m,n\n    \n#    if m==n:\n#        df[c] = df[c] - n\n#    else:\n#        df[c] = (df[c] - n) / (m - n)\n    \n#    if df[c].dtype == np.float64:\n#        df[c] = df[c].astype(np.float16)\n#    elif df[c].dtype == np.int64:\n#        df[c] = df[c].astype(np.int16)\n#   else:\n#        df[c] = df[c].astype(np.int8)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Airports latitude and longitude\n\nThis feature may be important for the model to learn the geographical aspect of delays: areas of congestion or regular meteorological phenomena for example.","metadata":{}},{"cell_type":"code","source":"#airports latitude and longitude\nairports = pd.read_csv('../input/eurecom-aml-2021-challenge-1/data/airports.csv',usecols=['IATA_CODE','LATITUDE','LONGITUDE'])\n\n#add latitude and longitude of origin and destination airports in datasets\nX_train_df = pd.merge(X_train_df, airports,left_on='ORIGIN_AIRPORT', right_on = 'IATA_CODE', how='left')\nX_train_df = X_train_df.rename(columns={\"LATITUDE\": \"ORIGIN_LAT\", \"LONGITUDE\": \"ORIGIN_LONG\"}, errors=\"raise\")\nX_train_df = pd.merge(X_train_df, airports,left_on='DESTINATION_AIRPORT', right_on = 'IATA_CODE', how='left')\nX_train_df = X_train_df.rename(columns={\"LATITUDE\": \"DESTINATION_LAT\", \"LONGITUDE\": \"DESTINATION_LONG\"}, errors=\"raise\")\n\nprediction_df = pd.merge(prediction_df, airports,left_on='ORIGIN_AIRPORT', right_on = 'IATA_CODE', how='left')\nprediction_df = prediction_df.rename(columns={\"LATITUDE\": \"ORIGIN_LAT\", \"LONGITUDE\": \"ORIGIN_LONG\"}, errors=\"raise\")\nprediction_df = pd.merge(prediction_df, airports,left_on='DESTINATION_AIRPORT', right_on = 'IATA_CODE', how='left')\nprediction_df = prediction_df.rename(columns={\"LATITUDE\": \"DESTINATION_LAT\", \"LONGITUDE\": \"DESTINATION_LONG\"}, errors=\"raise\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert date-time features from str to datetime format\ndef parse_hhmm(x):\n    try: return pd.datetime.strptime(str(x), '%H%M')\n    except: return pd.NaT\n\nX_train_df.DEPARTURE_TIME = X_train_df.DEPARTURE_TIME.apply(parse_hhmm)\nX_train_df.SCHEDULED_ARRIVAL = X_train_df.SCHEDULED_ARRIVAL.apply(parse_hhmm)\nprediction_df.DEPARTURE_TIME = prediction_df.DEPARTURE_TIME.apply(parse_hhmm)\nprediction_df.SCHEDULED_ARRIVAL = prediction_df.SCHEDULED_ARRIVAL.apply(parse_hhmm)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the arrival time and departure time into minutes since the midnight\ndef minutes_since_midnight(dt):\n    return dt.hour * 60 + dt.minute\n\nX_train_df.SCHEDULED_ARRIVAL = X_train_df.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\nX_train_df.DEPARTURE_TIME = X_train_df.DEPARTURE_TIME.apply(minutes_since_midnight)\nprediction_df.DEPARTURE_TIME = prediction_df.DEPARTURE_TIME.apply(minutes_since_midnight)\nprediction_df.SCHEDULED_ARRIVAL = prediction_df.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#merge X_train_df and Y_train_df\nmerge_df = pd.merge(X_train_df, Y_train_df, on='ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create new columns used for data exploration\n# DELAYED and DATE columns are useful to make data analytics (not for the training)\nmerge_df['DELAYED'] = (merge_df.ARRIVAL_DELAY > 0).astype(str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Add the OD feature\n\nWe use a standard deviation and mean arrival delay of an Origin-Destination (OD) feature to help the neural network to learn this very important aspect of the data.\n\n### OD mean","metadata":{}},{"cell_type":"code","source":"# Use a travel delay to help the NN\n\nairline_failure = merge_df.groupby(['AIRLINE'])['ARRIVAL_DELAY'].agg(['mean'])\nairline_failure_list = airline_failure.values\nairline_failure_list = [item for sublist in airline_failure_list for item in sublist]\nairline_failure_index = list(airline_failure.index)\n\ntail_failure = merge_df.groupby(['TAIL_NUMBER'])['ARRIVAL_DELAY'].agg(['mean'])\ntail_failure_list = tail_failure.values\ntail_failure_list = [item for sublist in tail_failure_list for item in sublist]\ntail_failure_index = list(tail_failure.index)\n\narrival_airport_failure = merge_df.groupby(['DESTINATION_AIRPORT'])['ARRIVAL_DELAY'].agg(['mean'])\narrival_airport_failure_list = arrival_airport_failure.values\narrival_airport_failure_list = [item for sublist in arrival_airport_failure_list for item in sublist]\narrival_airport_failure_index = list(arrival_airport_failure.index)\n\ntravel_failure = merge_df.groupby(['ORIGIN_AIRPORT','DESTINATION_AIRPORT'])['ARRIVAL_DELAY'].agg(['mean'])\nmerge_df['TRAVEL_ID'] = merge_df['ORIGIN_AIRPORT'] + merge_df['DESTINATION_AIRPORT']\ntravel_failure_list = travel_failure.values\ntravel_failure_list = [item for sublist in travel_failure_list for item in sublist]\ntravel_failure_index = list(travel_failure.index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df['ORIGIN_AIRPORT_2'] = merge_df['ORIGIN_AIRPORT']\nmerge_df['DESTINATION_AIRPORT_2'] = merge_df['DESTINATION_AIRPORT']\nprediction_df['ORIGIN_AIRPORT_2'] = prediction_df['ORIGIN_AIRPORT']\nprediction_df['DESTINATION_AIRPORT_2'] = prediction_df['DESTINATION_AIRPORT']\nmerge_df['AIRLINE_2'] = merge_df['AIRLINE']\nprediction_df['AIRLINE_2'] = prediction_df['AIRLINE']\nmerge_df['TAIL_NUMBER_2'] = merge_df['TAIL_NUMBER']\nprediction_df['TAIL_NUMBER_2'] = prediction_df['TAIL_NUMBER']\n\n# travel delay feature\ndef change_travel_freq(origin,dest):\n    try: return(travel_failure_list[travel_failure_index.index((origin,dest))])\n    except: return pd.NaT\n\nmerge_df['TRAVEL_FAILURE_MEAN'] = np.nan\nprediction_df['TRAVEL_FAILURE_MEAN'] = np.nan\n\nmerge_df.TRAVEL_FAILURE_MEAN = merge_df.apply(lambda x: change_travel_freq(x.ORIGIN_AIRPORT, x.DESTINATION_AIRPORT), axis=1)\nprediction_df.TRAVEL_FAILURE_MEAN = prediction_df.apply(lambda x: change_travel_freq(x.ORIGIN_AIRPORT, x.DESTINATION_AIRPORT), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# airlines delay feature\ndef change_airline_freq(airline):\n    try: return(airline_failure_list[airline_failure_index.index(airline)])\n    except: return pd.NaT\n\nmerge_df.AIRLINE = merge_df.AIRLINE.apply(change_airline_freq)\nprediction_df.AIRLINE = prediction_df.AIRLINE.apply(change_airline_freq)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tail delay feature\ndef change_tail_freq(tail):\n    try: return(tail_failure_list[tail_failure_index.index(tail)])\n    except: return pd.NaT\n                             \nmerge_df.TAIL_NUMBER = merge_df.TAIL_NUMBER.apply(change_tail_freq)\nprediction_df.TAIL_NUMBER = prediction_df.TAIL_NUMBER.apply(change_tail_freq)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# destination airport delay feature\ndef change_dest_airport_freq(airport):\n    try: return(arrival_airport_failure_list[arrival_airport_failure_index.index(airport)])\n    except: return pd.NaT\n\nmerge_df.DESTINATION_AIRPORT = merge_df.DESTINATION_AIRPORT.apply(change_dest_airport_freq)\nprediction_df.DESTINATION_AIRPORT = prediction_df.DESTINATION_AIRPORT.apply(change_dest_airport_freq)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### OD standard deviation","metadata":{}},{"cell_type":"code","source":"airline_failure = merge_df.groupby(['AIRLINE_2'])['ARRIVAL_DELAY'].agg(['std'])\nairline_failure_list = airline_failure.values\nairline_failure_list = [item for sublist in airline_failure_list for item in sublist]\nairline_failure_index = list(airline_failure.index)\n\ntail_failure = merge_df.groupby(['TAIL_NUMBER_2'])['ARRIVAL_DELAY'].agg(['std'])\ntail_failure_list = tail_failure.values\ntail_failure_list = [item for sublist in tail_failure_list for item in sublist]\ntail_failure_index = list(tail_failure.index)\n\narrival_airport_failure = merge_df.groupby(['DESTINATION_AIRPORT_2'])['ARRIVAL_DELAY'].agg(['std'])\narrival_airport_failure_list = arrival_airport_failure.values\narrival_airport_failure_list = [item for sublist in arrival_airport_failure_list for item in sublist]\narrival_airport_failure_index = list(arrival_airport_failure.index)\n\ntravel_failure = merge_df.groupby(['ORIGIN_AIRPORT_2','DESTINATION_AIRPORT_2'])['ARRIVAL_DELAY'].agg(['std'])\ntravel_failure_list = travel_failure.values\ntravel_failure_list = [item for sublist in travel_failure_list for item in sublist]\ntravel_failure_index = list(travel_failure.index)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df['TRAVEL_FAILURE_STD'] = np.nan\nprediction_df['TRAVEL_FAILURE_STD'] = np.nan\n\nmerge_df.TRAVEL_FAILURE_STD = merge_df.apply(lambda x: change_travel_freq(x.ORIGIN_AIRPORT_2, x.DESTINATION_AIRPORT_2), axis=1)\nprediction_df.TRAVEL_FAILURE_STD = prediction_df.apply(lambda x: change_travel_freq(x.ORIGIN_AIRPORT_2, x.DESTINATION_AIRPORT_2), axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df.AIRLINE_2 = merge_df.AIRLINE_2.apply(change_airline_freq)\nprediction_df.AIRLINE_2 = prediction_df.AIRLINE_2.apply(change_airline_freq)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df.TAIL_NUMBER_2 = merge_df.TAIL_NUMBER_2.apply(change_tail_freq)\nprediction_df.TAIL_NUMBER_2 = prediction_df.TAIL_NUMBER_2.apply(change_tail_freq)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df.DESTINATION_AIRPORT_2 = merge_df.DESTINATION_AIRPORT_2.apply(change_dest_airport_freq)\nprediction_df.DESTINATION_AIRPORT_2 = prediction_df.DESTINATION_AIRPORT_2.apply(change_dest_airport_freq)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merge_df = merge_df.rename(columns={\"AIRLINE\": \"AIRLINE_FAILURE_MEAN\", \"TAIL_NUMBER\": \"TAIL_FAILURE_MEAN\", \"DESTINATION_AIRPORT\": \"DESTINATION_AIRPORT_FAILURE_MEAN\", \"AIRLINE_2\": \"AIRLINE_FAILURE_STD\", \"TAIL_NUMBER_2\": \"TAIL_FAILURE_STD\", \"DESTINATION_AIRPORT_2\": \"DESTINATION_AIRPORT_FAILURE_STD\"}, errors=\"raise\")\nprediction_df = prediction_df.rename(columns={\"AIRLINE\": \"AIRLINE_FAILURE_MEAN\", \"TAIL_NUMBER\": \"TAIL_FAILURE_MEAN\", \"DESTINATION_AIRPORT\": \"DESTINATION_AIRPORT_FAILURE_MEAN\", \"AIRLINE_2\": \"AIRLINE_FAILURE_STD\", \"TAIL_NUMBER_2\": \"TAIL_FAILURE_STD\", \"DESTINATION_AIRPORT_2\": \"DESTINATION_AIRPORT_FAILURE_STD\"}, errors=\"raise\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Drop useless columns for regression\n\ncolumn_list = features_names + target_names\n\nprint(list(merge_df))\n\ntraining_set = merge_df[column_list]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data correlation","metadata":{}},{"cell_type":"code","source":"#Correlation map to see how features are correlated with Arrival Delay\ncorrmat = training_set.corr()\nplt.subplots(figsize=(12,9))\nsns.heatmap(corrmat, vmax=0.9, square=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Visualization\n\n### Features Histograms","metadata":{}},{"cell_type":"code","source":"#Features histograms\n\nmerge_df.hist(figsize = (30,30))\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Travel (OD) delays barplot","metadata":{}},{"cell_type":"code","source":"#Travels delays barplot\n\n#plt.figure(figsize=(30,30))\n#visualization_df = merge_df[~merge_df.TRAVEL_ID.duplicated(keep='first')]\n#sns.barplot(x = 'TRAVEL_ID', y = 'TRAVEL_FAILURE_MEAN', data = merge_df)\n#plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Origin-Destination feature on map","metadata":{}},{"cell_type":"code","source":"# Visualize OD feature on map\n\nfrom mpl_toolkits.basemap import Basemap\nfrom matplotlib.colors import Normalize, LinearSegmentedColormap, PowerNorm\n\ndef plot_map(color_mode='screen',\n             out_filename='flights_map_mpl.png', absolute=False):\n    \"\"\"Plots the given CSV data files use matplotlib basemap and saves it to\n    a PNG file.\n    Args:\n        in_filename: Filename of the CSV containing the data points.\n        out_filename: Output image filename\n        color_mode: Use 'screen' if you intend to use the visualisation for\n                    on screen display. Use 'print' to save the visualisation\n                    with printer-friendly colors.\n        absolute: set to True if you want coloring to depend on your dataset\n                  parameter value (ie for comparison).\n                  When set to false, each coordinate pair gets a different\n                  color.\n    \"\"\"\n\n    if color_mode == 'screen':\n        bg_color = (0.0, 0.0, 0, 1.0)\n        coast_color = (204/255.0, 0, 153/255.0, 0.7)\n        color_list = [(0.0, 0.0, 0.0, 0.0),\n                      (204/255.0, 0, 153/255.0, 0.6),\n                      (255/255.0, 204/255.0, 230/255.0, 1.0)]\n    else:\n        bg_color = (1.0, 1.0, 1.0, 1.0)\n        coast_color = (10.0/255.0, 10.0/255.0, 10/255.0, 0.8)\n        color_list = [(1.0, 1.0, 1.0, 0.0),\n                      (255/255.0, 204/255.0, 230/255.0, 1.0),\n                      (204/255.0, 0, 153/255.0, 0.6)\n                      ]\n\n    # define the expected CSV columns\n    CSV_COLS = ['ORIGIN_LONG', 'ORIGIN_LAT', 'DESTINATION_LONG', 'DESTINATION_LAT',\n                'TRAVEL_FAILURE_MEAN']\n\n    routes = merge_df[~merge_df.TRAVEL_ID.duplicated(keep='first')]\n    routes = routes[CSV_COLS]\n    routes = routes.dropna()\n\n    num_routes = len(routes.index)\n\n    # normalize the dataset for color scale\n    norm = PowerNorm(0.3, routes['TRAVEL_FAILURE_MEAN'].min(),\n                     routes['TRAVEL_FAILURE_MEAN'].max())\n    #norm = Normalize(routes['TRAVEL_FAILURE_MEAN'].min(), routes['TRAVEL_FAILURE_MEAN'].max())\n\n    # create a linear color scale with enough colors\n    if absolute:\n        n = routes['TRAVEL_FAILURE_MEAN'].max()\n    else:\n        n = num_routes\n    cmap = LinearSegmentedColormap.from_list('cmap_flights', color_list,\n                                             N=n)\n    # create the map and draw country boundaries\n    plt.figure(figsize=(135, 100))\n    m = Basemap(projection='mill', lon_0=0)\n    m.drawcoastlines(color=coast_color, linewidth=1.0)\n    m.fillcontinents(color=bg_color, lake_color=bg_color)\n    m.drawmapboundary(fill_color=bg_color)\n\n    # plot each route with its color depending on the number of flights\n    for i, route in enumerate(routes.sort_values(by='TRAVEL_FAILURE_MEAN',\n                              ascending=True).iterrows()):\n        route = route[1]\n        if absolute:\n            color = cmap(norm(int(route['nb_flights'])))\n        else:\n            color = cmap(i * 1.0 / num_routes)\n\n        line, = m.drawgreatcircle(route['ORIGIN_LONG'], route['ORIGIN_LAT'],\n                                  route['DESTINATION_LONG'], route['DESTINATION_LAT'],\n                                  linewidth=0.5, color=color)\n        # if the path wraps the image, basemap plots a nasty line connecting\n        # the points at the opposite border of the map.\n        # we thus detect path that are bigger than 30km and split them\n        # by adding a NaN\n        path = line.get_path()\n        cut_point, = np.where(np.abs(np.diff(path.vertices[:, 0])) > 30000e3)\n        if len(cut_point) > 0:\n            cut_point = cut_point[0]\n            vertices = np.concatenate([path.vertices[:cut_point, :],\n                                      [[np.nan, np.nan]],\n                                      path.vertices[cut_point+1:, :]])\n            path.codes = None  # treat vertices as a serie of line segments\n            path.vertices = vertices\n\n    # save the map\n    plt.savefig(out_filename, format='png', bbox_inches='tight')\n    \nplot_map('screen', absolute=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see on the map that some Origin-Destination elements are clearly more delayed than others. These are probably the ones that get the most traffic.","metadata":{}},{"cell_type":"markdown","source":"## Data split and rescaling","metadata":{}},{"cell_type":"code","source":"#drop lines with NaN values in departure or arrival times\n#it is ok for training but for prediction we have to keep each line\ncolumn_means = training_set.mean()\n\n#training_set = training_set.dropna()\ntraining_set = training_set.fillna(column_means)\n\nprediction_df = prediction_df.fillna(column_means)\n\nX = training_set[features_names].values\ny = training_set[target_names].values\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=0)\n\n# 4 possible types of scalers\n#sc = QuantileTransformer()\n#sc = PowerTransformer(method='yeo-johnson')\n#sc = RobustScaler()\nsc = StandardScaler()\n\nX_train = sc.fit_transform(X_train)\nX_test = sc.transform(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DataGenerator","metadata":{}},{"cell_type":"code","source":"#DataGenerator used to batch the inputs to optimize memory when using one-hot encoding\n\n#class DataGenerator(keras.utils.Sequence):\n#    'Generates data for Keras'\n    \n#    def __init__(self, df, batch_size = 32):\n#        'Initialization'\n#        self.batch_size = batch_size\n#        self.df         = df\n\n#    def __len__(self):\n#        'Denotes the number of batches per epoch'\n#        return int(np.floor(len(self.df) / self.batch_size))\n\n#    def __getitem__(self, index):\n#        'Generate one batch of data'\n        \n#        X = self.df.sample(n = self.batch_size)\n#        y = X.pop('ARRIVAL_DELAY')\n\n#        return X.values, y.values\n\n#training_generator = DataGenerator(\n#                        df         = df,\n#                        batch_size = 2**9\n#                    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"# TRAINING \n\n# DNN improvements ideas :\n# batch normalization layer between output and last dense layer\n# add dropout layers, increase model size of course and use L2 regularizer to fight overfitting\n\ncheckpoint_name = 'best_weights.hdf5' \ncheckpoint = ModelCheckpoint(checkpoint_name, monitor='val_loss', verbose = 1, save_best_only = True, mode ='auto')\ncallbacks_list = [checkpoint]\n\nNN_model = Sequential()\n\n# Input + 1st hidden layer\nNN_model.add(Dense(2048, input_shape=(X.shape[1],), activation='relu'))\n\n# Hidden layers\nNN_model.add(Dense(1024, activation='relu'))\nNN_model.add(Dense(512, activation='relu'))\nNN_model.add(Dense(256, activation='relu'))\nNN_model.add(Dense(128, activation='relu'))\nNN_model.add(Dense(64, activation='relu'))\nNN_model.add(Dense(32, activation='relu'))\n\n# Output layer\nNN_model.add(Dense(1))\n\nNN_model.compile(loss=\"mean_squared_error\" , optimizer=\"adam\", metrics=[\"mean_squared_error\"])\n\nNN_model.summary()\n\n# I added shuffle = True just in case\nhistory = NN_model.fit(X_train, y_train, batch_size=128, epochs=50, verbose=1, validation_split=0.1, validation_data=(X_test,y_test), shuffle=True, callbacks=callbacks_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Prediction","metadata":{}},{"cell_type":"code","source":"# PREDICTION\n\n# Load weights file of the best model :\nweights_file = './best_weights.hdf5' # choose the best checkpoint\nNN_model.load_weights(weights_file) # load it\n\nNN_model.evaluate(X_test,y_test)\n\nprediction_values = prediction_df[features_names].values\n\nto_predict = sc.fit_transform(prediction_values)\n\nNN_prediction = NN_model.predict(to_predict)\n\n# Create a dataframe containing the predictions\nsubmission_df = pd.DataFrame(data={'ID': prediction_df.ID.values,'ARRIVAL_DELAY': NN_prediction.squeeze()})\n\nsubmission_df.to_csv(\"./submission.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Memory check","metadata":{}},{"cell_type":"code","source":"import sys\ndef sizeof_fmt(num, suffix='B'):\n    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n        if abs(num) < 1024.0:\n            return \"%3.1f %s%s\" % (num, unit, suffix)\n        num /= 1024.0\n    return \"%.1f %s%s\" % (num, 'Yi', suffix)\n\nfor name, size in sorted(((name, sys.getsizeof(value)) for name, value in locals().items()),\n                         key= lambda x: -x[1])[:10]:\n    print(\"{:>30}: {:>8}\".format(name, sizeof_fmt(size)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Plot history","metadata":{}},{"cell_type":"code","source":"def plot_loss(history):\n  plt.plot(history.history['loss'], label='loss')\n  plt.plot(history.history['val_loss'], label='val_loss')\n  plt.ylim([0, 500])\n  plt.xlabel('Epoch')\n  plt.ylabel('Error [MSE]')\n  plt.legend()\n  plt.grid(True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_loss(history)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}