{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Air travel delay forecasting","metadata":{}},{"cell_type":"markdown","source":"#### Eliot Calimez\n#### Al Mahdi Sersouri \n#### Anas El Aissaoui \n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nfrom pylab import *\nimport seaborn as sns\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy import stats\n\nfrom matplotlib import rc \n\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_dtype = {\n    'ID'                   : int,\n    'YEAR'                 : int,  \n    'MONTH'                : int,  \n    'DAY'                  : int,  \n    'DAY_OF_WEEK'          : int,  \n    'AIRLINE'              : str, \n    'FLIGHT_NUMBER'        : str,  \n    'TAIL_NUMBER'          : str, \n    'ORIGIN_AIRPORT'       : str, \n    'DESTINATION_AIRPORT'  : str, \n    'SCHEDULED_DEPARTURE'  : str,  \n    'DEPARTURE_TIME'       : str, \n    'DEPARTURE_DELAY'      : float,\n    'TAXI_OUT'             : str, \n    'WHEELS_OFF'           : str,\n    'SCHEDULED_TIME'       : float,\n    'AIR_TIME'             : float,\n    'DISTANCE'             : int,\n    'SCHEDULED_ARRIVAL'    : str,\n    'DIVERTED'             : int,  \n    'CANCELLED'            : int,  \n    'CANCELLATION_REASON'  : str\n}\n\ny_dtype = {\n    'ID'                   : int,\n    \"ARRIVAL_DELAY\"        : float\n}\ny_train_df = pd.read_csv(\"../input/eurecom-aml-2021-challenge-1/data/train_targets.csv\", dtype=y_dtype)\nairports = pd.read_csv('/kaggle/input/eurecom-aml-2021-challenge-1/data/airports.csv',\n                       usecols=['IATA_CODE', 'AIRPORT', 'LATITUDE', 'LONGITUDE']).rename({'AIRPORT': 'NAME'}, axis='columns')\n\nX_train_df= pd.read_csv(\"../input/eurecom-aml-2021-challenge-1/data/train_features.csv\", dtype = X_dtype)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def parse_hhmm(x):\n    try: return pd.datetime.strptime(x, '%H%M')#.time()\n    except: return pd.NaT\n\nX_train_df.DEPARTURE_TIME = X_train_df.DEPARTURE_TIME.apply(parse_hhmm)\nX_train_df.SCHEDULED_ARRIVAL = X_train_df.SCHEDULED_ARRIVAL.apply(parse_hhmm)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## dropping columns that won't be available when making prediction\n    \ndf = X_train_df.drop(['DIVERTED','CANCELLED','CANCELLATION_REASON'],axis=1)\ndf = pd.merge(df,y_train_df, how='inner', on='ID')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['DELAYED'] = (df.ARRIVAL_DELAY > 0).astype(str)\ndf['DATE'] = pd.to_datetime(df[['YEAR', 'MONTH', 'DAY']])\ndf.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How many unique origin airports?\n\n","metadata":{}},{"cell_type":"code","source":"print(\"There are \",df['ORIGIN_AIRPORT'].nunique(),\" unique origin airports.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How Many unique destiantion airports: 629.","metadata":{}},{"cell_type":"code","source":"df['DESTINATION_AIRPORT'].value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## There are 14 carriers. ","metadata":{}},{"cell_type":"code","source":"print(\"Number of carriers:\", len(df.groupby('AIRLINE')['ID']))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Is there any relationship between the number of flights and the days of week?\n","metadata":{}},{"cell_type":"code","source":"l=df.groupby('DAY_OF_WEEK')['ID'].nunique()\nx=[i for i in range(1,8)]\ny=[l[i]  for i in range(1,8)]\nd=['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']\nplt.figure(figsize=(9,5))\nplt.subplot(211)\nplt.bar(d,y)\n\n\nplt.ylabel('number of flights')\n\n\nplt.subplot(212)\ntotal=column_count=df['ID'].nunique()\nyp=[l[i]/total*100  for i in range(1,8)]\nplt.bar(d,yp)\nd=['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']\nplt.ylabel('% of flights (over total flights)')\n\n\nplt.suptitle('Relation between the number of flights and the days of the week')\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How many flights that have a scheduled departure time later than 18h00?\n","metadata":{}},{"cell_type":"code","source":"subset_df=df[df['SCHEDULED_DEPARTURE'].astype(int)>1800]\ncolumn_count_sub= subset_df['ID'].nunique()\ncolumn_count=df['ID'].nunique()\nprint(\"There are \",column_count_sub,\"flights that have a scheduled departure time later than 18h00.\")\nprint(\"It represents \", \"{:.2f}\".format(round(column_count_sub/column_count*100, 2)),\"% of the flights.\" )\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## How many flights in different days of months and in different hours of days ?? ","metadata":{}},{"cell_type":"code","source":"df_per_month=[df[df['MONTH']== i] for i in range(1,13)]\nn_flights_per_m=[len(df_per_month[i]) for i in range(12)]\ndf_per_day=[df[df['DAY']== i] for i in range(1,32)]\nn_flights_per_day=[len(df_per_day[i]) for i in range(31)]\n\ndf_per_day=[df[df['DAY']== i] for i in range(1,32)]\nn_flights_per_day=[len(df_per_day[i]) for i in range(31)]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Set the width and height of the figure\nplt.figure(figsize=(20,5))\n\nsubplot(1,2,1)\nplt.title(\"number of flights per months\")\n\n# Bar chart showing average arrival delay for Spirit Airlines flights by month\nsns.barplot(x=[1,2,3,4,5,6,7,8,9,10,11,12], y=n_flights_per_m)\n\n# Add label for vertical axis\nplt.ylabel(\"number of flights\")\n\nsubplot(1,2,2)\n# Add title\nplt.title(\"number of flights per day\")\n\n# Bar chart showing average arrival delay for Spirit Airlines flights by month\nsns.barplot(x=[i for i in range(1,32)], y=n_flights_per_day)\n\n# Add label for vertical axis\nplt.ylabel(\"number of flights\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Which are the top 20 busiest airports (this depends on inbound and outbound traffic)?","metadata":{}},{"cell_type":"code","source":"df.groupby(['ORIGIN_AIRPORT']).count()['ID'].sort_values(ascending=False)\nairports_business = pd.concat([df.groupby(['ORIGIN_AIRPORT']).count()['ID'],df.groupby(['DESTINATION_AIRPORT']).count()['ID']], axis=1).agg('sum',axis=1).sort_values(ascending=False)\ntop_20_airports=list(airports_business.head(20).index)\nprint('Busiest airport:', top_20_airports)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"carrier_20  = df['AIRLINE'].value_counts().head(20)\ncarrier_20_id = list(carrier_20.index)\ndf_delayed = df[df['DELAYED'] == 'True']\ncarrier_delay = df_delayed['AIRLINE'].value_counts()\ncarrier_delay_id = list(carrier_delay.index)\nratio_delayed = [0 for i in range (len(carrier_20_id))]\n\n\nfor i in range (len(carrier_20)):\n    j = 0 \n    while carrier_20_id[i] != carrier_delay_id[j]:\n        j += 1\n    ratio_delayed[i] = [carrier_delay[j] * 100 / carrier_20[i]][0]\n\n\nplt.subplot(211)\nplt.bar(carrier_20_id,ratio_delayed)\nplt.ylabel('% of flights (over total flights)')\nplt.suptitle('20 busiest carriers')\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### What is the percentage of delayed flights (over total flights) for different hours of the day?\n","metadata":{}},{"cell_type":"code","source":"### finding percentage of delayed flights per hour:\n#for that we just take the column Delayed and SCHEDULED_DEPARTURE\n\ndf2=pd.DataFrame(df[['SCHEDULED_DEPARTURE','DELAYED']])\n\n#then we transform the scheduled time into an integer telling the hour\n#and we make count ow many for each hour are delayed or not\ndf2['s_dep']=df2.apply(lambda row: int(int(row.SCHEDULED_DEPARTURE)/100), axis=1)\ndf2t=df2[df2['DELAYED']=='True'].groupby(['s_dep']).count()\ndf2f=df2[df2['DELAYED']=='False'].groupby(['s_dep']).count()\ndf2perc=pd.DataFrame({'perc': list(100 * df2t['DELAYED']/ (df2f['DELAYED']+df2t['DELAYED'])),\n                        's_dep': list(range(24))}) \ndf2perc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.title(\"percentage of delayed flights per hour\")\n\nsns.barplot(x=list(df2perc.s_dep), y=list(df2perc.perc))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### What is the percentage of delayed flights which depart from one of the top 20 busiest airports?","metadata":{}},{"cell_type":"code","source":"top_20_airports=list(df['ORIGIN_AIRPORT'].value_counts().head(20).index)\ndf.groupby(['ORIGIN_AIRPORT']).count()['ID'].sort_values(ascending=False)\nairports_business = pd.concat([df.groupby(['ORIGIN_AIRPORT']).count()['ID'],df.groupby(['DESTINATION_AIRPORT']).count()['ID']], axis=1).agg('sum',axis=1).sort_values(ascending=False)\ntop_20_airports=list(airports_business.head(20).index)\nprint(top_20_airports)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#making a column of whether the flight is delayed\ndf['DELAYED'] = (df.ARRIVAL_DELAY > 0).astype(str)\ndelayed_flights=df['DELAYED'].value_counts()[1]\non_time_flights=df['DELAYED'].value_counts()[0]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#making columns of wether the is from/to the 20 busiest airports\ndf['from20']=(df.ORIGIN_AIRPORT.isin(top_20_airports) ).astype(str)\ndf['to20']=( df.DESTINATION_AIRPORT.isin(top_20_airports) ).astype(str)\ndf['fromto20']=( (df.DESTINATION_AIRPORT.isin(top_20_airports)) | (df.ORIGIN_AIRPORT.isin(top_20_airports))).astype(str)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**below we can see that 53% of the delays are from the 20 busiest airports**","metadata":{}},{"cell_type":"code","source":"df1=pd.DataFrame(df[['DELAYED','from20']][df.DELAYED == 'True'])\ndf1.value_counts(normalize=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#calculate the mean delay per hour\ndf3=pd.DataFrame(df[['ARRIVAL_DELAY','SCHEDULED_DEPARTURE']][df.DELAYED == 'True'])\ndf3['s_dep']=df3.apply(lambda row: int(int(row.SCHEDULED_DEPARTURE)/100), axis=1)\ndf3m=df3.groupby('s_dep').mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.title(\"mean of the arrival delays per hour\")\nsns.barplot(x=list(df3m.index), y=list(df3m.ARRIVAL_DELAY))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['s_dep']=df2.apply(lambda row: int(int(row.SCHEDULED_DEPARTURE)/100), axis=1)\ndfbis=df.groupby('s_dep').count()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(15,5))\nplt.title(\"number of flights per hour\")\nsns.barplot(x=list(dfbis.index), y=list(dfbis.ID))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Which hours of the day are characterized by the longest flight delay?","metadata":{}},{"cell_type":"code","source":"delay_hour = []\nfor i in range(24):   \n    subset_df=df[df['SCHEDULED_DEPARTURE'].astype(int)>=i*100]\n    subset_df=subset_df[subset_df['SCHEDULED_DEPARTURE'].astype(int)<(i+1)*100]\n    delay_hour.append(subset_df[subset_df['DELAYED']=='True'].max()['ARRIVAL_DELAY'])\nx = [i for i in range (24)]\n\nplt.bar(x,delay_hour)\nplt.ylabel('longest arrival delay')\nplt.xlabel('hours of day')\nplt.title('Time delay per hour day')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### & according to distance ? ","metadata":{}},{"cell_type":"code","source":"distanc = df.groupby('DISTANCE')['ID'].nunique()\ndist = distanc.index\ndista = list(dist)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"avg_delay_dist = []\nnb_delay_dist = []\n\n\nfor i in range (len(dist)):\n    df_dist = df[df['DISTANCE'] == dist[i]]\n    dist_delay = df_dist[df_dist['DELAYED'] == 'True']\n    avg_delay_dist.append(dist_delay['ARRIVAL_DELAY'].mean())\n    nb_delay_dist.append(len(dist_delay)*100/(len(df_dist)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10 ))\nplt.plot(dist,avg_delay_dist, '--')\nplt.ylabel('Arrival delay (time)')\nplt.xlabel('Distance between two airports')\nplt.title('Average Delay regarding distance between airports')\nplt.show()\n\n\nplt.figure(figsize=(10, 10))\nplt.plot(dist,nb_delay_dist, '.')\nplt.title('Ratio of Delayed flights regarding distance between airports')\nplt.xlabel('Distance between two airports')\nplt.ylabel('Percentage of delayed flights ')\nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"####  What is the percentage of delayed flights for different days of the week? (with and without taking flights in advance)?\n","metadata":{}},{"cell_type":"code","source":"delay=df.groupby('DAY_OF_WEEK')['ARRIVAL_DELAY'].sum()\nflights=df.groupby('DAY_OF_WEEK')['ID'].nunique()\n\n\n\n\nd=['Monday','Tuesday','Wednesday','Thursday','Friday','Saturday','Sunday']\ny=[delay[i]/flights[i] for i in range(1,8)]\n\n\nplt.figure(figsize=(9,7))\nplt.subplot(211)\nplt.bar(d,y)\nplt.ylabel('average delay (minuts)')\nplt.title('Average delay for different days of the week (taking into account flights in advance)')\n\n\n\n\nsubset_df=df[df['DELAYED']=='True']\ndelay2=subset_df.groupby('DAY_OF_WEEK')['ARRIVAL_DELAY'].sum()\nflights2=subset_df.groupby('DAY_OF_WEEK')['ID'].nunique()\ny2=[delay2[i]/flights2[i] for i in range(1,8)]\nplt.subplot(212)\nplt.bar(d,y2)\nplt.ylabel('average delay (minuts)')\nplt.title('Average delay for different days of the week (without flights in advance)')\n\n\n\n\nplt.show()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Number of delayed flights between two airport ! ","metadata":{}},{"cell_type":"code","source":"nb_delay = [[0 for i in range (len(top_20_airports))] for i in range (len(top_20_airports))]\n\nfor i in range (len(top_20_airports)):\n    df_airport = df[df['ORIGIN_AIRPORT'] == top_20_airports[i]] #select dataframe that correponds to one airport\n    df_airport_delayed = df_airport[df_airport['DELAYED'] == 'True'].groupby('DESTINATION_AIRPORT')['ID'].count() #select delayed flights data from this airport grouped & counted by airline distinctly\n    list_airport = list(df_airport_delayed.index)\n    for k in range (len(list_airport)):\n        j = 0\n        while list_airport[k]!= top_20_airports[j]:  #we consider only top 20 airportt\n            j += 1\n            if j >= 20: \n                break\n                \n        if j < 20 : \n            nb_delay[i][j] = df_airport_delayed[k]\n\nheat_df = pd.DataFrame(nb_delay, index=top_20_airports, columns=top_20_airports)\nplt.figure(figsize=(20, 10 ))\nplt.title('Number of delays regarding two airport  ')\nsns.heatmap(heat_df, annot=True,linewidths=.5, cmap=\"YlGnBu\",  fmt=\"d\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"airport_20 = df['ORIGIN_AIRPORT'].value_counts().head(20)\nnb_delay_ratio = [[nb_delay[j][i] * 100/airport_20[j] for i in range (len(airport_20))] for j in range(len(airport_20))]\nheat_df_r = pd.DataFrame(nb_delay_ratio, index=top_20_airports, columns=top_20_airports)\nplt.figure(figsize=(20, 10 ))\nplt.title('Ratio of delays regarding two airport  ')\nsns.heatmap(heat_df_r, annot=True,linewidths=.5, cmap=\"YlGnBu\")","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Impact of taxi out time to the delay. ","metadata":{}},{"cell_type":"code","source":"list=df.groupby(df[\"TAXI_OUT\"])[\"ID\"].count()\nmaxi=df[\"TAXI_OUT\"].nunique()\nres=0\nlist_d=[]\nlist=[]\nfor i in range(maxi):\n    list_d.append(df[(df[\"DELAYED\"]==\"True\") & (df[\"TAXI_OUT\"]==str(i))][\"ID\"].count())\n    list.append(df[df[\"TAXI_OUT\"]==str(i)][\"ID\"].count())\n    res+=df[(df[\"DELAYED\"]==\"True\") & (df[\"TAXI_OUT\"]==str(i))][\"ID\"].count()\nprint(res)\nprint(list)\nprint(list_d)\n\nx=[i for i in range(maxi)]\ny=[list_d[i]/list[i]*100 for i in range(maxi)]\n\nplt.plot(x,y)\nplt.xlabel('Taxi out (in min')\nplt.ylabel('% of delayed flights')\nplt.title('Influence of the feature \"TAXI_OUT\"')\n\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data pre-processing","metadata":{}},{"cell_type":"code","source":"df['YEAR'].describe()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().values.any()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we clean the dataset and only keep columns we believe meaningful \n#(for example exclude year that is the same for all and Flight number which couldnt be a reason for delay/\n# and which info is known before the flight for exemple we took out wheel_off and AIR_TIME(which seemed to be an information entered after the flight ended)\n# also rather than keeping the airports of origin/destination we guessed we could simplify our model ( maybe change this later and measure the difference)\n# by replacing these columns by just if the flight is from or to one of the top20 airports\ndf = pd.merge(df,y_train_df, how='inner', on='ID')\nfeatures = ['DAY',  'DAY_OF_WEEK', 'AIRLINE', 'TAIL_NUMBER', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT', 'SCHEDULED_DEPARTURE',\n                'DEPARTURE_DELAY','WHEELS_OFF','TAXI_OUT','AIR_TIME', 'SCHEDULED_TIME', 'SCHEDULED_ARRIVAL']\ntarget_name = ['ARRIVAL_DELAY']\ndfc = df[features]\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def minutes_since_midnight(dt):\n    return dt.hour * 60 + dt.minute\ndef parse_hhmm(x):\n    try: return pd.datetime.strptime(x, '%H%M')#.time()\n    except: return pd.NaT\n\ndfc.SCHEDULED_DEPARTURE = dfc.SCHEDULED_DEPARTURE.apply(parse_hhmm)\ndfc.SCHEDULED_ARRIVAL = dfc.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\ndfc.SCHEDULED_DEPARTURE = dfc.SCHEDULED_DEPARTURE.apply(minutes_since_midnight)\ndfc.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#in order to decide whether we want to normalize or standarize our data let's see how they are distributed:\n\ndecide = dfc\nordonnées = [list(decide.groupby(feature).count()['DESTINATION_AIRPORT']) for feature in features]\nabscisses = [list(decide.groupby(feature).count().index) for feature in features]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig = plt.figure(figsize=(15,20))\na1 = fig.add_axes([0,0,1,1])\nfor i in range(4):\n    \n    plt.subplot(4,1,i+1)\n    plt.title(features[i])\n    plt.plot(abscisses[i],ordonnées[i])\n\n\nplt.show()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we use the z method to check if there are any outliners\nz = np.abs(stats.zscore(dfc))\nthreshold=3\nprint(np.where(z > threshold))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfc[[ 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT','TAIL_NUMBER']]= dfc[[ 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT','TAIL_NUMBER']].astype('category')\ndfc['ORIGIN_AIRPORT_b']= dfc[ 'ORIGIN_AIRPORT'].cat.codes\ndfc['DESTINATION_AIRPORT_b']= dfc[ 'DESTINATION_AIRPORT'].cat.codes\ndfc['TAIL_NUMBER_b']= dfc['TAIL_NUMBER'].cat.codes\ndfcc= dfc.drop( ['ORIGIN_AIRPORT', 'DESTINATION_AIRPORT','TAIL_NUMBER'], axis=1)\ndfccd=dfcc\ndfccd['AIRLINE']=df['AIRLINE']\ndfccd = pd.get_dummies(dfccd, columns=['AIRLINE'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(\n   dfccd, y_train_df, random_state=0, test_size=0.3)\ny_val = y_val[['ARRIVAL_DELAY']]\ny_train = y_train[['ARRIVAL_DELAY']]\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(X_train)\nX_train = imputer.transform(X_train)\nX_val = imputer.transform(X_val)\ny_train.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_scaler = StandardScaler()\nX_scaler.fit(X_train)\nX_train = X_scaler.transform(X_train)\nX_val = X_scaler.transform(X_val)\ny_scaler = StandardScaler()\ny_scaler.fit(y_train)\ny_train = y_scaler.transform(y_train.values)\ny_val = y_scaler.transform(y_val.values)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training model","metadata":{}},{"cell_type":"markdown","source":"### Some useful functions \n","metadata":{}},{"cell_type":"markdown","source":"###### Accuracy\n","metadata":{}},{"cell_type":"code","source":"def accuracy(y_hat, y):\n    n = len(y)\n    return sum([y_hat[i] == y[i] for i in range(n)])/n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###### Make prediction","metadata":{}},{"cell_type":"code","source":"def make_prediction(X, model, scaler):\n    \"\"\"Makes predictions given a preprocessed dataset.\n\n    Parameters\n    ----------\n    X: numpy array.\n        The input data, which already is pre-processed.\n    model: an hypopt or sklearn model.\n        The trained model used for making predictions.\n    scaler: sklearn.preprocessing.StandardScaler.\n        The scaler used to normalize the targets.\n\n    Returns\n    -------\n    y_pred: numpy array.\n        The unnormalized predictions.\n    \"\"\"\n    y_pred = scaler.inverse_transform(model.predict(X))\n    return y_pred","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### We will go for some regression methods : the random forest, the neural network and the ridge regression. ","metadata":{}},{"cell_type":"markdown","source":"## Ridge regression ","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\nfrom sklearn.metrics import accuracy_score\nclassifierR = Ridge(alpha=0)\nclassifierR.fit(X_train, y_train)\ny_train_hat = classifierR.predict(X_train)\n\n\nprint(y_hat_train)\nprint(f\"Training accuracy: {100*accuracy_score(y_train, y_train_hat)}:.3f%\")\n\n\ny_val_hat = classifierR.predict(X_val)\n\nprint(f\"Testing accuracy: {100*accuracy_score(y_val, y_val_hat)}:.3f%\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from hypopt import GridSearch\n\nparam_grid = [\n  {'alpha': [0.001, 0.01, 0.1, 0., 1.0, 10.0, 100.]},\n]\n\nopt = GridSearch(model=Ridge(tol=0.0001), param_grid=param_grid)\nopt.fit(X_train, y_train, scoring='neg_mean_squared_error')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred_ridge = make_prediction(X_train, opt, y_scaler)\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred_ridge, y_train))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random forest training","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import Ridge\nfrom sklearn.ensemble import RandomForestRegressor\n\n##classifier = RandomForestRegressor(max_depth=20000)\n##classifier.fit(X_train,y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Prediction training & testing ","metadata":{}},{"cell_type":"code","source":"y_train_pred = make_prediction(X_train, classifier, y_scaler)\ny_test_pred = make_prediction(X_val, classifier, y_scaler)\n\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))\ntest_rmse = np.sqrt(mean_squared_error(y_test_pred, y_val))\n\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\nprint(\"Testing RMSE: {:.5f}\".format(float(test_rmse)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Neural Network training","metadata":{}},{"cell_type":"code","source":"from sklearn.neural_network import MLPRegressor\n\n##classifierNN = MLPRegressor(solver='lbfgs', alpha=1e-3, hidden_layer_sizes=(20,20,20), random_state=1)\n##classifierNN.fit(X_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Prediction training & testing ","metadata":{}},{"cell_type":"code","source":"y_train_pred = make_prediction(X_train, classifierNN, y_scaler)\ny_test_pred = make_prediction(X_val, classifierNN, y_scaler)\n\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train))\ntest_rmse = np.sqrt(mean_squared_error(y_test_pred, y_val))\n\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\nprint(\"Testing RMSE: {:.5f}\".format(float(test_rmse)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Selection & submission","metadata":{}},{"cell_type":"markdown","source":"#### The random forest accuracy seems to be way better ","metadata":{}},{"cell_type":"markdown","source":"## Submission & conclusion","metadata":{}},{"cell_type":"code","source":"\n# read in test data (won't have labels)\nX_test_df = pd.read_csv(\"data/test_features.csv\", dtype=X_dtype)\ndft = X_test_df.drop(['DIVERTED','CANCELLED','CANCELLATION_REASON'],axis=1)\ndft = dft[features]\ndft.head()","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dft.SCHEDULED_DEPARTURE = dft.SCHEDULED_DEPARTURE.apply(parse_hhmm)\ndft.SCHEDULED_ARRIVAL = dft.SCHEDULED_ARRIVAL.apply(parse_hhmm)\ndft.SCHEDULED_ARRIVAL = dft.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\ndft.SCHEDULED_DEPARTURE = dft.SCHEDULED_DEPARTURE.apply(minutes_since_midnight)\ndft[[ 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT','TAIL_NUMBER']]= dft[[ 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT','TAIL_NUMBER']].astype('category')\ndft['ORIGIN_AIRPORT_b']= dft[ 'ORIGIN_AIRPORT'].cat.codes\ndft['DESTINATION_AIRPORT_b']= dft[ 'DESTINATION_AIRPORT'].cat.codes\ndft['TAIL_NUMBER_b']= dft[ 'TAIL_NUMBER'].cat.codes\ndfttd = dft.drop( ['ORIGIN_AIRPORT', 'DESTINATION_AIRPORT','TAIL_NUMBER'], axis=1)\nxtest = dfttd\nxtest['AIRLINE']=dft['AIRLINE']\nxtest = pd.get_dummies(xtest, columns=['AIRLINE'])\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(xtest)\nxtest = imputer.transform(xtest)\nX_scaler = StandardScaler()\nX_scaler.fit(xtest)\nX_train = X_scaler.transform(xtest)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred = make_prediction(X_train, classifier, y_scaler)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### import csv\n\nwith open('submission_pred.csv', mode='w') as submission:\n    #sub_writer = csv.writer(submission, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n    #sub_writer.writerow(y_hat_test_sub)\n    fieldnames = ['ID', 'ARRIVAL_DELAY']\n    writer = csv.DictWriter(submission, fieldnames=fieldnames)\n    writer.writeheader()\n    for i in range (len(y_hat_test_sub)):\n        writer.writerow({'ID':X_test_df['ID'][i] , 'ARRIVAL_DELAY': y_train_pred[i]})\n     ","metadata":{},"execution_count":null,"outputs":[]}]}