{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Analysis & Data processing","metadata":{}},{"cell_type":"markdown","source":"## Data Analysis","metadata":{}},{"cell_type":"code","source":"X_dtype = {\n    'ID'                   : int,\n    'YEAR'                 : int,  \n    'MONTH'                : int,  \n    'DAY'                  : int,  \n    'DAY_OF_WEEK'          : int,  \n    'AIRLINE'              : str, \n    'FLIGHT_NUMBER'        : str,  \n    'TAIL_NUMBER'          : str, \n    'ORIGIN_AIRPORT'       : str, \n    'DESTINATION_AIRPORT'  : str, \n    'SCHEDULED_DEPARTURE'  : str,  \n    'DEPARTURE_TIME'       : str, \n    'DEPARTURE_DELAY'      : float,\n    'TAXI_OUT'             : str, \n    'WHEELS_OFF'           : str,\n    'SCHEDULED_TIME'       : float,\n    'AIR_TIME'             : float,\n    'DISTANCE'             : int,\n    'SCHEDULED_ARRIVAL'    : str,\n    'DIVERTED'             : int,  \n    'CANCELLED'            : int,  \n    'CANCELLATION_REASON'  : str\n}\n\ny_dtype = {\n    'ID'                   : int,\n    \"ARRIVAL_DELAY\"        : float\n}\n\nX_train_df = pd.read_csv(\"/kaggle/input/eurecom-aml-2021-challenge-1/data/train_features.csv\", dtype=X_dtype)\ny_train_df = pd.read_csv(\"/kaggle/input/eurecom-aml-2021-challenge-1/data/train_targets.csv\", dtype=y_dtype)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge feature dataframe and target dataframe for data exploration\ndf = pd.merge(X_train_df, y_train_df, on='ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"missing_df = df.isnull().sum(axis=0).reset_index()\nmissing_df.columns = ['variable', 'missing values']\nmissing_df['filling factor (%)']=(df.shape[0]-missing_df['missing values'])/df.shape[0]*100\nmissing_df.sort_values('filling factor (%)').reset_index(drop = True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['ID', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT']].head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--> some airports (origin and destination) have IATA_code in the wrong format","metadata":{}},{"cell_type":"markdown","source":"## Creation of corrected dataset","metadata":{}},{"cell_type":"code","source":"number_name = pd.read_csv(\"/kaggle/input/bts/L_AIRPORT_ID.csv\")\nname_IATA = pd.read_csv(\"/kaggle/input/bts/L_AIRPORT.csv\")\nnumber_IATA =  number_name.merge(name_IATA, on=\"Description\")\nnumber_IATA.Code_x = number_IATA.Code_x.astype(str)\nnumber_IATA = number_IATA.set_index('Code_x')\n\ndef convert_airports(df):\n    cond_origin = (df.ORIGIN_AIRPORT.apply(len)==5)\n    f= lambda x: number_IATA.loc[str(x), 'Code_y']\n    df.loc[cond_origin, 'ORIGIN_AIRPORT'] = df.loc[cond_origin, 'ORIGIN_AIRPORT'].map(f)\n    cond_dest = (df.DESTINATION_AIRPORT.apply(len)==5)\n    df.loc[cond_dest, 'DESTINATION_AIRPORT'] = df.loc[cond_dest, 'DESTINATION_AIRPORT'].map(f)\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_df = convert_airports(X_train_df)\n\nX_test_df = pd.read_csv(\"/kaggle/input/eurecom-aml-2021-challenge-1/data/test_features.csv\", dtype=X_dtype)\nX_test_df = convert_airports(X_test_df)\n\n#Fixing some issues with the conversion\nX_train_df.loc[(X_train_df.ORIGIN_AIRPORT == 'Code_x\\n16218    NYL\\n16218    YUM\\nName: Code_y, dtype: object'),'ORIGIN_AIRPORT']='NYL'\nX_train_df.loc[(X_train_df.ORIGIN_AIRPORT == 'Code_x\\n10423    AUS\\n10423    BSM\\nName: Code_y, dtype: object'),'ORIGIN_AIRPORT']='AUS'\nX_train_df.loc[(X_train_df.DESTINATION_AIRPORT == 'Code_x\\n16218    NYL\\n16218    YUM\\nName: Code_y, dtype: object'),'DESTINATION_AIRPORT']='NYL'\nX_train_df.loc[(X_train_df.DESTINATION_AIRPORT == 'Code_x\\n10423    AUS\\n10423    BSM\\nName: Code_y, dtype: object'),'DESTINATION_AIRPORT']='AUS'\nX_test_df.to_csv(\"/kaggle/working/X_train_df.csv\")\n\nX_test_df.loc[(X_test_df.ORIGIN_AIRPORT == 'Code_x\\n16218    NYL\\n16218    YUM\\nName: Code_y, dtype: object'),'ORIGIN_AIRPORT']='NYL'\nX_test_df.loc[(X_test_df.ORIGIN_AIRPORT == 'Code_x\\n10423    AUS\\n10423    BSM\\nName: Code_y, dtype: object'),'ORIGIN_AIRPORT']='AUS'\nX_test_df.loc[(X_test_df.DESTINATION_AIRPORT == 'Code_x\\n16218    NYL\\n16218    YUM\\nName: Code_y, dtype: object'),'DESTINATION_AIRPORT']='NYL'\nX_test_df.loc[(X_test_df.DESTINATION_AIRPORT == 'Code_x\\n10423    AUS\\n10423    BSM\\nName: Code_y, dtype: object'),'DESTINATION_AIRPORT']='AUS'\nX_test_df.to_csv(\"/kaggle/working/X_test_df.csv\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#After saving the new dataset, to work with it\nX_train_df = pd.read_csv(\"/kaggle/input/corrected-dataset-airports/X_train_df.csv\", dtype=X_dtype)\ny_train_df = pd.read_csv(\"/kaggle/input/eurecom-aml-2021-challenge-1/data/train_targets.csv\", dtype=y_dtype)\n\nX_test_df = pd.read_csv(\"/kaggle/input/corrected-dataset-airports/X_test_df.csv\", dtype=X_dtype)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Analysis : study of parameters","metadata":{}},{"cell_type":"code","source":"# Convert date-time features from str to datetime format\ndef parse_hhmm(x):\n    try: return pd.datetime.strptime(x, '%H%M')#.time()\n    except: return pd.NaT\n\nX_train_df.DEPARTURE_TIME = X_train_df.DEPARTURE_TIME.apply(parse_hhmm)\nX_train_df.SCHEDULED_ARRIVAL = X_train_df.SCHEDULED_ARRIVAL.apply(parse_hhmm)\n\n# Merge feature dataframe and target dataframe for data exploration\ndf = pd.merge(X_train_df, y_train_df, on='ID')\n\ndf['DELAYED'] = df.ARRIVAL_DELAY > 0\ndf['DATE'] = pd.to_datetime(df[['YEAR', 'MONTH', 'DAY']])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[['ID', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT']].head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_stats_delay(group):\n    return {'mean_delay': group.mean(), 'ratio_delay': len(group[group>0])/len(group)}","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Departure time","metadata":{}},{"cell_type":"code","source":"delayed = df['DEPARTURE_DELAY'].groupby(df['DEPARTURE_TIME'].dt.hour).apply(get_stats_delay).unstack()\nplt.xlabel(\"Hours\")\nplt.ylabel(\"Ratio of departure delay\")\nplt.title('The radio of departure delay over hours in day')\nplt.grid(True,which=\"both\",ls=\"-\")\nbars = plt.bar(range(0,24), delayed['ratio_delay'], align='center', edgecolor = \"black\")\n\nfor i in range(0, len(bars)):\n    color = 'red'\n    if delayed['mean_delay'][i] < 0:\n        color = 'lightgreen'\n    elif delayed['mean_delay'][i] < 2:\n        color = 'green'\n    elif delayed['mean_delay'][i] < 4:\n        color = 'yellow'\n    elif delayed['mean_delay'][i] < 8:\n        color = 'orange'\n\n    bars[i].set_color(color)\n        \npatch1 = mpatches.Patch(color='lightgreen', label='Depart earlier')\npatch2 = mpatches.Patch(color='green', label='departure delay < 2 minutes')\npatch3 = mpatches.Patch(color='yellow', label='departure delay < 4 minutes')\npatch4 = mpatches.Patch(color='orange', label='departure delay < 8 minutes')\npatch5 = mpatches.Patch(color='red', label='departure delay >= 8 minutes')\nplt.legend(handles=[patch1, patch2, patch3, patch4, patch5], bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n\nplt.margins(0.05, 0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Airlines","metadata":{}},{"cell_type":"code","source":"delayed_airlines = df['ARRIVAL_DELAY'].groupby(df['AIRLINE']).apply(get_stats_delay).unstack()\nplt.xlabel(\"Airlines\")\nplt.ylabel(\"Ratio of arrival delay\")\nplt.title('The radio of arrival delay depending on airlines')\nplt.grid(True,which=\"both\",ls=\"-\")\nbars = plt.bar(delayed_airlines.index, delayed_airlines['ratio_delay'], align='center', edgecolor = \"black\")\n\nfor i in range(0, len(bars)):\n    color = 'red'\n    if delayed_airlines['mean_delay'][i] < 0:\n        color = 'lightgreen'\n    elif delayed_airlines['mean_delay'][i] < 2:\n        color = 'green'\n    elif delayed_airlines['mean_delay'][i] < 4:\n        color = 'yellow'\n    elif delayed_airlines['mean_delay'][i] < 8:\n        color = 'orange'\n\n    bars[i].set_color(color)\n        \npatch1 = mpatches.Patch(color='lightgreen', label='Arrival earlier')\npatch2 = mpatches.Patch(color='green', label='arrival delay < 2 minutes')\npatch3 = mpatches.Patch(color='yellow', label='arrival delay < 4 minutes')\npatch4 = mpatches.Patch(color='orange', label='arrival delay < 8 minutes')\npatch5 = mpatches.Patch(color='red', label='arrival delay >= 8 minutes')\nplt.legend(handles=[patch1, patch2, patch3, patch4, patch5], bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n\nplt.margins(0.05, 0)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Airports","metadata":{}},{"cell_type":"code","source":"def get_stats(group):\n    return {'min': group.min(), 'max': group.max(),\n            'count': group.count(), 'mean': group.mean()}\n#ORIGIN AIRPORT\ninbound = df['ARRIVAL_DELAY'].groupby(df['ORIGIN_AIRPORT']).apply(get_stats).unstack()\ninbound = inbound.sort_values('count')\n\n#DESTINATION AIPORT\noutbound = df['ARRIVAL_DELAY'].groupby(df['DESTINATION_AIRPORT']).apply(get_stats).unstack()\noutbound = outbound.sort_values('count')\n\n_df1 = df.groupby('ORIGIN_AIRPORT').agg({'ARRIVAL_DELAY':'count'}).rename(columns={'ARRIVAL_DELAY': 'COUNT'}).rename_axis('AIRPORT')\n_df1=_df1.sort_values('COUNT')\n_df2 = df.groupby('DESTINATION_AIRPORT').agg({'ARRIVAL_DELAY':'count'}).rename(columns={'ARRIVAL_DELAY': 'COUNT'}).rename_axis('AIRPORT')\n_df2=_df2.sort_values('COUNT')\n\nn=15\ntop_airports = _df1.join(_df2, rsuffix='_ORIGIN', lsuffix='_DEST').sum(axis=1).sort_values(ascending=False).index[:n]\nbottom_airports = _df1.join(_df2, rsuffix='_ORIGIN', lsuffix='_DEST').sum(axis=1).sort_values(ascending=True).index[:n]\n\n_df1 = df.groupby('ORIGIN_AIRPORT').agg({'ARRIVAL_DELAY':'count'}).rename(columns={'ARRIVAL_DELAY': 'COUNT'}).rename_axis('AIRPORT')\n_df1=_df1.sort_values('COUNT')\n_df2 = df.groupby('DESTINATION_AIRPORT').agg({'ARRIVAL_DELAY':'count'}).rename(columns={'ARRIVAL_DELAY': 'COUNT'}).rename_axis('AIRPORT')\n_df2=_df2.sort_values('COUNT')\n\nn=15\ntop_airports = _df1.join(_df2, rsuffix='_ORIGIN', lsuffix='_DEST').sum(axis=1).sort_values(ascending=False).index[:n]\nbottom_airports = _df1.join(_df2, rsuffix='_ORIGIN', lsuffix='_DEST').sum(axis=1).sort_values(ascending=True).index[:n]\n_airports = [0]*n*2;\nfor i in range (n):\n    _airports[i]=top_airports[i]\nfor i in range (n):\n    _airports[n+i]=bottom_airports[n-i-1]\n    \ndelayed_flights_a = [df[(df.ARRIVAL_DELAY > 0) & (df.ORIGIN_AIRPORT == _airports[i])].count()[1] for i in range(2*n)]\ntotal_flights_dest_a = [df[(df.DESTINATION_AIRPORT == _airports[i])].count()[1] for i in range(2*n)]\ntotal_flights_src_a = [df[(df.ORIGIN_AIRPORT == _airports[i])].count()[1] for i in range(2*n)]\ntotal_flights = [total_flights_dest_a[i] + total_flights_src_a[i] for i in range (2*n)]\n\npercent_delay_a = [delayed_flights_a[i]/total_flights_src_a[i] for i in range(2*n)] \n\nplt.figure(figsize=(18,6))\nbars=plt.bar(_airports,percent_delay_a)\nfor i in range (0,n):\n    bars[i].set_facecolor('purple')\nfor j in range (n, 2*n):\n    bars[j].set_facecolor('olive')\nplt.title('Delay probability for each of the 15 busiest airports and 15 freest airports')\nplt.show()\n\nplt.figure(figsize=(18,6))\nplt.plot(_airports, percent_delay_a, 'r--')\nplt.title('Flight volume for each of the 15 busiest airports and 15 freest airports')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Flight number","metadata":{}},{"cell_type":"code","source":"def get_stats2(group):\n    return {'min': group.min(), 'max': group.max(),\n            'count': group.count(), 'mean': group.mean(), 'variance': group.var()}\nfl_nb = df['ARRIVAL_DELAY'].groupby(df['FLIGHT_NUMBER']).apply(get_stats2).unstack()\nfl_nb = fl_nb.sort_values('mean')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preparation of the data","metadata":{}},{"cell_type":"code","source":"X_train_df, X_val_df, y_train_df, y_val_df = train_test_split(\n    X_train_df, y_train_df, random_state=1, test_size=0.2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert the arrival time and departure time into minutes since the midnight\ndef minutes_since_midnight(dt):\n    return dt.hour * 60 + dt.minute\n\nX_train_df.SCHEDULED_ARRIVAL = X_train_df.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\nX_train_df.DEPARTURE_TIME = X_train_df.DEPARTURE_TIME.apply(minutes_since_midnight)\n\nX_val_df.SCHEDULED_ARRIVAL = X_val_df.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\nX_val_df.DEPARTURE_TIME = X_val_df.DEPARTURE_TIME.apply(minutes_since_midnight)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_data(df, feature_names, imputer, scaler):\n    \"\"\"Preprocess data.\n\n    Parameters\n    ----------\n    df: pandas DataFrame.\n        The input data.\n    feature_names: list of strings.\n        The names of selected features.\n    imputer: sklearn.impute.SimpleImputer\n        The imputation transformer for completing missing values.\n    scaler: sklearn.preprocessing.StandardScaler.\n        The scaler used to normalize the features.\n\n    Returns\n    -------\n    X: numpy array.\n        The preprocessed data.\n    \"\"\"\n    # Select features\n    X_df = df[feature_names]\n    \n    # Pre-process datetime features\n    X_df.DEPARTURE_TIME = X_df.DEPARTURE_TIME.apply(parse_hhmm)\n    X_df.SCHEDULED_ARRIVAL = X_df.SCHEDULED_ARRIVAL.apply(parse_hhmm)\n\n    X_df.SCHEDULED_ARRIVAL = X_df.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\n    X_df.DEPARTURE_TIME = X_df.DEPARTURE_TIME.apply(minutes_since_midnight)\n\n    # Impute missing values\n    X = imputer.transform(X_df)\n    \n    # Normalize features\n    X = scaler.transform(X)\n\n    return X","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Comparing encodings","metadata":{}},{"cell_type":"code","source":"#### Numeric encoding\nfrom sklearn import preprocessing\nfeature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'AIR_TIME', 'SCHEDULED_ARRIVAL', 'DEPARTURE_TIME', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT', 'AIR_TIME', 'AIRLINE']\ndf_numeric_train = X_train_df.copy()\n\ndf_numeric_val = X_val_df.copy()\n\nle = preprocessing.LabelEncoder()\ntrained_le = le.fit(df_numeric_train['AIRLINE'])\ndf_numeric_train['AIRLINE'] = trained_le.transform(df_numeric_train['AIRLINE'])\ndf_numeric_val['AIRLINE'] = trained_le.transform(df_numeric_val['AIRLINE'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### Delay mean\n#feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'AIR_TIME', 'SCHEDULED_ARRIVAL', 'DEPARTURE_TIME', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT', 'AIR_TIME', 'AIRLINE']\ndef get_stats(group):\n    return {'mean': group.mean()}\n#DEPARTURE DELAY\nglobal_stats_airline = df['ARRIVAL_DELAY'].groupby(df['AIRLINE']).apply(get_stats).unstack()\n\ndf_numeric_train = X_train_df.copy()\ndf_numeric_train.AIRLINE = df_numeric_train.AIRLINE.map(global_stats_airline['mean'])\n\ndf_numeric_val = X_val_df.copy()\ndf_numeric_val.AIRLINE = df_numeric_train.AIRLINE.map(global_stats_airline['mean'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### Ordinal encoding(delay mean)\n\ndef get_stats(group):\n    return {'mean': group.mean()}\n#DEPARTURE DELAY\nglobal_stats_airline = df['ARRIVAL_DELAY'].groupby(df['AIRLINE']).apply(get_stats).unstack()\n\ndf_numeric_train = X_train_df.copy()\ndf_numeric_train.AIRLINE = df_numeric_train.AIRLINE.map(global_stats_airline['mean'])\n\ndf_numeric_val = X_val_df.copy()\ndf_numeric_val.AIRLINE = df_numeric_val.AIRLINE.map(global_stats_airline['mean'])\n\nfrom sklearn.preprocessing import OrdinalEncoder\nenc = OrdinalEncoder()\ndf_numeric_train['AIRLINE'] = enc.fit_transform(df_numeric_train[['AIRLINE']])\ndf_numeric_val['AIRLINE'] = enc.fit_transform(df_numeric_val[['AIRLINE']])\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#### One-hot encoding\ndf_numeric_train = X_train_df.copy()\ndf_numeric_val = X_val_df.copy()\n\ndf_hot_train =pd.get_dummies(df_numeric_train.AIRLINE)\ndf_hot_val =pd.get_dummies(df_numeric_val.AIRLINE)\n\ndf_numeric_train = df_numeric_train.merge(df_hot_train,left_index=True, right_index=True)\ndf_numeric_val = df_numeric_val.merge(df_hot_val,left_index=True, right_index=True)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"-> choose one hot encoding of airline and numeric encoding of airports","metadata":{}},{"cell_type":"code","source":"def encoding(df):\n    df_hot_air = pd.get_dummies(df.AIRLINE)\n    df = df.merge(df_hot_air,left_index=True, right_index=True)\n    #feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'AIR_TIME', 'SCHEDULED_ARRIVAL', 'DEPARTURE_TIME', 'ORIGIN_AIRPORT', 'DEPARTURE_DELAY', 'AIR_TIME']+list(df_hot_air.columns)\n    feature_names = ['MONTH', 'DAY', 'DAY_OF_WEEK', 'DISTANCE', 'AIR_TIME', 'SCHEDULED_ARRIVAL', 'DEPARTURE_TIME', 'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT', 'AIR_TIME', 'AIRLINE']\n    return df, feature_names","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import preprocessing\n\n\n#df_numeric_train, feature_names = encoding(df_numeric_train)\n\n#df_numeric_val,_ = encoding(df_numeric_val)\n\nle = preprocessing.LabelEncoder()\ntrained_le = le.fit(df_numeric_train['ORIGIN_AIRPORT'])\ndf_numeric_train['ORIGIN_AIRPORT'] = trained_le.transform(df_numeric_train['ORIGIN_AIRPORT'])\ndf_numeric_val['ORIGIN_AIRPORT'] = trained_le.transform(df_numeric_val['ORIGIN_AIRPORT'])\ndf_numeric_train['DESTINATION_AIRPORT'] = trained_le.transform(df_numeric_train['DESTINATION_AIRPORT'])\ndf_numeric_val['DESTINATION_AIRPORT'] = trained_le.transform(df_numeric_val['DESTINATION_AIRPORT'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_name = ['ARRIVAL_DELAY']\nX_train_df_num = df_numeric_train[feature_names]\ny_train_df_num = y_train_df[target_name]\nX_val_df_num = df_numeric_val[feature_names]\ny_val_df_num = y_val_df[target_name]\n\n# Filling missing values by the mean along each column.\n# These statistics should be estimated by using the training set.\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(X_train_df_num)\nX_train_num = imputer.transform(X_train_df_num)\nX_val_num = imputer.transform(X_val_df_num)\n\n# Standardize the features by removing the mean and scaling to unit variance\n# Similarly to the preivous step, the statistics used for standardization\n# should be computed across the training set only\nX_scaler = StandardScaler()\nX_scaler.fit(X_train_num)\nX_train_num = X_scaler.transform(X_train_num)\nX_val_num = X_scaler.transform(X_val_num)\n\n\n# We should also standardize the targets.\ny_scaler = StandardScaler()\ny_scaler.fit(y_train_df_num)\ny_train_num = y_scaler.transform(y_train_df_num)\ny_val_num = y_scaler.transform(y_val_df_num)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"def make_prediction(X, model, scaler):\n    \"\"\"Makes predictions given a preprocessed dataset.\n\n    Parameters\n    ----------\n    X: numpy array.\n        The input data, which already is pre-processed.\n    model: an hypopt or sklearn model.\n        The trained model used for making predictions.\n    scaler: sklearn.preprocessing.StandardScaler.\n        The scaler used to normalize the targets.\n\n    Returns\n    -------\n    y_pred: numpy array.\n        The unnormalized predictions.\n    \"\"\"\n    y_pred = scaler.inverse_transform(model.predict(X))\n    return y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Linear regression","metadata":{}},{"cell_type":"markdown","source":"### Hand made","metadata":{}},{"cell_type":"code","source":"def prepare_data(array):\n    variable=array.copy()\n    #variable=variable.apply(lambda x: pd.to_numeric(x, errors='ignore'))\n    #variable_array = np.array(variable)\n    variable_pol = PolynomialFeatures(1)\n    variable_pol = variable_pol.fit_transform(variable)\n\n    return variable_pol\n\ninput_train = prepare_data(X_train)\ninput_val = prepare_data(X_val)\noutput_train = np.array(y_train)\noutput_val = np.array(y_val)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class my_linear_regression:\n    def __init__(self) : # initialize constructor for the object to assign the object its properties\n        self.X_train = []\n        self.y_train = []\n        self.weights = []\n        \n    def fit(self, X, y) :\n        self.X_train = X\n        self.y_train = y\n        self.weights = np.linalg.solve(X.T@X,X.T@y)\n        #print(len(self.weights))\n        #print(self.weights)\n        self.weights=np.transpose(self.weights)\n        #print(len(self.weights[0]))\n        #print(self.weights)\n    \n    def predict(self,x_test,y_test) : # method of the object that can be used\n        self.y_hat=np.sum(x_test*self.weights,axis=1)\n        #self.MSE= (np.sum((y_test-self.y_hat)**2))/len(y_test)\n        \n        return self.y_hat\nmodel_1 = my_linear_regression()\nmodel_1.fit(input_train, output_train)\nmodel_1.predict(input_val, output_val)\n\n#MSE_1_train = model_1.MSE\nprint(\"weight[0] : {}, weight[1:] : {}\".format(model_1.weights[0][0], model_1.weights[0][1:]))\n#print(\"MSE : {}\".format(MSE_1_train))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### With Library","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nregr = LinearRegression(\n   fit_intercept = True, normalize = True, copy_X = True, n_jobs = 2\n).fit(X_train,y_train)\nregr.predict(X_val)\nprint(regr.score(X_train,y_train))\n#print(\"Weights:\")\n#print(regr.coef_)\n#print(regr.intercept_)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coeff_df = pd.DataFrame(np.concatenate((regr.coef_[0],regr.intercept_)), feature_names+['w[0]'] ,columns=['Weights'])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred = make_prediction(X_train, regr, y_scaler)\ntrain_rmse = np.sqrt(mean_squared_error(y_train_pred, y_train_df.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\n\ny_val_pred = make_prediction(X_val, regr, y_scaler)\nval_rmse = np.sqrt(mean_squared_error(y_val_pred, y_val_df.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Elastic Net","metadata":{}},{"cell_type":"code","source":"#Grid Search\nfrom sklearn.model_selection import GridSearchCV\nclf = ElasticNet()\ngrid_values = {'alpha': [0.001, 0.01, 0.1, 0., 1.0, 10.0, 100.],'l1_ratio':[0.001,0.001,0.01,0.1,0.3,0.7,1]}\ngrid_clf_acc = GridSearchCV(clf, param_grid = grid_values,scoring = 'recall')\ngrid_clf_acc.fit(X_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print the optimal hyper-parameters\nprint(grid_clf_acc.best_estimator_)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import ElasticNet\nela = ElasticNet(alpha=0, l1_ratio=0 ,fit_intercept = True, normalize = True, copy_X = True, random_state=0).fit(X_train,y_train)\nela.predict(X_val)\nprint(ela.score(X_train,y_train))\nprint(\"Weights:\")\nprint(ela.coef_)\nprint(ela.intercept_)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coeff_df_ela = pd.DataFrame(np.concatenate((ela.coef_,ela.intercept_)), feature_names+['w[0]'] ,columns=['Weights'])\ncoeff_df_ela","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred_ela = make_prediction(X_train, ela, y_scaler)\ntrain_rmse_ela = np.sqrt(mean_squared_error(y_train_pred_ela, y_train_df.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse_ela)))\n\ny_val_pred_ela = make_prediction(X_val, ela, y_scaler)\nval_rmse_ela = np.sqrt(mean_squared_error(y_val_pred_ela, y_val_df.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse_ela)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Scatter Matrix","metadata":{}},{"cell_type":"code","source":"from pandas.plotting import scatter_matrix\nto_plot=pd.concat([X_val_df,y_val_df],axis=1)\nscatter_matrix(to_plot, figsize=(30,18))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the scatter matrix, only distance and air time seem to be correlated (=> logical that these features are correlated)\n\n","metadata":{}},{"cell_type":"markdown","source":"## Trees","metadata":{}},{"cell_type":"code","source":"#Grid Search\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn import tree\nX_train_copy=X_train.copy()\ny_train_copy=y_train.copy()\nclf = tree.DecisionTreeRegressor()\ngrid_values = {'max_depth': [15,30,50,64],'min_samples_leaf':[1,10,50,100]}\ngrid_clf_acc = GridSearchCV(clf, param_grid = grid_values,scoring = 'recall')\ngrid_clf_acc.fit(X_train_copy, y_train_copy)\n# Print the optimal hyper-parameters\nprint(grid_clf_acc.best_estimator_)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import tree\nX_train_copy=X_train.copy()\ny_train_copy=y_train.copy()\nX_val_copy=X_val.copy()\nclf = tree.DecisionTreeRegressor(max_depth=15)\nclf = clf.fit(X_train_copy,y_train_copy)\nclf.predict(X_val_copy)\nprint(clf.score(X_train_copy,y_train_copy))\nprint(clf.tree_.max_depth) #64 with default parameters","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp_df_clf = pd.DataFrame(clf.feature_importances_, feature_names ,columns=['Importance'])\nimp_df_clf","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most important features: distance (0.16), air time (0.089), scheduled arrival (0.39) and departure time (0.29). Other features are <0.03. Remembering the correlation (scatter matrix) between the distance and the air time, we may only keep the distance (and not the air time): that is confirmed by the importance when the max depth is 15 (distance: 0.11 and air time: 0.04).","metadata":{}},{"cell_type":"code","source":"y_train_pred_clf = make_prediction(X_train, clf, y_scaler)\ntrain_rmse_clf = np.sqrt(mean_squared_error(y_train_pred_clf, y_train_df.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse_clf)))\n\ny_val_pred_clf = make_prediction(X_val, clf, y_scaler)\nval_rmse_clf = np.sqrt(mean_squared_error(y_val_pred_clf, y_val_df.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse_clf)))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With same parameters as baseline\n\nValidation RMSE=24.33 a bit better than before where = 38.507.\n\nValidation RMSE=23.57 for a tree of max depth=15\n\nAdvantage of tree: transparency","metadata":{}},{"cell_type":"markdown","source":"## Random Forest ","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import ExtraTreesRegressor\nX_train_copy2=X_train_num.copy()\ny_train_copy2=y_train_num.copy()\nX_val_copy2=X_val_num.copy()\nextra = ExtraTreesRegressor(n_estimators=5) #nb of trees in the forest. Default=10: mais out of memory. Essais avec 5, 7: ok\nextra = extra.fit(X_train_copy2,y_train_copy2)\nextra.predict(X_val_copy2)\nprint(extra.score(X_train_copy2,y_train_copy2))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra.estimators_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"imp_df_extra = pd.DataFrame(extra.feature_importances_, feature_names ,columns=['Importance'])\nimp_df_extra","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train_pred_extra = make_prediction(X_train_copy2, extra, y_scaler)\ntrain_rmse_extra = np.sqrt(mean_squared_error(y_train_pred_extra, y_train_df_num.values))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse_extra)))\n\ny_val_pred_extra = make_prediction(X_val_copy2, extra, y_scaler)\nval_rmse_extra = np.sqrt(mean_squared_error(y_val_pred_extra, y_val_df_num.values))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse_extra)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"a bit better Validation RMSE (with same parameters as baseline)\n\n(with 5 trees in the forest): 19.87\n\n(with 7 trees in the forest): 19.31\n\n(with 5 or 10 trees in the forest + max depth=15): 23.27. A lot quicker!","metadata":{}},{"cell_type":"markdown","source":"# Make prediction","metadata":{}},{"cell_type":"code","source":"# Load the test data\nX_test_df = pd.read_csv(\"/kaggle/input/corrected-dataset-airports/X_test_df.csv\", dtype=X_dtype)\nX_test_df, _ = encoding(X_test_df)\n#X_test_df.AIRLINE = trained_le.transform(X_test_df['AIRLINE'])\nX_test_df['ORIGIN_AIRPORT'] = trained_le.transform(X_test_df['ORIGIN_AIRPORT'])\nX_test_df['DESTINATION_AIRPORT'] = trained_le.transform(X_test_df['DESTINATION_AIRPORT'])\n# Preprocessing data\n\nX_test = preprocess_data(X_test_df, feature_names, imputer, X_scaler)\n\n# Make predictions\ny_test_pred = make_prediction(X_test, extra, y_scaler)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a dataframe containing the predictions\nsubmission_df = pd.DataFrame(data={'ID': X_test_df.ID.values,\n                                   'ARRIVAL_DELAY': y_test_pred.squeeze()})\n\n# Save the predictions into a csv file\n# Notice that this file should be saved under the directory `/kaggle/working` \n# so that you can download it later\nsubmission_df.to_csv(\"/kaggle/working/submission_airline_with_dep_delay_air_time.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}