{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom statistics import mean\n\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation\nThis first part will focus on creating DataFrames and cleaning the data","metadata":{}},{"cell_type":"code","source":"X_dtype = {\n    'ID'                   : int,\n    'YEAR'                 : int,  \n    'MONTH'                : int,  \n    'DAY'                  : int,  \n    'DAY_OF_WEEK'          : int,  \n    'AIRLINE'              : str, \n    'FLIGHT_NUMBER'        : str,  \n    'TAIL_NUMBER'          : str, \n    'ORIGIN_AIRPORT'       : str, \n    'DESTINATION_AIRPORT'  : str, \n    'SCHEDULED_DEPARTURE'  : str,  \n    'DEPARTURE_TIME'       : str, \n    'DEPARTURE_DELAY'      : float,\n    'TAXI_OUT'             : str, \n    'WHEELS_OFF'           : str,\n    'SCHEDULED_TIME'       : float,\n    'AIR_TIME'             : float,\n    'DISTANCE'             : int,\n    'SCHEDULED_ARRIVAL'    : str,\n    'DIVERTED'             : int,  \n    'CANCELLED'            : int,  \n    'CANCELLATION_REASON'  : str\n}\n\ny_dtype = {\n    'ID'                   : int,\n    \"ARRIVAL_DELAY\"        : float\n}\n\nX_train_df = pd.read_csv(\"/kaggle/input/eurecom-aml-2021-challenge-1/data/train_features.csv\", dtype=X_dtype)\ny_train_df = pd.read_csv(\"/kaggle/input/eurecom-aml-2021-challenge-1/data/train_targets.csv\", dtype=y_dtype)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The flight with delayed are the ones with \"Arrival_delay\" being > 0, we are going to transform it to had a column just to have a boolean delayed/not delayed.","metadata":{}},{"cell_type":"code","source":"y_train_df['DELAYED'] = (y_train_df.ARRIVAL_DELAY > 0).astype(str)\n#y_train_df['DATE'] = pd.to_datetime(df[['YEAR', 'MONTH', 'DAY']])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Analysing Data\n","metadata":{}},{"cell_type":"raw","source":"Here the goal is to observe the different features to analyse what our prediction model needs.","metadata":{}},{"cell_type":"code","source":"merged_df = pd.merge(X_train_df, y_train_df, on='ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"months_delay = []\nday_delay = []\nfor i in range(12):\n    Total_nb = len(merged_df.loc[(merged_df['MONTH'] == i+1)].index)\n    Delayed_nb = len(merged_df.loc[(merged_df['MONTH'] == i+1) & (merged_df['DELAYED'] == 'True')].index)\n    months_delay.append(Delayed_nb/Total_nb * 100)\n    \nfor i in range(7):\n    Total_nb = len(merged_df.loc[(merged_df['DAY_OF_WEEK'] == i+1)].index)\n    Delayed_nb = len(merged_df.loc[(merged_df['DAY_OF_WEEK'] == i+1) & (merged_df['DELAYED'] == 'True')].index)\n    day_delay.append(Delayed_nb/Total_nb * 100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Plotting chart\n\nmonths = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\ndays = ['Mon','Tue','Wed','Thu','Fri','Sat','Sun']\n\nmonths_delay=np.array(months_delay)\nday_delay = np.array(day_delay)\n\nplt.plot(months,months_delay)\nplt.xlabel(\"Months\")\nplt.ylabel(\"Percentage of delayed flights %\")\nplt.show()\nplt.plot(days,day_delay)\nplt.xlabel(\"Days\")\nplt.ylabel(\"Percentage of delayed flights %\")\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Percentage of delayed flights by origin airport\n\ntotal_origin = merged_df['ORIGIN_AIRPORT'].value_counts().to_frame()\ndelayed_origin = merged_df.loc[merged_df['DELAYED'] == 'True']['ORIGIN_AIRPORT'].value_counts().to_frame()\norigin_df = total_origin.merge(delayed_origin, left_index=True, right_index=True)\norigin_df['Percentage'] = origin_df['ORIGIN_AIRPORT_y']/origin_df['ORIGIN_AIRPORT_x']*100\nairport = np.array(origin_df.index)\npercentage_origin = np.array(origin_df['Percentage'])\n\n#Percentage of delayed flights by destination airport\n\ntotal_destination = merged_df['DESTINATION_AIRPORT'].value_counts().to_frame()\ndelayed_destination = merged_df.loc[merged_df['DELAYED'] == 'True']['DESTINATION_AIRPORT'].value_counts().to_frame()\ndestination_df = total_destination.merge(delayed_destination, left_index=True, right_index=True)\ndestination_df['Percentage'] = destination_df['DESTINATION_AIRPORT_y']/destination_df['DESTINATION_AIRPORT_x']*100\nairport = np.array(destination_df.index)\npercentage_destination = np.array(destination_df['Percentage'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Percentage of delayed flights by destination airport and origin airport\n\nplt.plot(airport[:20],percentage_origin[:20])\nplt.plot(airport[:20],percentage_destination[:20])\nplt.legend(['origin','destination'])\nplt.xlabel('Airports')\nplt.ylabel('Percentage of delayed flights')\nplt.autoscale(enable=True, axis='both', tight=None)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Percentage of delayed flights by schedule time arriving\n\nhours = []\npercentage_hours = []\nfor i in range (24):\n    hours.append(str(i)+\"h\")\n    if i<9:    \n        Total_hours = len(merged_df.loc[(merged_df['SCHEDULED_ARRIVAL']>= \"0\"+str(i)+\"00\") & (merged_df['SCHEDULED_ARRIVAL'] < \"0\"+str(i+1)+\"00\")].index)\n        Delayed_hours = len(merged_df.loc[((merged_df['SCHEDULED_ARRIVAL']) >= \"0\"+str(i)+\"00\")&((merged_df['SCHEDULED_ARRIVAL']) < \"0\"+str(i+1)+\"00\") & (merged_df['DELAYED'] == 'True')].index)\n    elif i==9:\n        Total_hours = len(merged_df.loc[(merged_df['SCHEDULED_ARRIVAL']>= \"0\"+str(i)+\"00\") & (merged_df['SCHEDULED_ARRIVAL'] < str(i+1)+\"00\")].index)\n        Delayed_hours = len(merged_df.loc[((merged_df['SCHEDULED_ARRIVAL']) >= \"0\"+str(i)+\"00\")&((merged_df['SCHEDULED_ARRIVAL']) < str(i+1)+\"00\") & (merged_df['DELAYED'] == 'True')].index)\n    else:\n        Total_hours = len(merged_df.loc[(merged_df['SCHEDULED_ARRIVAL']>= str(i)+\"00\") & (merged_df['SCHEDULED_ARRIVAL'] < str(i+1)+\"00\")].index)\n        Delayed_hours = len(merged_df.loc[((merged_df['SCHEDULED_ARRIVAL']) >= str(i)+\"00\")&((merged_df['SCHEDULED_ARRIVAL']) < str(i+1)+\"00\") & (merged_df['DELAYED'] == 'True')].index)\n    if Total_hours !=0:\n        percentage_hours.append(Delayed_hours/Total_hours * 100)\n    else:\n        percentage_hours.append(0)\n        print(\"Attention il n'y a pas de vols à \" + str(i) +\"h !\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Percentage of delayed flights by schedule time departure\n\npercentage_hours_departure = []\nfor i in range (24):\n    if i<9:    \n        Total_hours = len(merged_df.loc[(merged_df['SCHEDULED_DEPARTURE']>= \"0\"+str(i)+\"00\") & (merged_df['SCHEDULED_DEPARTURE'] < \"0\"+str(i+1)+\"00\")].index)\n        Delayed_hours = len(merged_df.loc[((merged_df['SCHEDULED_DEPARTURE']) >= \"0\"+str(i)+\"00\")&((merged_df['SCHEDULED_DEPARTURE']) < \"0\"+str(i+1)+\"00\") & (merged_df['DELAYED'] == 'True')].index)\n    elif i==9:\n        Total_hours = len(merged_df.loc[(merged_df['SCHEDULED_DEPARTURE']>= \"0\"+str(i)+\"00\") & (merged_df['SCHEDULED_DEPARTURE'] < str(i+1)+\"00\")].index)\n        Delayed_hours = len(merged_df.loc[((merged_df['SCHEDULED_DEPARTURE']) >= \"0\"+str(i)+\"00\")&((merged_df['SCHEDULED_DEPARTURE']) < str(i+1)+\"00\") & (merged_df['DELAYED'] == 'True')].index)\n    else:\n        Total_hours = len(merged_df.loc[(merged_df['SCHEDULED_DEPARTURE']>= str(i)+\"00\") & (merged_df['SCHEDULED_DEPARTURE'] < str(i+1)+\"00\")].index)\n        Delayed_hours = len(merged_df.loc[((merged_df['SCHEDULED_DEPARTURE']) >= str(i)+\"00\")&((merged_df['SCHEDULED_DEPARTURE']) < str(i+1)+\"00\") & (merged_df['DELAYED'] == 'True')].index)\n    if Total_hours !=0:\n        percentage_hours_departure.append(Delayed_hours/Total_hours * 100)\n    else:\n        percentage_hours_departure.append(0)\n        print(\"Attention il n'y a pas de vols à \" + str(i) +\"h !\")\n\nplt.plot(hours,percentage_hours)\nplt.plot(hours,percentage_hours_departure)\nplt.legend(['arrival','departure'])\nplt.xlabel('Hours')\nplt.ylabel('Percentage of delayed flights')\nplt.autoscale(enable=True, axis='both', tight=None)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#This shows that the DISTANCE shouldn't be used as it is absolutely not discriminative so it will not improve the results\n\naverage_distance = merged_df['DISTANCE'].mean()\naverage_distance_delayed = merged_df.loc[merged_df['DELAYED'] == 'True']['DISTANCE'].mean()\nprint('average_distance' ,average_distance)\nprint('average_distance_delayed' , average_distance_delayed)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Percentage of delayed flights by airline company\n\ntotal_airline = merged_df['AIRLINE'].value_counts().to_frame()\ndelayed_airline = merged_df.loc[merged_df['DELAYED'] == 'True']['AIRLINE'].value_counts().to_frame()\nairline_df = total_airline.merge(delayed_airline, left_index=True, right_index=True)\nairline_df['Percentage'] = airline_df['AIRLINE_y']/airline_df['AIRLINE_x']*100\npercentage_airline = np.array(airline_df['Percentage'])\nairline = np.array(airline_df.index)\n\nplt.plot(airline,percentage_airline)\nplt.xlabel('Airline')\nplt.ylabel('Percentage of delayed flights')\nplt.autoscale(enable=True, axis='both', tight=None)\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We decided to add the information about airports","metadata":{}},{"cell_type":"code","source":"airports = pd.read_csv('/kaggle/input/eurecom-aml-2021-challenge-1/data/airports.csv',\n                       usecols=['IATA_CODE', 'LATITUDE', 'LONGITUDE'])\n\nX_train_df = pd.merge(X_train_df, airports.rename({'LATITUDE':'LATITUDE_ORIGIN', 'LONGITUDE':'LONGITUDE_ORIGIN'},axis='columns', errors = 'raise'),how='left',left_on='ORIGIN_AIRPORT', right_on = 'IATA_CODE')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Merging Y_train and X_train \n\nmerged_df = pd.merge(X_train_df, y_train_df, on='ID')\nmerged_df.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Engineering\nCan you create and add new data features? Perhaps there are attributes that can be decomposed into multiple new values (like categories, dates or strings) or attributes that can be aggregated to signify an event (like a count, binary flag or statistical summary).\n\nWe decided to create a coefficient/ score for ech airport based on the number of delayed flight we have. Then we combine the origin and destination airport to create a delayed score for each flight.\n","metadata":{}},{"cell_type":"code","source":"def correlation_heatmap(train):\n    correlations = train.corr()\n\n    fig, ax = plt.subplots(figsize=(10,10))\n    sns.heatmap(correlations, vmax=1.0, center=0, fmt='.2f',\n                square=True, linewidths=.5, annot=True, cbar_kws={\"shrink\": .70})\n    plt.show();\n\n\ncorrelation_heatmap(merged_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#CREATION OF FEATURE\n\n## get the percentage of lateness depending on airports\n\ntotal_origin = merged_df['ORIGIN_AIRPORT'].value_counts().to_frame()\ndelayed_origin = merged_df.loc[merged_df['DELAYED'] == 'True']['ORIGIN_AIRPORT'].value_counts().to_frame()\norigin_df = total_origin.merge(delayed_origin, left_index=True, right_index=True)\norigin_df['Percentage'] = origin_df['ORIGIN_AIRPORT_y']/origin_df['ORIGIN_AIRPORT_x']*100\norigin_df = origin_df.drop(columns=['ORIGIN_AIRPORT_x','ORIGIN_AIRPORT_y'])\n\ntotal_destination = merged_df['DESTINATION_AIRPORT'].value_counts().to_frame()\ndelayed_destination = merged_df.loc[merged_df['DELAYED'] == 'True']['DESTINATION_AIRPORT'].value_counts().to_frame()\ndestination_df = total_destination.merge(delayed_destination, left_index=True, right_index=True)\ndestination_df['Percentage'] = destination_df['DESTINATION_AIRPORT_y']/destination_df['DESTINATION_AIRPORT_x']*100\ndestination_df = destination_df.drop(columns=['DESTINATION_AIRPORT_x','DESTINATION_AIRPORT_y'])\n\n#merging\n\norigin_df.reset_index(inplace=True)\norigin_df = origin_df.rename(columns = {'index':'ORIGIN_AIRPORT'})\ndestination_df.reset_index(inplace=True)\ndestination_df = destination_df.rename(columns = {'index':'DESTINATION_AIRPORT'})\nmerged_df = pd.merge(merged_df,origin_df,on=\"ORIGIN_AIRPORT\")\nmerged_df = pd.merge(merged_df,destination_df,on=\"DESTINATION_AIRPORT\")\n\n\nmerged_df['DELAY_SCORE'] = (merged_df['Percentage_x']+merged_df['Percentage_y'])/2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DelayedScore= merged_df[['ID','DELAY_SCORE']]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre-Processing\n\nThe preprocessing is probably the most important stage here, we need to prepare and adapt the data so that it improves our results and so that the data can be processed through the models\n","metadata":{}},{"cell_type":"code","source":"#We want to use the airport names as input so we will encode them using label encoder\n\n\nfrom sklearn.preprocessing import LabelEncoder\nlabelencoder = LabelEncoder()\n\nX,y = X_train_df,y_train_df #that is used to keep the values of the df to avoid re executing all cells\n\nle = LabelEncoder()\nX_train_df['ORIGIN_AIRPORT'] = le.fit_transform(X_train_df.ORIGIN_AIRPORT.values)\nX_train_df['DESTINATION_AIRPORT'] = le.fit_transform(X_train_df.DESTINATION_AIRPORT.values)\nX_train_df['AIRLINE'] = le.fit_transform(X_train_df.AIRLINE.values)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nX_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X1, X2 = X_train, X_test\ny1, y2 = y_train, y_test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Taking out features that are useless according to us\n\nX_train.TAXI_OUT = X_train.TAXI_OUT.astype(int)\nfeatures_name =['MONTH', 'DAY','AIRLINE','TAXI_OUT','DEPARTURE_DELAY','LATITUDE_ORIGIN','LONGITUDE_ORIGIN']\n\ny_train = y1[['ARRIVAL_DELAY']]\nX_train = X1[features_name]\n\nX_test = X2[features_name]\ny_test = y2[['ARRIVAL_DELAY']]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ultimately we decided to choose those feature, only 5 so that the process isn't that heavy and the models can train, we took those because we think they are the most pertinent according to our study. We didn't put the feature we created because it didn't improve our results.","metadata":{}},{"cell_type":"code","source":"\n# Filling missing values by the mean along each column.\n# These statistics should be estimated by using the training set.\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean')\nimputer.fit(X_train)\n\nX_train = imputer.transform(X_train)\nX_test = imputer.transform(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_scaler = StandardScaler().fit(X_train)\nX_train = X_scaler.transform(X_train)\nX_test = X_scaler.transform(X_test)\n\n#y_scaler = StandardScaler().fit(y_train)\n#y_train = y_scaler.transform(y_train)\n#y_test = y_scaler.transform(y_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def preprocess_data(df, feature_names, imputer, scaler):\n    \n    # Select features\n    \n    le = LabelEncoder()\n    df['ORIGIN_AIRPORT'] = le.fit_transform(df.ORIGIN_AIRPORT.values)\n    df['DESTINATION_AIRPORT'] = le.fit_transform(df.DESTINATION_AIRPORT.values)\n    df['AIRLINE'] = le.fit_transform(df.AIRLINE.values)\n    X_df = df[feature_names]\n    \n    # Pre-process datetime features\n    X_df.TAXI_OUT = X_df.TAXI_OUT.astype(int)\n    \n    X_df.DEPARTURE_TIME = X_df.DEPARTURE_TIME.apply(parse_hhmm)\n    X_df.SCHEDULED_ARRIVAL = X_df.SCHEDULED_ARRIVAL.apply(parse_hhmm)\n\n    X_df.SCHEDULED_ARRIVAL = X_df.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\n    X_df.DEPARTURE_TIME = X_df.DEPARTURE_TIME.apply(minutes_since_midnight)\n\n    \n    # Impute missing values\n    X = imputer.transform(X_df)\n    \n    # Normalize features\n    X = scaler.transform(X)\n\n    return X","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Feature Selection\n\nWe tried using a skikit learn tool to choose the best features to use , the LASSO method and SelectFromModel,  but Kagglecan't handle the processing of those so we didn't manage to use them after all.\n","metadata":{}},{"cell_type":"code","source":"#Lasso Method\n\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.linear_model import LogisticRegression\n\nembeded_lr_selector = SelectFromModel(LogisticRegression(C=1, penalty='l1', solver='liblinear'), max_features=5)\nembeded_lr_selector.fit(X_train, np.ravel(y_train))\n\nembeded_lr_support = embeded_lr_selector.get_support()\nembeded_lr_feature = X_train.loc[:,embeded_lr_support].columns.tolist()\nprint(str(len(embeded_lr_feature)), 'selected features')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model Selection \nWe compared numerous models to look for the ONE but none of them was really a step ahead, the real aspect was the process time","metadata":{}},{"cell_type":"code","source":"from sklearn import neighbors, datasets, preprocessing\nfrom sklearn.ensemble import RandomForestRegressor as RFR\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.svm import LinearSVR\nfrom sklearn.linear_model import SGDRegressor","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##KNN\nknn = neighbors.KNeighborsRegressor(n_neighbors=5)\nknn.fit(X_train, np.ravel(y_train))\ny_pred_KNN = knn.predict(X_test)\n\n#KNN is easily interpretable so for us it is the best to start\n\n\n##Gradient Boosting Regressor\n\nreg = GradientBoostingRegressor(random_state=0)\nreg.fit(X_train, np.ravel(y_train))\ny_pred_GBM = reg.predict(X_test)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfr = RFR(n_estimators = 20, verbose=10)\nrfr.fit(X_train,np.ravel(y_train))\ny_pred_RF = rfr.predict(X_test)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Random forest isn't working with more than 20 trees for this number of features. The kaggle ram keeps crashing. We manage to process more trees but the results isn't really better so we decided to stick at 20 trees.","metadata":{}},{"cell_type":"code","source":"ridge =  Ridge(alpha=10,solver ='svd').fit(X_train,y_train)\ny_pred_ridge = ridge.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SVR =  LinearSVR().fit(X_train,np.ravel(y_train))\ny_pred_svr = SVR.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SGD =  SGDRegressor(learning_rate='optimal',alpha= 0.001, loss ='squared_loss', penalty= 'l1').fit(X_train,np.ravel(y_train))\ny_pred_sgd = SGD.predict(X_test)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_prediction(X, model):\n    y_pred = model.predict(X)\n    return y_pred","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Parameter Optimization\n\nThis part of our work consisted in optimazing several models to observe if there was a significant change in the results. The execution is very heavy therefore we took it step by step and for certain case only by optimizing one parameter (the most pertinent in our eyes).\nOne algorithm we didn't optimize was GBR because Kaggle couldn't handle it, it kept crashing because of the RAM. We decided it wasn't a real issue because GBR being to heavy we didn't want it as our main model.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import KFold\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# KNN optimization\n\nparam_grid = dict(n_neighbors = np.array([5,6,7,8,9,10,15]))\n\nkfold = KFold(n_splits=10)\n\ngrid = GridSearchCV(estimator=neighbors.KNeighborsRegressor(), param_grid=param_grid, scoring='neg_mean_squared_error', cv=kfold).fit(X_train, y_train)\n\ny_pred_knn_opt = grid.predict(X_test)\nprint(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"This optimization showed us that the parameter neighbors should be equal to 5 to optimize the results. But the improvement wasn't that great and didn't put KNN on our list of good model for those prediction.","metadata":{}},{"cell_type":"code","source":"#Ridge Optimization\nparam_grid = dict(alpha = np.array([1,10]),solver = ['auto','svd', 'cholesky', 'lsqr', 'sparse_cg', 'sag', 'saga'])\n\nkfold = KFold(n_splits=3)\n\ngrid_ridge = GridSearchCV(estimator=Ridge(tol=0.0001), param_grid=param_grid, scoring='neg_mean_squared_error', cv=kfold,verbose=10).fit(X_train, y_train)\n\ny_pred_ridge_opt = grid_ridge.predict(X_test)\nprint(\"Best: %f using %s\" % (grid_ridge.best_score_, grid_ridge.best_params_))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The optimization of Ridge was too heavy to do multiple parameter so we decided to start with alpha and then do another round for the solver. This gives us alpha = 10 and the solver being svd.","metadata":{}},{"cell_type":"code","source":"## Random Forest optimization \n\n# Number of trees in random forest\nparam_grid = dict(n_estimators  = np.array([10,30,50]))\n# Number of features to consider at every split\nmax_features = ['auto', 'sqrt']\n# Maximum number of levels in tree\nmax_depth = [int(x) for x in np.linspace(10, 110, num = 11)]\n\n# Create the random grid\nrandom_grid = {'n_estimators': n_estimators,\n               }\n\nkfold = KFold(n_splits=5)\ngrid = GridSearchCV(estimator=RFR(), param_grid=param_grid, scoring='neg_mean_squared_error', cv=kfold).fit(X_train, np.ravel(y_train))\n\ny_pred_rfr_opt = grid.predict(X_test)\nprint(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We tried to optimize RFR but this process it too heavy for kaggle so we let down this optimization","metadata":{}},{"cell_type":"code","source":"param_grid = {'C': [0.1, 1, 10]}\n\nkfold = KFold(n_splits=3)\n\ngrid = GridSearchCV(estimator=LinearSVR(), param_grid=param_grid, scoring='neg_mean_squared_error', cv=kfold, verbose=10).fit(X_train, np.ravel(y_train))\n\ny_pred_svr_opt = grid.predict(X_test)\n\nprint(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We found some parameter that optimize Linear SVR, C= 10 ","metadata":{}},{"cell_type":"code","source":"param_grid = {'loss': ['squared_loss', 'huber', 'epsilon_insensitive'],'penalty': ['l1', 'l2', 'elasticnet'], 'alpha': [0.001, 0.0001, 0.00001]}\n\nkfold = KFold(n_splits=3)\ngrid = GridSearchCV(estimator=SGDRegressor(learning_rate='optimal'), param_grid=param_grid, scoring='neg_mean_squared_error', cv=kfold, verbose=10).fit(X_train, np.ravel(y_train))\n\ny_pred_svr_opt = grid.predict(X_test)\nprint(\"Best: %f using %s\" % (grid.best_score_, grid.best_params_))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The optimization of SGD was the one we did the most deeply because SGD being pretty light it allowed us to explore a lot more parameter without crashing kaggle. Our results were : {'alpha': 0.001, 'loss': 'squared_loss', 'penalty': 'l1'}\n","metadata":{}},{"cell_type":"markdown","source":"## Model Performance","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\nrmse = mean_squared_error(y_pred_KNN, y_test, squared = False )\n\nprint(\"Validation RMSE for KNN: {:.5f}\".format(float(rmse)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\nrmse = mean_squared_error(y_test, y_pred_RF, squared = False )\n\nprint(\"Validation RMSE for RF: {:.5f}\".format(float(rmse)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmse = mean_squared_error(y_test, y_pred_GBM, squared = False )\n\nprint(\"Validation RMSE for GBM: {:.5f}\".format(float(rmse)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nrmse = mean_squared_error(y_test, y_pred_ridge, squared = False )\n\nprint(\"Validation RMSE for Ridge Regression: {:.5f}\".format(float(rmse)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nrmse = mean_squared_error(y_test, y_pred_svr, squared = False )\n\nprint(\"Validation RMSE for SVR: {:.5f}\".format(float(rmse)))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rmse = mean_squared_error(y_test, y_pred_sgd, squared = False)\n\nprint(\"Validation RMSE for SGD optimized: {:.5f}\".format(float(rmse)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Results :\n\nModel Tested :\n    Random Forest /\n    KNN /\n    GBM / \n    Logistic Regression /\n    Ridge /\n    SVR /\n    SGD\n    \n The best model so far is SGD Regressor. This model gives the same results as RFR and GBM but is way less heavy to process which is for us a very important aspect. We think that to improve our result we now need to work more on the features because we haven't manage to improve our results even though we digged into optimization for each algorithm.\n        ","metadata":{}},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"# Load the test data\nX_test_df = pd.read_csv(\"/kaggle/input/eurecom-aml-2021-challenge-1/data/test_features.csv\", dtype=X_dtype)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocessing data\nX_test = preprocess_data(X_test_df, features_name, imputer, X_scaler)\n\n# Make predictions\n\ny_test_pred = make_prediction(X_test, reg)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create a dataframe containing the predictions\nsubmission_df = pd.DataFrame(data={'ID': X_test_df.ID.values,\n                                   'ARRIVAL_DELAY': y_test_pred.squeeze()})\n\n# Save the predictions into a csv file\n# Notice that this file should be saved under the directory `/kaggle/working` \n# so that you can download it later\nsubmission_df.to_csv(\"/kaggle/working/submission5.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}