{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Challenge 1 - Airplane delay\n# Airplane delay: Analysis","metadata":{"papermill":{"duration":0.015482,"end_time":"2021-04-23T16:45:17.953066","exception":false,"start_time":"2021-04-23T16:45:17.937584","status":"completed"},"tags":[],"id":"decimal-parking"}},{"cell_type":"code","source":"import sys\nimport json\nimport warnings\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport matplotlib.dates as mdates\nimport seaborn as sns\nimport tensorflow as tf\nimport numpy as np\nimport tensorflow_probability as tfp\n\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.model_selection import train_test_split, GridSearchCV\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.linear_model import Ridge\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.pipeline import Pipeline\n#from hyperopt import GridSearch\nfrom matplotlib import rc\nfrom google.colab import files \n\nif not sys.warnoptions:\n    warnings.simplefilter(\"ignore\")\n\nrc('figure', **{'dpi': 200})","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:45:45.945096Z","iopub.status.busy":"2021-04-23T16:45:45.944203Z","iopub.status.idle":"2021-04-23T16:45:54.108227Z","shell.execute_reply":"2021-04-23T16:45:54.109148Z"},"papermill":{"duration":8.186618,"end_time":"2021-04-23T16:45:54.109358","exception":false,"start_time":"2021-04-23T16:45:45.92274","status":"completed"},"tags":[],"id":"seasonal-louis"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"files.upload()\n","metadata":{"id":"Ii5Cz1Ur9Mqy","outputId":"6cb966c8-4a10-4ddc-a400-924df88c15da"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! mkdir ~/.kaggle \n! cp kaggle.json ~/.kaggle/\n! chmod 600 ~/.kaggle/kaggle.json\n! kaggle competitions download -c 'eurecom-aml-2021-challenge-1'\n! unzip '*.zip'","metadata":{"id":"YXNr42_G9SPH","outputId":"9785bc43-dea6-4c05-c6cb-2378bc87dd96"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls","metadata":{"id":"wcAvgQDm9zXh","outputId":"356cd0d1-e2d2-4e2a-c576-7b50ee529ab3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_dtype = {\n    'ID'                   : int,\n    'YEAR'                 : int,  \n    'MONTH'                : int,  \n    'DAY'                  : int,  \n    'DAY_OF_WEEK'          : int,  \n    'AIRLINE'              : str, \n    'FLIGHT_NUMBER'        : str,  \n    'TAIL_NUMBER'          : str, \n    'ORIGIN_AIRPORT'       : str, \n    'DESTINATION_AIRPORT'  : str, \n    'SCHEDULED_DEPARTURE'  : str,  \n    'DEPARTURE_TIME'       : str, \n    'DEPARTURE_DELAY'      : float,\n    'TAXI_OUT'             : str, \n    'WHEELS_OFF'           : str,\n    'SCHEDULED_TIME'       : float,\n    'AIR_TIME'             : float,\n    'DISTANCE'             : int,\n    'SCHEDULED_ARRIVAL'    : str,\n    'DIVERTED'             : int,  \n    'CANCELLED'            : int,  \n    'CANCELLATION_REASON'  : str\n}\n\ny_dtype = {\n    'ID'                   : int,\n    \"ARRIVAL_DELAY\"        : float\n}\n\nX_train_df = pd.read_csv(\"train_features.csv\", dtype=X_dtype)\ny_train_df = pd.read_csv(\"train_targets.csv\", dtype=y_dtype)\nX_test_df = pd.read_csv(\"test_features.csv\", dtype=X_dtype)","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:45:54.146718Z","iopub.status.busy":"2021-04-23T16:45:54.145795Z","iopub.status.idle":"2021-04-23T16:46:18.49506Z","shell.execute_reply":"2021-04-23T16:46:18.494354Z"},"papermill":{"duration":24.371462,"end_time":"2021-04-23T16:46:18.495206","exception":false,"start_time":"2021-04-23T16:45:54.123744","status":"completed"},"tags":[],"id":"level-camel"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_df.shape","metadata":{"id":"Xk6S7Zjq8n89","outputId":"cfb170ef-7282-4238-9fa2-0cf4bc857dca"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def minutes_since_midnight(dt):\n    return dt.hour * 60 + dt.minute\n\ndef parse_hhmm(x):\n    if x == \"2400\":\n      x = \"0000\"\n    try: return pd.datetime.strptime(x, '%H%M')#.time()\n    except: \n      print(\"found Invalid datetime\", x)\n      return pd.NaT\n\ndef parse_times(dataframe):\n\n  dataframe.TAXI_OUT = dataframe.TAXI_OUT.astype(int)\n  dataframe.SCHEDULED_DEPARTURE = dataframe.SCHEDULED_DEPARTURE.apply(parse_hhmm)\n  dataframe.SCHEDULED_ARRIVAL = dataframe.SCHEDULED_ARRIVAL.apply(parse_hhmm)\n  dataframe.DEPARTURE_TIME = dataframe.DEPARTURE_TIME.apply(parse_hhmm)\n\n  dataframe.SCHEDULED_DEPARTURE = dataframe.SCHEDULED_DEPARTURE.apply(minutes_since_midnight)\n  dataframe.SCHEDULED_ARRIVAL = dataframe.SCHEDULED_ARRIVAL.apply(minutes_since_midnight)\n  dataframe.DEPARTURE_TIME = dataframe.DEPARTURE_TIME.apply(minutes_since_midnight)\n\n  pd.set_option('display.max_rows', 22)\n  dataframe.dtypes\n\nparse_times(X_train_df)\nparse_times(X_test_df)","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:46:18.538513Z","iopub.status.busy":"2021-04-23T16:46:18.537812Z","iopub.status.idle":"2021-04-23T16:50:11.399524Z","shell.execute_reply":"2021-04-23T16:50:11.400059Z"},"papermill":{"duration":232.890636,"end_time":"2021-04-23T16:50:11.400245","exception":false,"start_time":"2021-04-23T16:46:18.509609","status":"completed"},"tags":[],"id":"brief-inventory","outputId":"9bb7a43b-07b2-434a-ee0d-645ba67f69d8"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"You may also use additional information about airports and airlines, which can be found in the files `/kaggle/input/eurecom-aml-2021-challenge-1/data/airports.csv` and `/kaggle/input/eurecom-aml-2021-challenge-1/data/airlines.csv`, respectively.","metadata":{"papermill":{"duration":0.014716,"end_time":"2021-04-23T16:50:11.429987","exception":false,"start_time":"2021-04-23T16:50:11.415271","status":"completed"},"tags":[],"id":"naval-martin"}},{"cell_type":"code","source":"airports = pd.read_csv('airports.csv',\n                       usecols=['IATA_CODE', 'AIRPORT', 'LATITUDE', 'LONGITUDE']).rename({'AIRPORT': 'NAME'}, axis='columns')\n","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:50:11.465168Z","iopub.status.busy":"2021-04-23T16:50:11.464494Z","iopub.status.idle":"2021-04-23T16:50:11.48388Z","shell.execute_reply":"2021-04-23T16:50:11.483197Z"},"papermill":{"duration":0.039223,"end_time":"2021-04-23T16:50:11.484023","exception":false,"start_time":"2021-04-23T16:50:11.4448","status":"completed"},"tags":[],"id":"apart-animation"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare new columns used for data exploration\n\ndef merge(X, y):\n  df = pd.merge(X, airports.rename({'LATITUDE':'LATITUDE_ORIGIN', 'LONGITUDE':'LONGITUDE_ORIGIN'},axis='columns', errors = 'raise'),how='left',left_on='ORIGIN_AIRPORT', right_on = 'IATA_CODE')\n  df = pd.merge(df, airports.rename({'LATITUDE':'LATITUDE_DESTINATION', 'LONGITUDE':'LONGITUDE_DESTINATION'}, axis='columns',errors = 'raise'),how='left',left_on='DESTINATION_AIRPORT', right_on = 'IATA_CODE')\n  if y is not None:\n    df = pd.merge(df, y, on='ID')\n  else:\n    df['ARRIVAL_DELAY'] = 0\n  return df\n\ndf = merge(X_train_df, y_train_df)\ndf_test = merge(X_test_df, None)","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:50:12.057409Z","iopub.status.busy":"2021-04-23T16:50:12.056695Z","iopub.status.idle":"2021-04-23T16:50:53.080989Z","shell.execute_reply":"2021-04-23T16:50:53.08034Z"},"papermill":{"duration":41.581484,"end_time":"2021-04-23T16:50:53.081145","exception":false,"start_time":"2021-04-23T16:50:11.499661","status":"completed"},"tags":[],"id":"present-cemetery"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.shape","metadata":{"id":"REuhKaLJ8xFm","outputId":"a4146b7e-a22d-49ab-e96e-2f4e46d98a6c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_drop = ['ID', 'YEAR', 'TAIL_NUMBER', 'FLIGHT_NUMBER', 'WHEELS_OFF', 'DIVERTED',\n                'CANCELLED', 'CANCELLATION_REASON',\n                'IATA_CODE_x', 'NAME_x', 'IATA_CODE_y', 'NAME_y',\n                'ORIGIN_AIRPORT', 'DESTINATION_AIRPORT']\n\ndef drops(dataframe, cols_to_drop):\n  return dataframe.drop(cols_to_drop, axis = 1)\n\ndf = drops(df, cols_to_drop)","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:50:53.118498Z","iopub.status.busy":"2021-04-23T16:50:53.1178Z","iopub.status.idle":"2021-04-23T16:50:57.342505Z","shell.execute_reply":"2021-04-23T16:50:57.34178Z"},"papermill":{"duration":4.246254,"end_time":"2021-04-23T16:50:57.342651","exception":false,"start_time":"2021-04-23T16:50:53.096397","status":"completed"},"tags":[],"id":"magnetic-kingdom"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_drop.remove('ID')\ndf_test = drops(df_test, cols_to_drop)","metadata":{"id":"8TtgKEtmq_TB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\npd.set_option('display.max_rows', 50)\ndf.head(6)","metadata":{"id":"8jMl62_Dq0B4","outputId":"adf6eec8-43b0-4805-d3e1-d36a88786687"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical = ['AIRLINE', 'DAY_OF_WEEK', 'MONTH']\ndef make_categorical(features, dataframe):\n  df_cat = dataframe[features].astype('category')\n  dataframe = pd.concat([dataframe,pd.get_dummies(df_cat)],axis=1)\n  return dataframe.drop(categorical, axis = 1)\n\ndf = make_categorical(categorical, df)\ndf_test = make_categorical(categorical, df_test)\npd.set_option('display.max_rows', 100)\ndf_test.dtypes\n","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:50:57.505267Z","iopub.status.busy":"2021-04-23T16:50:57.50429Z","iopub.status.idle":"2021-04-23T16:50:58.950713Z","shell.execute_reply":"2021-04-23T16:50:58.949611Z"},"papermill":{"duration":1.591928,"end_time":"2021-04-23T16:50:58.950881","exception":false,"start_time":"2021-04-23T16:50:57.358953","status":"completed"},"tags":[],"id":"warming-martial","outputId":"ebfc1766-ed65-4ee3-9a48-9ffd4c586c2a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf_test.dtypes","metadata":{"id":"23IvfKZJlTm_","outputId":"c91ae5ea-0605-4e7b-e8b9-e7dbd3a77798"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Split into training and validation data \ntarget_name = 'ARRIVAL_DELAY'\ny = df[target_name]\nX = df.drop(target_name,axis = 1) \nX_test = df_test.drop(target_name,axis = 1) ","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:50:59.157405Z","iopub.status.busy":"2021-04-23T16:50:59.053964Z","iopub.status.idle":"2021-04-23T16:51:01.847475Z","shell.execute_reply":"2021-04-23T16:51:01.84676Z"},"papermill":{"duration":2.880587,"end_time":"2021-04-23T16:51:01.847619","exception":false,"start_time":"2021-04-23T16:50:58.967032","status":"completed"},"tags":[],"id":"initial-rachel"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xtrain, Xval, ytrain, yval = train_test_split(\n    X, y, random_state=10, test_size=0.4)","metadata":{"id":"V4voV_XWliP8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import SGDRegressor","metadata":{"id":"n2eR-aFeAEX-"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"param_space0 = {'est__learning_rate': ['optimal', 'invscaling']}\nparam_space1 = {'est__loss': ['squared_loss', 'huber', 'epsilon_insensitive']}\nparam_space2 = {'est__penalty': ['l1', 'l2', 'elasticnet'], 'est__alpha': [0.001, 0.0001, 0.00001]}","metadata":{"id":"miAaNBOVEQ7J"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we limit the max_iter in the param_search to keep time down\nmodel = SGDRegressor(tol=0.001, max_iter=250)","metadata":{"id":"kQTqTHo0Gke1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe0 = Pipeline([('scl', StandardScaler()), \n                      ('imp', SimpleImputer()),\n                      ('est', model)])\n\n                      ","metadata":{"id":"7sNVpJjdFlbG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdcv0 = GridSearchCV(estimator=pipe0, param_grid=param_space0, scoring='neg_root_mean_squared_error', verbose=1, cv=3)","metadata":{"id":"b8cRy9p5G43Q"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdcv0.fit(Xtrain, ytrain)","metadata":{"id":"EiqbddEdJDAE","outputId":"de97a72b-15bd-4c24-ff47-3c80f7304fa8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best score: %0.3f\" % gdcv0.best_score_)\nprint(\"Best parameters set:\")\nbest_parameters = gdcv0.best_estimator_.get_params()\nfor param_name in sorted(param_space0.keys()):\n    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))","metadata":{"id":"iDwPoZLNIx0F","outputId":"e241ce8f-37f9-4ad0-da68-8077fc5a325b"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SGDRegressor(tol=0.001, learning_rate='optimal')","metadata":{"id":"5S1qlWjvKUV4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe1 = Pipeline([('scl', StandardScaler()), \n                      ('imp', SimpleImputer()),\n                      ('est', model)])","metadata":{"id":"YgYfsiAtPuxh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdcv1 = GridSearchCV(estimator=pipe1, param_grid=param_space1, scoring='neg_root_mean_squared_error', verbose=1, cv=3)","metadata":{"id":"XHoGHf46J076"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdcv1.fit(Xtrain, ytrain)","metadata":{"id":"fS78527NJ2lu","outputId":"5bc129a7-cb39-4500-b06c-7c289cde9aca"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best score: %0.3f\" % gdcv1.best_score_)\nprint(\"Best parameters set:\")\nbest_parameters = gdcv1.best_estimator_.get_params()\nfor param_name in sorted(param_space1.keys()):\n    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))","metadata":{"id":"ULY4w0IzJ6Ah","outputId":"b2b78fae-02c3-44a6-ab97-adb4e9b8e174"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = SGDRegressor(tol=0.001, learning_rate='optimal', loss='squared_loss')","metadata":{"id":"HETOZqYRKVvY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pipe2= Pipeline([('scl', StandardScaler()), \n                      ('imp', SimpleImputer()),\n                      ('est', model)])","metadata":{"id":"CrUsW-icRPsT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdcv2 = GridSearchCV(estimator=pipe2, param_grid=param_space2, scoring='neg_root_mean_squared_error', verbose=1, n_jobs=-1)","metadata":{"id":"G4m-vRpjJ9dX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gdcv2.fit(Xtrain, ytrain)","metadata":{"id":"PfcQEmMwJ_XM","outputId":"fed42f09-7c16-4f27-e788-fd46f78268cd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best score: %0.3f\" % gdcv2.best_score_)\nprint(\"Best parameters set:\")\nbest_parameters = gdcv2.best_estimator_.get_params()\nfor param_name in sorted(param_space2.keys()):\n    print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))","metadata":{"id":"K5WUZQWGKAk-","outputId":"48b73c94-c0f7-429a-aa32-7c68286d7f59"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_pipe = make_pipeline(StandardScaler(),\n                     SimpleImputer(),\n                     SGDRegressor(tol=0.0001, alpha=0.001, learning_rate='optimal', loss='squared_loss', penalty='l1'),\n                     )","metadata":{"id":"PvNiGH1L__HR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_pipe.fit(Xtrain, ytrain)","metadata":{"id":"N2XiHar1AIs2","outputId":"88b50694-1acf-40ce-c01a-7ae2dc263f68"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X","metadata":{"id":"pkR-_7soxsSM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_score = mean_squared_error(y, final_pipe.predict(X), squared=False)\nprint(f'Best score on validation set: {round(val_score, 3)}')","metadata":{"id":"jJxbZUnjAqrF","outputId":"c7f7855c-ce81-473a-fb1e-13f0f6f26cc6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_train_pred = y_scaler.inverse_transform(pl.predict(X_train))\n# y_val_pred = y_scaler.inverse_transform(pl.predict(X_val))\ny_train_pred = (final_pipe.predict(X))\ny_val_pred =(final_pipe.predict(Xval))","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:52:11.346566Z","iopub.status.busy":"2021-04-23T16:52:11.345883Z","iopub.status.idle":"2021-04-23T16:52:11.530666Z","shell.execute_reply":"2021-04-23T16:52:11.531594Z"},"papermill":{"duration":0.206417,"end_time":"2021-04-23T16:52:11.531891","exception":false,"start_time":"2021-04-23T16:52:11.325474","status":"completed"},"tags":[],"id":"failing-wyoming"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rmse = np.sqrt(mean_squared_error(y_train_pred, y))\nprint(\"Training RMSE: {:.5f}\".format(float(train_rmse)))\nval_rmse = np.sqrt(mean_squared_error(y_val_pred, yval))\nprint(\"Validation RMSE: {:.5f}\".format(float(val_rmse)))","metadata":{"execution":{"iopub.execute_input":"2021-04-23T16:52:11.602768Z","iopub.status.busy":"2021-04-23T16:52:11.601716Z","iopub.status.idle":"2021-04-23T16:52:11.64379Z","shell.execute_reply":"2021-04-23T16:52:11.64445Z"},"papermill":{"duration":0.081357,"end_time":"2021-04-23T16:52:11.644626","exception":false,"start_time":"2021-04-23T16:52:11.563269","status":"completed"},"tags":[],"id":"endless-rhythm","outputId":"513d66d0-dc6d-4cde-9b3e-77e8a9bd28c7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit on the whole dataset and make prediction on the test set","metadata":{"id":"Ys51SEZgRvh3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_pipe.fit(X, y)","metadata":{"id":"ySXOUloKbqTu","outputId":"6712e95e-0c4d-406f-907e-cd68ed967b6a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test = df_test.drop(['ID', 'ARRIVAL_DELAY'], axis=1)","metadata":{"id":"OpqkKe1w6csF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test.shape","metadata":{"id":"nbLCgRGU8gX5","outputId":"da0ce1e5-be82-4fbf-9355-b565b3a203d6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_pred = final_pipe.predict(X_test)\n","metadata":{"id":"IlC7wPXvcagu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(data={'ID': df_test.ID.values,\n                                   'ARRIVAL_DELAY': y_test_pred.squeeze()})","metadata":{"id":"S7iAi6TL7XbG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time","metadata":{"id":"UyXDwZ_U7lkT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(f\"submission{time.time()}.csv\", index=False)","metadata":{"id":"2lOtBXUT7fv8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"53_11BPX7r9Z"},"execution_count":null,"outputs":[]}]}