{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\n\nimport matplotlib\nfrom matplotlib import rc\n\nrc(\"font\", **{\"family\": \"sans-serif\", \"sans-serif\": \"DejaVu Sans\"})\nrc(\"figure\", **{\"dpi\": 200})\nrc(\n    \"axes\",\n    **{\"spines.right\": False, \"spines.top\": False, \"xmargin\": 0.0, \"ymargin\": 0.05}\n)\npd.set_option(\"display.max_rows\", 200)\n","metadata":{"_cell_guid":"994dad23-6226-4b87-a0be-56f9a378ac40","_uuid":"349a1362-e369-49ae-906d-1b5f8a48475d","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-05-11T09:55:41.857936Z","iopub.execute_input":"2022-05-11T09:55:41.858688Z","iopub.status.idle":"2022-05-11T09:55:43.117973Z","shell.execute_reply.started":"2022-05-11T09:55:41.858575Z","shell.execute_reply":"2022-05-11T09:55:43.11709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\n    '../input/eurecom-aml-2022-challenge-1/public/train.csv', low_memory=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:55:43.119446Z","iopub.execute_input":"2022-05-11T09:55:43.119707Z","iopub.status.idle":"2022-05-11T09:56:56.796441Z","shell.execute_reply.started":"2022-05-11T09:55:43.11968Z","shell.execute_reply":"2022-05-11T09:56:56.795072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\n    '../input/eurecom-aml-2022-challenge-1/public/test_feat.csv', low_memory=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:56:56.797781Z","iopub.execute_input":"2022-05-11T09:56:56.798031Z","iopub.status.idle":"2022-05-11T09:57:17.296876Z","shell.execute_reply.started":"2022-05-11T09:56:56.797998Z","shell.execute_reply":"2022-05-11T09:57:17.295094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:17.300246Z","iopub.execute_input":"2022-05-11T09:57:17.300572Z","iopub.status.idle":"2022-05-11T09:57:17.37385Z","shell.execute_reply.started":"2022-05-11T09:57:17.300537Z","shell.execute_reply":"2022-05-11T09:57:17.367819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data analysis\n\nIn this challenge, you are free (and encouraged) to explore in depth the data you have, you can run simple queries on the data, perform exploration and compute statistics.\n\n**NOTE**: finding the right question to ask is difficult! Don't be afraid to complement your analysis with your own questions. This can give you extra points!\n\n**NOTE 2**: the presentation quality is critical in any business-oriented data analysis. Take time to create few but informative plots, rather than endless tables!\n","metadata":{}},{"cell_type":"code","source":"tr_coordinates = df[['fact_latitude', 'fact_longitude']].drop_duplicates()\nte_coordinates = df_test[['fact_latitude', 'fact_longitude']].drop_duplicates()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:17.376088Z","iopub.execute_input":"2022-05-11T09:57:17.376449Z","iopub.status.idle":"2022-05-11T09:57:17.578456Z","shell.execute_reply.started":"2022-05-11T09:57:17.376401Z","shell.execute_reply":"2022-05-11T09:57:17.577603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged = te_coordinates.merge(tr_coordinates, how='left', indicator=True)\ntropic_coordinates = merged[merged['_merge'] == 'left_only']\ncommon_coordinates = merged[merged['_merge'] == 'both']\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:17.579774Z","iopub.execute_input":"2022-05-11T09:57:17.580021Z","iopub.status.idle":"2022-05-11T09:57:17.629257Z","shell.execute_reply.started":"2022-05-11T09:57:17.579992Z","shell.execute_reply":"2022-05-11T09:57:17.628528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax0, ax1) = plt.subplots(1, 2, figsize=[\n    10, 3], dpi=200, sharex=True, sharey=True)\nax0.scatter(tr_coordinates['fact_longitude'], tr_coordinates['fact_latitude'],\n            s=1, c='tab:green', label='Train data')\nax1.scatter(common_coordinates['fact_longitude'], common_coordinates['fact_latitude'],\n            s=1, c='tab:orange', label='In-domain test data')\nax1.scatter(tropic_coordinates['fact_longitude'], tropic_coordinates['fact_latitude'],\n            s=1, c='tab:red', label='Out-domain test data')\nax0.legend(), ax1.legend()\nfig.suptitle('Distribution of train and test points in the dataset',\n             y=1.01, fontsize=14)\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:17.630391Z","iopub.execute_input":"2022-05-11T09:57:17.630645Z","iopub.status.idle":"2022-05-11T09:57:18.613449Z","shell.execute_reply.started":"2022-05-11T09:57:17.630609Z","shell.execute_reply":"2022-05-11T09:57:18.612619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\\***\\*Time VS Temeprature Analysis\\*\\***\n","metadata":{}},{"cell_type":"code","source":"# Understanding the time format and converting it into days, months and years\n\nprint(df['fact_time'])\nb = pd.to_datetime(df['fact_time'], unit='s')\n\nb = b.to_frame()\ntarg = df['fact_temperature']\n# mergedtime = b.merge(targ, how='left', indicator=True)\ntimetemp = []\ntimetemp = b.join(targ)\n\n\ntimetemp['Date'] = pd.to_datetime(timetemp['fact_time']).dt.date\ntimetemp['Time'] = pd.to_datetime(timetemp['fact_time']).dt.time\ntimetemp['Month'] = pd.to_datetime(timetemp['fact_time']).dt.month\ntimetemp['Weekday'] = pd.to_datetime(timetemp['fact_time']).dt.weekday\n\n\ntimetemp.insert(6, \"Temp\", timetemp['fact_temperature'])\ntimetemp.pop('fact_temperature')\n\n\nprint(timetemp)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:18.61491Z","iopub.execute_input":"2022-05-11T09:57:18.615738Z","iopub.status.idle":"2022-05-11T09:57:21.768159Z","shell.execute_reply.started":"2022-05-11T09:57:18.615693Z","shell.execute_reply":"2022-05-11T09:57:21.767264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Number of months and years over which the data was collected\n\nz = pd.to_datetime(timetemp['Date']).dt.year\nprint(z.unique())\n\nprint(timetemp['Month'].unique())\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:21.769913Z","iopub.execute_input":"2022-05-11T09:57:21.770304Z","iopub.status.idle":"2022-05-11T09:57:22.601636Z","shell.execute_reply.started":"2022-05-11T09:57:21.770258Z","shell.execute_reply":"2022-05-11T09:57:22.600668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"day = timetemp.groupby('Weekday').Temp.agg(['mean'])\nday_w = ['sun', 'mon', 'tue', 'wed', 'thurs', 'fri', 'sat']\n\n# print(day['mean'])\n# print(day_w)\n\n\nax1 = sns.barplot(day_w, day['mean'])\nax1.bar_label(ax1.containers[0])\n\nax1.set_xlabel(\"Day_of_week\", fontsize=10)\nax1.set_ylabel(\"Mean temperature\", fontsize=10)\nax1.set_title('Day_of_week Vs Mean Temperature Analysis')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:22.605525Z","iopub.execute_input":"2022-05-11T09:57:22.606253Z","iopub.status.idle":"2022-05-11T09:57:22.986388Z","shell.execute_reply.started":"2022-05-11T09:57:22.606213Z","shell.execute_reply":"2022-05-11T09:57:22.98526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The mean teamperature over all the days is almost same. So we will now go to a more deeper analysis.\n","metadata":{}},{"cell_type":"code","source":"monthm = timetemp.groupby('Month').Temp.agg(['mean'])\n\nmonth = timetemp.Month.unique()\n\nax = sns.barplot(np.sort(month), monthm['mean'])\nax.bar_label(ax.containers[0])\nax.set_xlabel(\"Month\", fontsize=10)\nax.set_ylabel(\"Average temperature\", fontsize=10)\nax.set_title('Month Vs Mean Temperature Analysis')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:22.98815Z","iopub.execute_input":"2022-05-11T09:57:22.988424Z","iopub.status.idle":"2022-05-11T09:57:23.339549Z","shell.execute_reply.started":"2022-05-11T09:57:22.988387Z","shell.execute_reply":"2022-05-11T09:57:23.338628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The months over whihc data was considered were Sepetember, October, November and December of 2018, and January and February of 2019. The average temeperature is high during the Sepetember and October months as they are almost summer. The temperature keeps on decerasing from November till January due to the winter days. The temeperature starts increasing from February again as it is begining of Spring.\n","metadata":{}},{"cell_type":"code","source":"data = timetemp.groupby('Month').Temp.agg(['mean', 'min', 'max'])\ndata.reset_index(inplace=True)\nprint(data)\n\nmonths = data['Month']\nmeans = data['mean']\nmins = data['min']\nmaxs = data['max']\n\n\nplt.errorbar(months, means, fmt='ok')\nplt.errorbar(months, means, [means - mins, maxs - means],\n             ecolor='blue', lw=1)\nplt.xlim(0, 13)\nplt.title('Monthly Analysis')\n\n# ax1 = sns.boxplot(months,means)\n\nax = data.plot(x='Month', y='mean', c='white')\nplt.fill_between(x='Month', y1='min', y2='max', data=data)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:23.341157Z","iopub.execute_input":"2022-05-11T09:57:23.34141Z","iopub.status.idle":"2022-05-11T09:57:24.047075Z","shell.execute_reply.started":"2022-05-11T09:57:23.341379Z","shell.execute_reply":"2022-05-11T09:57:24.045976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The minimum, maximum and the average temperature were plotted and the behaviour was as expected as shown in the plots.\n","metadata":{}},{"cell_type":"code","source":"# OCTOBER\noctt = timetemp.loc[timetemp['Month'] == 10]\n# df.loc[df['col1'] == value]\n\ndays = octt.groupby('Weekday').Temp.agg(['mean'])\n\ndata = octt.groupby('Weekday').Temp.agg(['mean', 'min', 'max'])\ndata.reset_index(inplace=True)\n# print(data)\n\ndays = data['Weekday']\nmeans = data['mean']\nmins = data['min']\nmaxs = data['max']\n\nfig, ax = plt.subplots(2, 3, sharex=True, figsize=(10, 5))\n\nax[0, 1].errorbar(days, means, fmt='ok')\nax[0, 1].errorbar(days, means, [means - mins, maxs - means],\n                  ecolor='blue', lw=1)\nax[0, 1].set_xlabel(\"October Days\", fontsize=8)\nax[0, 1].set_ylabel(\"Temparature\", fontsize=8)\nax[0, 1].set_ylim([-50, 60])\n\n\nplt.xlim(-1, 7)\n\nfig.suptitle('Monthly Days vs Temperature Analysis')\n# print(octt)\n\n# SEPTEMBER\nsept = timetemp.loc[timetemp['Month'] == 9]\n# df.loc[df['col1'] == value]\n\ndays1 = sept.groupby('Weekday').Temp.agg(['mean'])\n\ndata1 = sept.groupby('Weekday').Temp.agg(['mean', 'min', 'max'])\ndata1.reset_index(inplace=True)\n# print(data)\n\ndays1 = data1['Weekday']\nmeans1 = data1['mean']\nmins1 = data1['min']\nmaxs1 = data1['max']\n\n\nax[0, 0].errorbar(days1, means1, fmt='ok')\nax[0, 0].errorbar(days1, means1, [means1 - mins1, maxs1 - means1],\n                  ecolor='blue', lw=1)\nax[0, 0].set_xlabel(\"September Days\", fontsize=8)\nax[0, 0].set_ylabel(\"Temparature\", fontsize=8)\nax[0, 0].set_ylim([-50, 60])\n\n\n# print(data1)\n\n# NOVEMBER\nnov = timetemp.loc[timetemp['Month'] == 11]\n# df.loc[df['col1'] == value]\n\ndays2 = nov.groupby('Weekday').Temp.agg(['mean'])\n\ndata2 = nov.groupby('Weekday').Temp.agg(['mean', 'min', 'max'])\ndata2.reset_index(inplace=True)\n# print(data)\n\ndays2 = data2['Weekday']\nmeans2 = data2['mean']\nmins2 = data2['min']\nmaxs2 = data2['max']\n\n\nax[0, 2].errorbar(days2, means2, fmt='ok')\nax[0, 2].errorbar(days2, means2, [means2 - mins2, maxs2 - means2],\n                  ecolor='blue', lw=1)\nax[0, 2].set_xlabel(\"November Days\", fontsize=8)\nax[0, 2].set_ylabel(\"Temparature\", fontsize=8)\nax[0, 2].set_ylim([-50, 60])\n\n\n# DECEMBER\ndec = timetemp.loc[timetemp['Month'] == 12]\n# df.loc[df['col1'] == value]\n\ndays3 = dec.groupby('Weekday').Temp.agg(['mean'])\n\ndata3 = dec.groupby('Weekday').Temp.agg(['mean', 'min', 'max'])\ndata3.reset_index(inplace=True)\n# print(data)\n\ndays3 = data3['Weekday']\nmeans3 = data3['mean']\nmins3 = data3['min']\nmaxs3 = data3['max']\n\n\nax[1, 0].errorbar(days3, means3, fmt='ok')\nax[1, 0].errorbar(days3, means3, [means3 - mins3, maxs3 - means3],\n                  ecolor='blue', lw=1)\nax[1, 0].set_xlabel(\"December Days\", fontsize=8)\nax[1, 0].set_ylabel(\"Temparature\", fontsize=8)\nax[1, 0].set_ylim([-50, 60])\n\n\n# JANUARY\njan = timetemp.loc[timetemp['Month'] == 1]\n# df.loc[df['col1'] == value]\n\ndays4 = jan.groupby('Weekday').Temp.agg(['mean'])\n\ndata4 = jan.groupby('Weekday').Temp.agg(['mean', 'min', 'max'])\ndata4.reset_index(inplace=True)\n# print(data)\n\ndays4 = data4['Weekday']\nmeans4 = data4['mean']\nmins4 = data4['min']\nmaxs4 = data4['max']\n\n\nax[1, 1].errorbar(days4, means4, fmt='ok')\nax[1, 1].errorbar(days4, means4, [means4 - mins4, maxs4 - means4],\n                  ecolor='blue', lw=1)\nax[1, 1].set_xlabel(\"January Days\", fontsize=8)\nax[1, 1].set_ylabel(\"Temparature\", fontsize=8)\nax[1, 0].set_ylim([-50, 60])\n\n\n# FEBRUARY\nfeb = timetemp.loc[timetemp['Month'] == 2]\n# df.loc[df['col1'] == value]\n\ndays5 = feb.groupby('Weekday').Temp.agg(['mean'])\n\ndata5 = feb.groupby('Weekday').Temp.agg(['mean', 'min', 'max'])\ndata5.reset_index(inplace=True)\n# print(data)\n\ndays5 = data5['Weekday']\nmeans5 = data5['mean']\nmins5 = data5['min']\nmaxs5 = data5['max']\n\n\nax[1, 2].errorbar(days5, means5, fmt='ok')\nax[1, 2].errorbar(days5, means5, [means5 - mins5, maxs5 - means5],\n                  ecolor='blue', lw=1)\nax[1, 2].set_xlabel(\"February Days\", fontsize=8)\nax[1, 2].set_ylabel(\"Temparature\", fontsize=8)\nax[1, 2].set_ylim([-50, 60])\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:24.049152Z","iopub.execute_input":"2022-05-11T09:57:24.049522Z","iopub.status.idle":"2022-05-11T09:57:25.477646Z","shell.execute_reply.started":"2022-05-11T09:57:24.04946Z","shell.execute_reply":"2022-05-11T09:57:25.476604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The temperature of all the days over all the months were plotted as the information was not clear about the day of the week and temeperature. During the weekends, the temeperature was at its extremeties (highest during the summer and low during the winter months). The rest of the days almost showed similar behaviour as with the other days during that particular month.\n","metadata":{}},{"cell_type":"markdown","source":"**Statical Information about the dataset**\n","metadata":{}},{"cell_type":"code","source":"df.describe()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:25.479178Z","iopub.execute_input":"2022-05-11T09:57:25.479477Z","iopub.status.idle":"2022-05-11T09:57:34.447457Z","shell.execute_reply.started":"2022-05-11T09:57:25.479441Z","shell.execute_reply":"2022-05-11T09:57:34.446546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking for datatypes\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:34.448642Z","iopub.execute_input":"2022-05-11T09:57:34.448869Z","iopub.status.idle":"2022-05-11T09:57:34.468389Z","shell.execute_reply.started":"2022-05-11T09:57:34.448843Z","shell.execute_reply":"2022-05-11T09:57:34.467659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Missing Data per column**\n","metadata":{}},{"cell_type":"code","source":"df_missing = df.isna().sum()\ndf_missing","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:34.469705Z","iopub.execute_input":"2022-05-11T09:57:34.470369Z","iopub.status.idle":"2022-05-11T09:57:34.916128Z","shell.execute_reply.started":"2022-05-11T09:57:34.470336Z","shell.execute_reply":"2022-05-11T09:57:34.915517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**SKEW Plot**\n","metadata":{}},{"cell_type":"code","source":"from scipy import stats\n\n# Extract training and testing data\\\\\\\n\ntestcol60 = [\n    'index',\n    'wrf_t2_interpolated',\n    'wrf_t2_next',\n    'gfs_temperature_97500',\n    'gfs_temperature_95000',\n    'climate_temperature',\n    'gfs_temperature_92500',\n    'gfs_temperature_90000',\n    'gfs_temperature_85000',\n    'gfs_temperature_80000',\n    'cmc_0_0_6_2',\n    'gfs_temperature_75000',\n    'gfs_temperature_70000',\n    'gfs_temperature_65000',\n    'cmc_0_1_0_0',\n    'gfs_temperature_60000',\n    'gfs_temperature_55000',\n    'gfs_temperature_50000',\n    'gfs_temperature_45000',\n    'gfs_temperature_40000',\n    'gfs_2m_dewpoint_next',\n    'gfs_2m_dewpoint',\n    'gfs_temperature_35000',\n    'gfs_temperature_30000',\n    'cmc_0_3_5_500',\n    'gfs_precipitable_water',\n    'gfs_temperature_25000',\n    'cmc_0_3_5_700',\n    'gfs_temperature_10000',\n    'gfs_temperature_15000',\n    'fact_time',\n    'gfs_temperature_7000',\n    'fact_latitude',\n    'cmc_0_0_7_2',\n    'cmc_0_3_1_0',\n    'sun_elevation',\n    'cmc_0_3_5_1000',\n    'cmc_0_3_5_850',\n    'gfs_humidity',\n    'cmc_0_2_2_500',\n    'cmc_0_2_2_700',\n    'cmc_0_1_66_0_next',\n    'cmc_0_1_66_0',\n    'gfs_temperature_20000',\n    'gfs_a_vorticity',\n    'wrf_rh2',\n    'gfs_total_clouds_cover_middle',\n    'gfs_total_clouds_cover_low',\n    'gfs_total_clouds_cover_low_next',\n    'cmc_0_2_2_850',\n    'cmc_0_0_7_1000',\n    'gfs_cloudness',\n    'topography_bathymetry',\n    'fact_longitude',\n    'cmc_0_2_2_925',\n    'cmc_0_2_2_10',\n    'climate_pressure',\n    'gfs_r_velocity',\n    'cmc_0_3_0_0',\n    'cmc_0_1_66_0_grad',\n    'gfs_pressure',\n    'fact_temperature'\n]\n\n\n# Check skewness of 60 features\nskew_val = list(df[testcol60].skew())\n\n\nplt.plot(skew_val)\nplt.ylim(-3, 21)\nplt.title(\"Skew Plot of most related features\")\n\n# Add labels\n\nplt.xlabel(\"i(th) Feature\\n Skew Plot\")\nplt.ylabel(\"Skew Score\")\nplt.savefig('/kaggle/working/skew_plot.png')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:34.917235Z","iopub.execute_input":"2022-05-11T09:57:34.917564Z","iopub.status.idle":"2022-05-11T09:57:37.885857Z","shell.execute_reply.started":"2022-05-11T09:57:34.917535Z","shell.execute_reply":"2022-05-11T09:57:37.885029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The overall dataset is very slightly skewed with a skew score of -0.077248, with individual columns going as high as 21.0. The skew scores are seen on the plot.\n","metadata":{}},{"cell_type":"markdown","source":"**Distribution**\n","metadata":{}},{"cell_type":"code","source":"df.hist(figsize=(40, 40))\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:57:37.887353Z","iopub.execute_input":"2022-05-11T09:57:37.888154Z","iopub.status.idle":"2022-05-11T09:58:00.872163Z","shell.execute_reply.started":"2022-05-11T09:57:37.888107Z","shell.execute_reply":"2022-05-11T09:58:00.868094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The data distribution for all the features are checked for normal distribution using the Shapiro-Wilk  \t Test. Features that are deviating from the normal bell have a p-value of 0, while the others have 1.0.","metadata":{}},{"cell_type":"code","source":"from scipy.stats import shapiro\n\ncols = df.columns\nstat=[]\np=[]\nfor i in cols:\n    stat_i, p_i = shapiro(df[i])\n    stat.append(i)\n    p.append(p_i)\n\nplt.plot(p)\nplt.title(\"Normality Check of All features\")\n\n# Add labels\n\nplt.xlabel(\"i(th) Feature\")\nplt.ylabel(\"P-Value\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:58:00.873554Z","iopub.execute_input":"2022-05-11T09:58:00.874303Z","iopub.status.idle":"2022-05-11T09:58:21.65356Z","shell.execute_reply.started":"2022-05-11T09:58:00.874266Z","shell.execute_reply":"2022-05-11T09:58:21.652855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Pre-processing\n\nThe previous step should give you a better understanding of which pre-processing is required for the data. This may include:\n\n- Normalising and standardising the given data;\n- Removing outliers;\n- Carrying out feature selection, possibly using metrics derived from information theory;\n- Handling missing information in the dataset;\n- Augmenting the dataset with external information;\n- Combining existing features.\n\nBelow is a very basic example of pre-processing steps.\n","metadata":{}},{"cell_type":"markdown","source":"**Handling Missing Information**\n\nWe resolve all the missing values by putting the mean.\n","metadata":{}},{"cell_type":"code","source":"df.fillna(df.mean(), inplace=True)\n\n# # REsolver all missing values by putting the mean value\n# from sklearn.impute import SimpleImputer\n# imp_mean = SimpleImputer(missing_values = np.nan, strategy = 'mean')\n# imp_mean.fit(df.iloc[:,1:-1])\n# df = imp_mean.transform(df.iloc[:,1:-1])\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:58:21.654697Z","iopub.execute_input":"2022-05-11T09:58:21.655144Z","iopub.status.idle":"2022-05-11T09:58:22.399146Z","shell.execute_reply.started":"2022-05-11T09:58:21.655108Z","shell.execute_reply":"2022-05-11T09:58:22.398292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:58:22.4005Z","iopub.execute_input":"2022-05-11T09:58:22.400745Z","iopub.status.idle":"2022-05-11T09:58:22.431407Z","shell.execute_reply.started":"2022-05-11T09:58:22.400718Z","shell.execute_reply":"2022-05-11T09:58:22.430436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Feature Selection**\n","metadata":{}},{"cell_type":"markdown","source":"**Correlation Matrix**\n","metadata":{}},{"cell_type":"code","source":"corrmat = np.abs(df).corr()\nmask = np.zeros_like(corrmat)\nmask[np.triu_indices_from(mask)] = True\nwith sns.axes_style(\"white\"):\n    f, ax = plt.subplots(figsize=(30, 30))\n    ax = sns.heatmap(corrmat, mask=mask, square=True, cmap=\"YlGnBu\")\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:58:22.433002Z","iopub.execute_input":"2022-05-11T09:58:22.433543Z","iopub.status.idle":"2022-05-11T09:59:42.432319Z","shell.execute_reply.started":"2022-05-11T09:58:22.433471Z","shell.execute_reply":"2022-05-11T09:59:42.429125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**UNIVARIATE Selection**\n","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import f_regression\n\nbestfeatures = SelectKBest(score_func=f_regression, k=60)\nfit = bestfeatures.fit(df.iloc[:, 1:-1], df.iloc[:, -1])\ndfscores = pd.DataFrame(fit.scores_)\ndfcolumns = pd.DataFrame(df.columns)\n# concat two dataframes for better visualization\nfeatureScores = pd.concat([dfcolumns, dfscores], axis=1)\nfeatureScores.columns = ['Specs', 'Score']\nprint(featureScores.nlargest(60, 'Score'))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:42.434184Z","iopub.execute_input":"2022-05-11T09:59:42.434707Z","iopub.status.idle":"2022-05-11T09:59:44.084876Z","shell.execute_reply.started":"2022-05-11T09:59:42.434653Z","shell.execute_reply":"2022-05-11T09:59:44.078769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Mutual Information betwen features**\n","metadata":{}},{"cell_type":"code","source":"# from sklearn.feature_selection import mutual_info_regression\n# mutual_info = mutual_info_regression(df.iloc[:, 1:-1], df.iloc[:, -1])\n# mutual_info = pd.Series(mutual_info)\n# mutual_info.index = df.iloc[:, 1:-1].columns\n# mutual_info.sort_values(ascending=False).plot.bar(figsize=(30, 30))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:44.089008Z","iopub.execute_input":"2022-05-11T09:59:44.089972Z","iopub.status.idle":"2022-05-11T09:59:44.098931Z","shell.execute_reply.started":"2022-05-11T09:59:44.089911Z","shell.execute_reply":"2022-05-11T09:59:44.097931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We select features with:\n\n- High correlation with fact_tempearature\n- High mutual information betwen the features and fact_tempearature]\n- Best features from univariate selection\n","metadata":{}},{"cell_type":"code","source":"col_selection = [\n    'index',\n    'wrf_t2_interpolated',\n    'wrf_t2_next',\n    'gfs_temperature_97500',\n    'gfs_temperature_95000',\n    'climate_temperature',\n    'gfs_temperature_92500',\n    'gfs_temperature_90000',\n    'gfs_temperature_85000',\n    'gfs_temperature_80000',\n    'cmc_0_0_6_2',\n    'gfs_temperature_75000',\n    'gfs_temperature_70000',\n    'gfs_temperature_65000',\n    'cmc_0_1_0_0',\n    'gfs_temperature_60000',\n    'gfs_temperature_55000',\n    'gfs_temperature_50000',\n    'gfs_temperature_45000',\n    'gfs_temperature_40000',\n    'gfs_2m_dewpoint_next',\n    'gfs_2m_dewpoint',\n    'gfs_temperature_35000',\n    'gfs_temperature_30000',\n    'cmc_0_3_5_500',\n    'gfs_precipitable_water',\n    'gfs_temperature_25000',\n    'cmc_0_3_5_700',\n    'gfs_temperature_10000',\n    'gfs_temperature_15000',\n    'fact_time',\n    'gfs_temperature_7000',\n    'fact_latitude',\n    'cmc_0_0_7_2',\n    'cmc_0_3_1_0',\n    'sun_elevation',\n    'cmc_0_3_5_1000',\n    'cmc_0_3_5_850',\n    'gfs_humidity',\n    'cmc_0_2_2_500',\n    'cmc_0_2_2_700',\n    'cmc_0_1_66_0_next',\n    'cmc_0_1_66_0',\n    'gfs_temperature_20000',\n    'gfs_a_vorticity',\n    'wrf_rh2',\n    'gfs_total_clouds_cover_middle',\n    'gfs_total_clouds_cover_low',\n    'gfs_total_clouds_cover_low_next',\n    'cmc_0_2_2_850',\n    'cmc_0_0_7_1000',\n    'gfs_cloudness',\n    'topography_bathymetry',\n    'fact_longitude',\n    'cmc_0_2_2_925',\n    'cmc_0_2_2_10',\n    'climate_pressure',\n    'gfs_r_velocity',\n    'cmc_0_3_0_0',\n    'cmc_0_1_66_0_grad',\n    'gfs_pressure',\n    'fact_temperature'\n]\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:44.100771Z","iopub.execute_input":"2022-05-11T09:59:44.105262Z","iopub.status.idle":"2022-05-11T09:59:44.122671Z","shell.execute_reply.started":"2022-05-11T09:59:44.105196Z","shell.execute_reply":"2022-05-11T09:59:44.121599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df[col_selection].dropna().iloc[:, 1:-1].values\ny = df[col_selection].dropna().iloc[:, -1].values\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:44.128741Z","iopub.execute_input":"2022-05-11T09:59:44.131762Z","iopub.status.idle":"2022-05-11T09:59:46.557399Z","shell.execute_reply.started":"2022-05-11T09:59:44.131701Z","shell.execute_reply":"2022-05-11T09:59:46.556722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xmean, Xstd, ymean, ystd = X.mean(0), X.std(0), y.mean(), y.std()\nXmin, Xmax, ymin, ymax = X.min(axis=0), X.max(axis=0), y.min(axis=0), y.max(axis=0)\nX = (X - Xmean) / Xstd\ny = (y - ymean) / ystd\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:46.558409Z","iopub.execute_input":"2022-05-11T09:59:46.559168Z","iopub.status.idle":"2022-05-11T09:59:47.9932Z","shell.execute_reply.started":"2022-05-11T09:59:46.559132Z","shell.execute_reply":"2022-05-11T09:59:47.992221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Selection\n\nPerhaps one of the most important segments of this challenge involves the selection of a model that can successfully handle the given data and yield sensible predictions. Instead of focusing exclusively on your final chosen model, it is also important to share your thought process in this notebook by additionally describing alternative candidate models. There is a wealth of models to choose from, such as decision trees, random forests, (Bayesian) neural networks, Gaussian processes, Lasso regression, and so on. There are several factors which may influence your decision:\n\n- What is the model's complexity?\n- Is the model interpretable?\n- Is the model capable of handling different data-types?\n- Does the model return uncertainty estimates along with predictions?\n\nIn this baseline solution, we use the Lasso regression model, which is a linear least-square model with L1 regularization on its parameters. There is a hyper-parameter that should be tuned which is the regularization strength α. Intuitively, this hyper-parameter controls the amount of shrinkage of the parameters of the model: the larger the value of α the greater the amount of shrinkage.\n\nSection 3.4.1 of the book The Elements of Statistical Learning: Data Mining, Inference, and Prediction from Trevor Hastie et al. (https://hastie.su.domains/Papers/ESLII.pdf) is a good reference for some classic regression models.\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nXtr, Xval, ytr, yval = train_test_split(X, y, random_state=1, test_size=10000)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:47.997488Z","iopub.execute_input":"2022-05-11T09:59:47.997779Z","iopub.status.idle":"2022-05-11T09:59:49.638125Z","shell.execute_reply.started":"2022-05-11T09:59:47.997746Z","shell.execute_reply":"2022-05-11T09:59:49.637047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_rmse(y, ypred, ystd=1.):\n    return np.mean((y - ypred)**2)**0.5 * ystd\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:49.639678Z","iopub.execute_input":"2022-05-11T09:59:49.64Z","iopub.status.idle":"2022-05-11T09:59:49.645111Z","shell.execute_reply.started":"2022-05-11T09:59:49.639959Z","shell.execute_reply":"2022-05-11T09:59:49.644243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Find Best alpha for Lasso model\n","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LassoCV\n\n# Lasso with 5 fold cross-validation\nlasso_modelCV = LassoCV(cv=10, random_state=0, max_iter=10000)\n\n# Fit model\nlasso_modelCV.fit(Xtr, ytr)\n\nalpha = lasso_modelCV.alpha_\nprint(alpha)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:59:49.646247Z","iopub.execute_input":"2022-05-11T09:59:49.646549Z","iopub.status.idle":"2022-05-11T10:01:03.296276Z","shell.execute_reply.started":"2022-05-11T09:59:49.646519Z","shell.execute_reply":"2022-05-11T10:01:03.295368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import Lasso\n\nlasso_model = Lasso(alpha=alpha)\nlasso_model.fit(Xtr, ytr)\nypred_tr = lasso_model.predict(Xtr)\nypred_val = lasso_model.predict(Xval)\n\nprint(f'Train RMSE: {compute_rmse(ytr, ypred_tr, ystd):.3f}')\nprint(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T10:01:03.297736Z","iopub.execute_input":"2022-05-11T10:01:03.298325Z","iopub.status.idle":"2022-05-11T10:01:35.052891Z","shell.execute_reply.started":"2022-05-11T10:01:03.298279Z","shell.execute_reply":"2022-05-11T10:01:35.051921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.tree import DecisionTreeRegressor\n\n# Build decision tree\ndecison_tree_model = DecisionTreeRegressor(max_depth=10, random_state=0)\ndecison_tree_model.fit(Xtr, ytr)\n\nypred_tr = decison_tree_model.predict(Xtr)\nypred_val = decison_tree_model.predict(Xval)\n\nprint(f'Train RMSE: {compute_rmse(ytr, ypred_tr, ystd):.3f}')\nprint(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T10:01:35.054947Z","iopub.execute_input":"2022-05-11T10:01:35.055639Z","iopub.status.idle":"2022-05-11T10:04:11.684795Z","shell.execute_reply.started":"2022-05-11T10:01:35.055588Z","shell.execute_reply":"2022-05-11T10:04:11.683803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBRegressor\n\nxgboost_model = XGBRegressor(max_depth=5, subsample=0.8, colsample_bytree=0.8)\nxgboost_model.fit(Xtr, ytr)\n\nypred_tr = xgboost_model.predict(Xtr)\nypred_val = xgboost_model.predict(Xval)\n\nprint(f'Train RMSE: {compute_rmse(ytr, ypred_tr, ystd):.3f}')\nprint(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T10:04:11.686153Z","iopub.execute_input":"2022-05-11T10:04:11.686384Z","iopub.status.idle":"2022-05-11T10:17:23.488114Z","shell.execute_reply.started":"2022-05-11T10:04:11.686355Z","shell.execute_reply":"2022-05-11T10:17:23.487336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestRegressor\n\nrandom_forest_model = RandomForestRegressor(n_estimators = 50, random_state = 50, bootstrap= True, \n                                            max_depth= 80, max_features= 'auto',min_samples_leaf= 4,\n                                            min_samples_split=10)\n\nrandom_forest_model.fit(Xtr, ytr)\n\nypred_tr = random_forest_model.predict(Xtr)\nypred_val = random_forest_model.predict(Xval)\n\nprint(f'Train RMSE: {compute_rmse(ytr, ypred_tr, ystd):.3f}')\nprint(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T10:17:23.489895Z","iopub.execute_input":"2022-05-11T10:17:23.490386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Parameter Optimisation\n\nIrrespective of your choice, it is highly likely that your model will have one or more parameters that require tuning. There are several techniques for carrying out such a procedure, including cross-validation, Bayesian optimisation, and several others. As before, an analysis into which parameter tuning technique best suits your model is expected before proceeding with the optimisation of your model.\n\nThe below cells demonstrate tuning the hyper-parameters α of the Ridge regression model by using cross-validation.\n","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\n\ndef get_cv_idx(n, test_size=0.2, n_splits=2):\n    train_idx, test_idx = [], []\n    for _ in range(n_splits):\n        idx = np.random.permutation(n)\n        train_size = int(n * (1 - test_size)\n                         ) if isinstance(test_size, float) else n - test_size\n        train_idx.append(idx[:train_size])\n        test_idx.append(idx[train_size:])\n    return train_idx, test_idx\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_idx, cv_idx = get_cv_idx(len(Xtr), test_size=10000, n_splits=10)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Random Forest Optimization**\n","metadata":{}},{"cell_type":"code","source":"# param_grid = {\n#     \"max_depth\": [30, 40, 60]\n# }\n\n# search_random_forest = GridSearchCV(random_forest_model,\n#                       param_grid,\n#                       n_jobs=-1,\n#                       verbose=1,\n#                       cv=zip(train_idx, cv_idx),\n#                       scoring='neg_root_mean_squared_error').fit(Xtr, ytr)\n# print('Done!')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Best parameters set found on cv set:\")\n# print(search_random_forest.best_params_)\n# print()\n# print(\"Grid scores on cv set:\")\n# means = search_random_forest.cv_results_[\"mean_test_score\"]\n# stds = search_random_forest.cv_results_[\"std_test_score\"]\n# for mean, std, params in zip(means, stds, search_random_forest.cv_results_[\"params\"]):\n#     print(\"%0.3f (+/-%0.03f) for %r\" % (-mean * ystd, (std * ystd) * 2, params))\n# print()\n# print(\"Error on the validation set\")\n# ypred_val = search_lasso.predict(Xval)\n# print(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Decison Tree Optimization**\n","metadata":{}},{"cell_type":"code","source":"# param_grid = {\n#     \"max_depth\": [20, 40, 70]\n# }\n\n# search_decison_tree = GridSearchCV(decison_tree_model,\n#                       param_grid,\n#                       n_jobs=-1,\n#                       verbose=1,\n#                       cv=zip(train_idx, cv_idx),\n#                       scoring='neg_root_mean_squared_error').fit(Xtr, ytr)\n# print('Done!')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(\"Best parameters set found on cv set:\")\n# print(search_decison_tree.best_params_)\n# print()\n# print(\"Grid scores on cv set:\")\n# means = search_decison_tree.cv_results_[\"mean_test_score\"]\n# stds = search_decison_tree.cv_results_[\"std_test_score\"]\n# for mean, std, params in zip(means, stds, search_decison_tree.cv_results_[\"params\"]):\n#     print(\"%0.3f (+/-%0.03f) for %r\" % (-mean * ystd, (std * ystd) * 2, params))\n# print()\n# print(\"Error on the validation set\")\n# ypred_val = search_decison_tree.predict(Xval)\n# print(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation\n\nSome form of pre-evaluation will inevitably be required in the preceding sections in order to both select an appropriate model and configure its parameters appropriately. In this final section, you may evaluate other aspects of the model such as:\n\n- Assessing the running time of your model;\n- Determining whether some aspects can be parallelised;\n- Training the model with smaller subsets of the data.\n- etc.\n\nRemember, the goal of this challenge is to construct a model for predicting the temperature around the globe.\n","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(\n    '../input/eurecom-aml-2022-challenge-1/public/test_feat.csv', low_memory=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_selection = [\n    'index',\n    'wrf_t2_interpolated',\n    'wrf_t2_next',\n    'gfs_temperature_97500',\n    'gfs_temperature_95000',\n    'climate_temperature',\n    'gfs_temperature_92500',\n    'gfs_temperature_90000',\n    'gfs_temperature_85000',\n    'gfs_temperature_80000',\n    'cmc_0_0_6_2',\n    'gfs_temperature_75000',\n    'gfs_temperature_70000',\n    'gfs_temperature_65000',\n    'cmc_0_1_0_0',\n    'gfs_temperature_60000',\n    'gfs_temperature_55000',\n    'gfs_temperature_50000',\n    'gfs_temperature_45000',\n    'gfs_temperature_40000',\n    'gfs_2m_dewpoint_next',\n    'gfs_2m_dewpoint',\n    'gfs_temperature_35000',\n    'gfs_temperature_30000',\n    'cmc_0_3_5_500',\n    'gfs_precipitable_water',\n    'gfs_temperature_25000',\n    'cmc_0_3_5_700',\n    'gfs_temperature_10000',\n    'gfs_temperature_15000',\n    'fact_time',\n    'gfs_temperature_7000',\n    'fact_latitude',\n    'cmc_0_0_7_2',\n    'cmc_0_3_1_0',\n    'sun_elevation',\n    'cmc_0_3_5_1000',\n    'cmc_0_3_5_850',\n    'gfs_humidity',\n    'cmc_0_2_2_500',\n    'cmc_0_2_2_700',\n    'cmc_0_1_66_0_next',\n    'cmc_0_1_66_0',\n    'gfs_temperature_20000',\n    'gfs_a_vorticity',\n    'wrf_rh2',\n    'gfs_total_clouds_cover_middle',\n    'gfs_total_clouds_cover_low',\n    'gfs_total_clouds_cover_low_next',\n    'cmc_0_2_2_850',\n    'cmc_0_0_7_1000',\n    'gfs_cloudness',\n    'topography_bathymetry',\n    'fact_longitude',\n    'cmc_0_2_2_925',\n    'cmc_0_2_2_10',\n    'climate_pressure',\n    'gfs_r_velocity',\n    'cmc_0_3_0_0',\n    'cmc_0_1_66_0_grad',\n    'gfs_pressure'\n]\n\n\nXte = df_test[col_selection].iloc[:, 1:].values\nXte = (Xte - Xmean) / Xstd\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"First we re train our best model using the whole dataset, with no spliting for validation (more data theorical mean better performance)\n","metadata":{}},{"cell_type":"code","source":"random_forest_model.fit(X, y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remember to un-standardize the predictions\nypred_te = random_forest_model.predict(Xte) * ystd + ymean\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission\n\nYour submission is a CSV file containing your final model's predictions on the given test data. This file should contain a header and have the following format:\n\n```\nindex,fact_temperature\n1993574,3.9865149124872303\n1993575,18.165092058370533\n1993576,16.53315442160854\n1993577,8.377598784006866\n...\n```\n\nA leaderboard for this challenge will be ranked using the root mean squared error between the predicted values and the observed arrival delays. However, you can use other metrics for regression tasks in your presentation notebook to evaluate many aspects of your model, including quantification of the uncertainty in the predictions.\n\nBelow is an example of creating a submission file.\n","metadata":{}},{"cell_type":"code","source":"submission_df = pd.DataFrame(data={'index': df_test['index'].values,\n                                   'fact_temperature': ypred_te.squeeze()})\n\n# Save the predictions into a csv file\n# Notice that this file should be saved under the directory `/kaggle/working`\n# so that you can download it later\nsubmission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the submission file\n! head - 6 \"/kaggle/working/submission.csv\"\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}