{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd \nimport matplotlib.pyplot as plt \nimport numpy as np \n\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.preprocessing import StandardScaler\n\nimport matplotlib\nfrom matplotlib import rc\n\nimport math\n\nrc(\"font\", **{\"family\": \"sans-serif\", \"sans-serif\": \"DejaVu Sans\"})\nrc(\"figure\", **{\"dpi\": 200})\nrc(\n    \"axes\",\n    **{\"spines.right\": False, \"spines.top\": False, \"xmargin\": 0.0, \"ymargin\": 0.05}\n)","metadata":{"_cell_guid":"994dad23-6226-4b87-a0be-56f9a378ac40","_uuid":"349a1362-e369-49ae-906d-1b5f8a48475d","execution":{"iopub.execute_input":"2022-04-20T12:06:00.664683Z","iopub.status.busy":"2022-04-20T12:06:00.663391Z","iopub.status.idle":"2022-04-20T12:06:00.671665Z","shell.execute_reply":"2022-04-20T12:06:00.670824Z","shell.execute_reply.started":"2022-04-20T12:06:00.664634Z"},"jupyter":{"outputs_hidden":false},"collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('public/train.csv', low_memory=True)\ndf_test = pd.read_csv('public/test_feat.csv', low_memory=True)","metadata":{"execution":{"iopub.execute_input":"2022-04-20T11:53:21.663111Z","iopub.status.busy":"2022-04-20T11:53:21.662828Z","iopub.status.idle":"2022-04-20T11:54:31.472034Z","shell.execute_reply":"2022-04-20T11:54:31.470628Z","shell.execute_reply.started":"2022-04-20T11:53:21.663072Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data analysis\n\nIn this challenge, you are free (and encouraged) to explore in depth the data you have, you can run simple queries on the data, perform exploration and compute statistics.\n\n**NOTE**: finding the right question to ask is difficult! Don't be afraid to complement your analysis with your own questions. This can give you extra points!\n\n**NOTE 2**: the presentation quality is critical in any business-oriented data analysis. Take time to create few but informative plots, rather than endless tables! ","metadata":{}},{"cell_type":"markdown","source":"# Geographic Analysis","metadata":{}},{"cell_type":"code","source":"tr_coordinates = df[['fact_latitude','fact_longitude']].drop_duplicates()\nte_coordinates = df_test[['fact_latitude','fact_longitude']].drop_duplicates()","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:03:23.224049Z","iopub.status.busy":"2022-04-20T12:03:23.223225Z","iopub.status.idle":"2022-04-20T12:03:23.500428Z","shell.execute_reply":"2022-04-20T12:03:23.499273Z","shell.execute_reply.started":"2022-04-20T12:03:23.223975Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged = te_coordinates.merge(tr_coordinates, how='left', indicator=True)\ntropic_coordinates = merged[merged['_merge']=='left_only']\ncommon_coordinates = merged[merged['_merge']=='both']","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:03:24.359441Z","iopub.status.busy":"2022-04-20T12:03:24.359089Z","iopub.status.idle":"2022-04-20T12:03:24.406464Z","shell.execute_reply":"2022-04-20T12:03:24.405299Z","shell.execute_reply.started":"2022-04-20T12:03:24.359402Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax0, ax1) = plt.subplots(1, 2, figsize=[10, 3], dpi=200, sharex=True, sharey=True)\nax0.scatter(tr_coordinates['fact_longitude'], tr_coordinates['fact_latitude'], \n            s=1, c='tab:green', label='Train data')\nax1.scatter(common_coordinates['fact_longitude'], common_coordinates['fact_latitude'], \n            s=1, c='tab:orange', label='In-domain test data')\nax1.scatter(tropic_coordinates['fact_longitude'], tropic_coordinates['fact_latitude'], \n            s=1, c='tab:red', label='Out-domain test data')\nax0.legend(), ax1.legend()\nfig.suptitle('Distribution of train and test points in the dataset', y=1.01, fontsize=14)\nplt.show()","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:13:07.842973Z","iopub.status.busy":"2022-04-20T12:13:07.842394Z","iopub.status.idle":"2022-04-20T12:13:08.666354Z","shell.execute_reply":"2022-04-20T12:13:08.665219Z","shell.execute_reply.started":"2022-04-20T12:13:07.842932Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion:\nWe have very few datasets for training from tropical zones. \nThey do appear though a lot in the test data.\nIt could be a good idea to \"predict\" values for samples in the tropical zone.\nBecause of a lack of time we did not implement such a prediction though.","metadata":{}},{"cell_type":"markdown","source":"# Availabilities","metadata":{}},{"cell_type":"code","source":"def check_availabilities(df):\n    all_cols = list(df.columns.values)\n    availability_cols = [col for col in all_cols if \"available\" in col]\n\n    for col in availability_cols:\n        print(col,\"Max:\", df[col].max(),\" Min:\", df[col].min())\n        print(\"\\t\",col,\"unavailable ratio:\",round(df[df[col] < 1].shape[0]/df.shape[0],8)*100,\"%\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train data\")\ncheck_availabilities(df)\nprint(\"\\nTest data\")\ncheck_availabilities(df_test)","metadata":{"scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conclusion \ncmc_available: Almost all train data, all test data -> Use!\n\ngfs_available: All train data, all test data -> Use!\n\ngfs_soil_temperature_available: Not available for  >16% of train data and >24% of test data -> Don't Use!\n\nwrf_available: For >5% of the train data not available, for almost all test data available -> Use!","metadata":{}},{"cell_type":"markdown","source":"# Availability Detection","metadata":{}},{"cell_type":"code","source":"def nan_column(df,tolerance,verbose=0):\n     \n    nan_col = []\n    nan_ratio = []\n    \n    original_rows = df.shape[0]\n    \n    for(colname,colval) in df.iteritems():\n        \n        nan_rows = df[colname].isna().sum() \n        ratio = nan_rows/original_rows\n\n        if ratio > tolerance:\n            \n            nan_col.append(colname)\n            nan_ratio.append(ratio)\n            if verbose:\n                print(colname,ratio)\n                \n            \n    return nan_col,nan_ratio","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train data\")\nnan_col,nan_ratio = nan_column(df,0,verbose=0)\nplt.barh(nan_col,nan_ratio)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Test data\")\nnan_col,nan_ratio = nan_column(df_test,0,verbose=0)\nplt.barh(nan_col,nan_ratio)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Train data\")\nprint(\"Min soil temperature: \",df[\"gfs_soil_temperature\"].min())\ndf_soil_missing = df[df[\"gfs_soil_temperature\"] == df[\"gfs_soil_temperature\"].min()]\nprint(\"Soil temperature is -9999.0 in:\", round(df_soil_missing.shape[0]/df.shape[0],5)*100,\"% of the rows\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Conlusion\n\nWe don't have any NaN values in the test data.\n\nMissing availabilites of cmc or wrf values in the train data are represented by NaN values.\n\n    We are going to drop those rows in the preprocessing step.\n\nMissing availabilites of gfs_soil_temperature are represented by \"-9999\" (Physically impossible temperature). \n\n    We are going to drop that column anyways though.\n","metadata":{}},{"cell_type":"markdown","source":"# Data Pre-processing\nDelete all availability columns and gfs_soil_temperature","metadata":{}},{"cell_type":"code","source":"all_cols = list(df.columns.values)\ncols_remove = [col for col in all_cols if \"available\" in col]\ncols_remove.append(\"gfs_soil_temperature\")\n\ncols_keep = [x for x in all_cols if x not in cols_remove]\ndf_filter = df[cols_keep]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Find columns that are highly correlated (positively or negatively) with the ouput column","metadata":{}},{"cell_type":"code","source":"def top_correlation_to_column(df,col,nr):\n\n    col_list = []\n    corr_list = []\n    \n    for(colname,colval) in df.iteritems():\n        corr = df[colname].corr(df['fact_temperature'])\n        corr_abs = abs(corr)\n        if not math.isnan(corr) and col != colname:\n            col_list.append(colname)\n            corr_list.append(corr)\n        \n    col_list = [col_list for _, col_list in sorted(zip(corr_list, col_list))]\n    corr_list.sort()\n    return col_list[-nr:],corr_list[-nr:]","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top_col, top_corr = top_correlation_to_column(df_filter,\"fact_temperature\",10)\nplt.barh(top_col,top_corr)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Add the top 10 most correlated columns with a number of (by the professor) preselected columns.\n\nWe tried PCA to find those correlations but it either didn't provide us with good results or it was too much for our PC to handle.","metadata":{}},{"cell_type":"code","source":"col_selection = [\n    'index', 'fact_time', 'fact_latitude', 'fact_longitude',\n    'topography_bathymetry', 'sun_elevation', 'cmc_precipitations', \n    'gfs_a_vorticity', 'gfs_cloudness', 'gfs_clouds_sea', 'gfs_humidity']\ncol_selection = col_selection + top_col+[\"fact_temperature\"]","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:14:08.77129Z","iopub.status.busy":"2022-04-20T12:14:08.770862Z","iopub.status.idle":"2022-04-20T12:14:08.776828Z","shell.execute_reply":"2022-04-20T12:14:08.775759Z","shell.execute_reply.started":"2022-04-20T12:14:08.771253Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Drop all NaN columns.\nSelect the right columns for X and y.\nStandardization will happen in the pipeline in the model selection. \nRemove fact_temperature from col_selection, to reuse it for the test data.","metadata":{}},{"cell_type":"code","source":"all_data = df[col_selection].dropna()\ncol_selection = list(all_data.columns.values)\n\nX = all_data.iloc[:, 1:-1].values\ny = all_data.iloc[:, -1].values\n\nX_tr, X_val, y_tr, y_val = train_test_split(X, y, random_state=1, test_size=10000)\n\ncol_selection_test = col_selection\ncol_selection_test.remove('fact_temperature')","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:14:12.76649Z","iopub.status.busy":"2022-04-20T12:14:12.765727Z","iopub.status.idle":"2022-04-20T12:14:13.544204Z","shell.execute_reply":"2022-04-20T12:14:13.543073Z","shell.execute_reply.started":"2022-04-20T12:14:12.76643Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Model Selection and Hyperparameters\n\nWe tried several models including Ridge Regression, SVMs and Neural Nets.\nIn the end we experienced the best results with a RandomForestRegressor though.\n\nWe tried GridSearchCV but because of the large dataset it was too much to handle for our computers.\nWe played around with the RandomForestRegressor Hyperparameters manually though.","metadata":{}},{"cell_type":"code","source":"pipeline = make_pipeline(StandardScaler(),RandomForestRegressor(n_jobs=-1, n_estimators=32, verbose=True))\n\npipeline.fit(X_tr, y_tr)\nypred_tr = pipeline.predict(X_tr)\nypred_val = pipeline.predict(X_val)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_rmse = mean_squared_error(y_tr, ypred_tr, squared=False)\nvalid_rmse = mean_squared_error(y_val, ypred_val, squared=False)\nprint(train_rmse,valid_rmse)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation\n\nSome form of pre-evaluation will inevitably be required in the preceding sections in order to both select an appropriate model and configure its parameters appropriately. In this final section, you may evaluate other aspects of the model such as:\n\n-    Assessing the running time of your model;\n-    Determining whether some aspects can be parallelised;\n-    Training the model with smaller subsets of the data.\n-    etc.\n\n- The model trains relatively fast. Predictions are very fast\n-If we had better hardware, it would have been easier to optimize the hyperparameters.\n-While our feature selection is probably not perfect, we believe that we select decent features.\n\n","metadata":{}},{"cell_type":"code","source":"X_te = df_test[col_selection_test].iloc[:, 1:].values\nypred_te = pipeline.predict(X_te)","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:16:31.298063Z","iopub.status.busy":"2022-04-20T12:16:31.29766Z","iopub.status.idle":"2022-04-20T12:16:31.326329Z","shell.execute_reply":"2022-04-20T12:16:31.32505Z","shell.execute_reply.started":"2022-04-20T12:16:31.298019Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Submission\nYour submission is a CSV file containing your final model's predictions on the given test data. This file should contain a header and have the following format:\n\n```\nindex,fact_temperature\n1993574,3.9865149124872303\n1993575,18.165092058370533\n1993576,16.53315442160854\n1993577,8.377598784006866\n...\n```\n\nA leaderboard for this challenge will be ranked using the root mean squared error between the predicted values and the observed arrival delays. However, you can use other metrics for regression tasks in your presentation notebook to evaluate many aspects of your model, including quantification of the uncertainty in the predictions.\n\nBelow is an example of creating a submission file.\n","metadata":{}},{"cell_type":"code","source":"submission_df = pd.DataFrame(data={'index': df_test['index'].values,\n                                   'fact_temperature': ypred_te.squeeze()})\n\n# Save the predictions into a csv file\n# Notice that this file should be saved under the directory `/kaggle/working` \n# so that you can download it later\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:17:06.770835Z","iopub.status.busy":"2022-04-20T12:17:06.770444Z","iopub.status.idle":"2022-04-20T12:17:08.111242Z","shell.execute_reply":"2022-04-20T12:17:08.110326Z","shell.execute_reply.started":"2022-04-20T12:17:06.770786Z"}},"execution_count":null,"outputs":[]}]}