{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AML Challenge 1 Group 2","metadata":{"id":"y-eiwFTCm-oe"}},{"cell_type":"code","source":"import pandas as pd \nimport matplotlib.pyplot as plt \nimport numpy as np \nimport seaborn as sns\nfrom tqdm.notebook import tqdm \n\nimport matplotlib\nfrom matplotlib import rc\n\n\nrc(\"font\", **{\"family\": \"sans-serif\", \"sans-serif\": \"DejaVu Sans\"})\nrc(\"figure\", **{\"dpi\": 200})\nrc(\n    \"axes\",\n    **{\"spines.right\": False, \"spines.top\": False, \"xmargin\": 0.0, \"ymargin\": 0.05}\n)","metadata":{"_cell_guid":"994dad23-6226-4b87-a0be-56f9a378ac40","_uuid":"349a1362-e369-49ae-906d-1b5f8a48475d","jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-05-10T14:21:12.243424Z","iopub.execute_input":"2022-05-10T14:21:12.243688Z","iopub.status.idle":"2022-05-10T14:21:12.249023Z","shell.execute_reply.started":"2022-05-10T14:21:12.243664Z","shell.execute_reply":"2022-05-10T14:21:12.248496Z"},"id":"vLb1QUUym-oi","collapsed":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi -L","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:21:12.252549Z","iopub.execute_input":"2022-05-10T14:21:12.252719Z","iopub.status.idle":"2022-05-10T14:21:12.548204Z","shell.execute_reply.started":"2022-05-10T14:21:12.252697Z","shell.execute_reply":"2022-05-10T14:21:12.547644Z"},"id":"74dZ1OPGm-om","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Wandb was a program used to track GPU-usage and memory\n#!pip install wandb\n#import wandb\n#wandb.init()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:21:12.549565Z","iopub.execute_input":"2022-05-10T14:21:12.549772Z","iopub.status.idle":"2022-05-10T14:21:12.552532Z","shell.execute_reply.started":"2022-05-10T14:21:12.549748Z","shell.execute_reply":"2022-05-10T14:21:12.552106Z"},"id":"rrMNzYDEm-on","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/train.csv', low_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:21:17.061263Z","iopub.execute_input":"2022-05-10T14:21:17.061501Z","iopub.status.idle":"2022-05-10T14:22:22.949956Z","shell.execute_reply.started":"2022-05-10T14:21:17.061471Z","shell.execute_reply":"2022-05-10T14:22:22.948672Z"},"id":"Ro-UUApcm-oo","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.sample(frac=1)","metadata":{"id":"E7AFPZGKm-oo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/test_feat.csv', low_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:22:22.9523Z","iopub.execute_input":"2022-05-10T14:22:22.952659Z","iopub.status.idle":"2022-05-10T14:22:39.96146Z","shell.execute_reply.started":"2022-05-10T14:22:22.952604Z","shell.execute_reply":"2022-05-10T14:22:39.960851Z"},"id":"IvzHmyvJm-op","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.max_columns', None)\npd.set_option('display.max_rows', 3000) #pd.options.display.max_rows = 4000\n","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:22:39.962874Z","iopub.execute_input":"2022-05-10T14:22:39.963291Z","iopub.status.idle":"2022-05-10T14:22:39.968611Z","shell.execute_reply.started":"2022-05-10T14:22:39.963253Z","shell.execute_reply":"2022-05-10T14:22:39.967523Z"},"id":"1f3_5w-jm-oq","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_concat = pd.concat([df, df_test], axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:22:39.970265Z","iopub.execute_input":"2022-05-10T14:22:39.970483Z","iopub.status.idle":"2022-05-10T14:22:40.384341Z","shell.execute_reply.started":"2022-05-10T14:22:39.970455Z","shell.execute_reply":"2022-05-10T14:22:40.383453Z"},"id":"9IExltNom-or","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data pre-processing \nOne of the feature engineerings we did to prepare the data was to seperate the different latitudes into \"bulks\" describing how far from equator the data point is, called eq_dist. If the datapoint has a latitude more than -5 or less than 5 it is on equator, and gets the eq_dist equal to 0. If it is not in that range, but still in the [-10,10] range it will get a value equal to 1. This goes for all the ranges down to -90 and up to 90. We did not end up using it tho, thats why its commented out. \n\nWe also reformatted the timestamp, so we had month, day and hour instead of the unix timestamp, and removed nan-values. The converstion of timestamp makes sure that the model can extract useful information like what hour of the day it is and which monthn youre in.","metadata":{"id":"x-A1y6IZm-os"}},{"cell_type":"code","source":"#import math\n#def calc_dist(val):\n#    ps = [range(0, 5), range(5,10), range(10, 15), range(15, 20), range(20, 25),range(25,30),range(30, 35),range(35,40), range(40,45), range(45,50), range(50,55), range(55,60), range(60,65), range(65,70), range(70, 75),range(75,80),range(80,85), range(85, 90)]\n#    for i in range(len(ps)): \n#        if math.ceil(abs(val)) in ps[i]:\n#            return i \n#    return 18\n#df_concat[\"eq_dist\"] = df_concat.apply(lambda x: calc_dist(val=x[\"fact_latitude\"]), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:22:40.38568Z","iopub.execute_input":"2022-05-10T14:22:40.385873Z","iopub.status.idle":"2022-05-10T14:23:20.859891Z","shell.execute_reply.started":"2022-05-10T14:22:40.385847Z","shell.execute_reply":"2022-05-10T14:23:20.85889Z"},"id":"IMZqrLnzm-ot","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Here we do the feature engineering on the timestamp value\ndf_concat['fact_time'] = pd.to_datetime(df_concat['fact_time'],unit='s')\ndf_concat['month'] = pd.DatetimeIndex(df_concat['fact_time']).month\ndf_concat['day'] = pd.DatetimeIndex(df_concat['fact_time']).day\ndf_concat['hour'] = pd.DatetimeIndex(df_concat['fact_time']).hour\nprint(df_concat.isna().sum())\n\n# Dropping CMC and WRF non-values (represented by the two columns given)\ndf_concat_nn = df_concat[df_concat[\"cmc_precipitations\"].notnull()]\ndf_concat_nn_nn = df_concat_nn[df_concat_nn[\"wrf_t2_next\"].notnull()]\nprint(df_concat_nn_nn.isna().sum())","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:23:20.861145Z","iopub.execute_input":"2022-05-10T14:23:20.861879Z","iopub.status.idle":"2022-05-10T14:23:25.134983Z","shell.execute_reply.started":"2022-05-10T14:23:20.861845Z","shell.execute_reply":"2022-05-10T14:23:25.13432Z"},"id":"KmvSCruQm-ou","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Further we want to look at the correlated columns, to see if there are columns contributing \"the same information\" (highly correlated). This way we could drop the related","metadata":{"id":"8Fc4NJg_m-ov"}},{"cell_type":"code","source":"#Preparing the training set which will be used for finding correlation \n\ndf['fact_time'] = pd.to_datetime(df['fact_time'],unit='s')\ndf['month'] = pd.DatetimeIndex(df['fact_time']).month\ndf['day'] = pd.DatetimeIndex(df['fact_time']).day\ndf['hour'] = pd.DatetimeIndex(df['fact_time']).hour","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:23:25.138232Z","iopub.execute_input":"2022-05-10T14:23:25.139602Z","iopub.status.idle":"2022-05-10T14:23:26.957929Z","shell.execute_reply.started":"2022-05-10T14:23:25.139524Z","shell.execute_reply":"2022-05-10T14:23:26.956778Z"},"id":"jm5OeXxpm-ov","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calc_most_correlated(df, corr_value): \n    corr_df = df.corr().abs()\n    filtered_corr_df = corr_df[(corr_df>=corr_value) & (corr_df != 1.000)]\n    correlated = {}\n    for index, row in filtered_corr_df.iterrows():\n        c = [filtered_corr_df.columns[i] for i in range(len(filtered_corr_df.columns)) if not np.isnan(row[i])]\n        if len(c)>0: \n            correlated[index]=c\n    corr_sort = sorted(correlated, key=lambda k: len(correlated[k]), reverse=True)\n    return corr_sort, correlated\n\n\ncorr_sort, correlated = calc_most_correlated(df, 0.95)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:23:26.959356Z","iopub.execute_input":"2022-05-10T14:23:26.959613Z","iopub.status.idle":"2022-05-10T14:24:16.715878Z","shell.execute_reply.started":"2022-05-10T14:23:26.959579Z","shell.execute_reply":"2022-05-10T14:24:16.715226Z"},"id":"l6MawoJCm-ow","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_columns_to_drop(corr_sort, correlated):\n    dropped = set()\n    for value in corr_sort: \n        if value not in dropped: \n            dropped = dropped.union({key for key in correlated[value]})\n    return dropped\n\n#corr_sort = calc_most_correlated(df, 0.95)\n#dropped = find_columns_to_drop(corr_sort, correlated)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:16.717129Z","iopub.execute_input":"2022-05-10T14:24:16.717351Z","iopub.status.idle":"2022-05-10T14:24:16.722857Z","shell.execute_reply.started":"2022-05-10T14:24:16.717322Z","shell.execute_reply":"2022-05-10T14:24:16.722037Z"},"id":"xM0SnFpzm-ox","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following commented out line of code was ment to be used to drop the not so important (correlating) columns, but we ended up using all columns.","metadata":{"id":"UH_dj1LZm-ox"}},{"cell_type":"code","source":"#df_concat_nn_nn_dropped = df_concat_nn_nn.drop(dropped)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:16.72414Z","iopub.execute_input":"2022-05-10T14:24:16.724406Z","iopub.status.idle":"2022-05-10T14:24:17.896081Z","shell.execute_reply.started":"2022-05-10T14:24:16.724355Z","shell.execute_reply":"2022-05-10T14:24:17.893183Z"},"id":"Zji5FM0sm-oy","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following code is used to create clusters as described in the powerpoint.\n","metadata":{"id":"K6dHHzxam-oy"}},{"cell_type":"code","source":"#in order for the KMEANS to run in reasonable time we had to just select the most important columns\ncol_selection = [\n    'index', 'fact_latitude', 'fact_longitude',\n    'topography_bathymetry', 'sun_elevation', 'cmc_precipitations', \n    'gfs_a_vorticity', 'gfs_cloudness', 'gfs_clouds_sea', 'gfs_humidity',\n    'fact_temperature', 'month', 'day', 'hour', 'cmc_0_0_6_2', 'cmc_0_0_7_2', 'wrf_rh2'\n]\ndf_selected = df_concat_nn_nn[col_selection]","metadata":{"id":"HmtYazyJnRcV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def lat_long_clustering(df):\n    from sklearn.cluster import KMeans\n    k_means = KMeans(n_clusters = 35, max_iter = 20, init='k-means++', random_state=42, verbose=1)\n    df['fact_latitude'] = df['fact_latitude']*3 #multiply by three to improve importance of latitude\n    lat_long_pairs = df[['fact_latitude','fact_longitude']]\n    #lat_long_pairs['fact_longitude'] = lat_long_pairs['fact_longitude']*3\n    target = (df[df['fact_temperature'].notnull()].fact_temperature)\n    \n    k_means.fit(df[df['fact_temperature'].notnull()][['fact_latitude','fact_longitude']],sample_weight = target)\n    df['lat_long_cluster'] = k_means.predict(lat_long_pairs)\n\n    return df\n\ndf_selected_clustered = lat_long_clustering(df_selected)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.897509Z","iopub.status.idle":"2022-05-10T14:24:17.897877Z","shell.execute_reply.started":"2022-05-10T14:24:17.897694Z","shell.execute_reply":"2022-05-10T14:24:17.897716Z"},"id":"gzvNQ6cNm-oz","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_concat_nn_nn[\"lat_long_cluster\"] = df_selected_clustered[\"lat_long_cluster\"]","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.898892Z","iopub.status.idle":"2022-05-10T14:24:17.899195Z","shell.execute_reply.started":"2022-05-10T14:24:17.899031Z","shell.execute_reply":"2022-05-10T14:24:17.899052Z"},"id":"HxXyYkdgm-oz","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# PCA\nCommented out becaus it was not used in the end","metadata":{"id":"paMQncXxm-o0"}},{"cell_type":"code","source":"#X = StandardScaler().fit_transform(X)\n#y = StandardScaler().fit_transform(y.reshape(-1, 1)).squeeze()\n\n#%%time\n#pca1 = PCA(n_components=1)\n#pca90 = PCA(n_components=0.9)\n#pca95 = PCA(n_components=0.95)\n#pca99 = PCA(n_components=0.99)\n#pca1.fit(X)\n#pca90.fit(X)\n##pca95.fit(X)\n#pca99.fit(X)\n\n#print(f\"PCA 90% component variance: {len(pca90.explained_variance_ratio_)}\")\n#print(f\"PCA 95% component variance: {len(pca95.explained_variance_ratio_)}\")\n#print(f\"PCA 99% component variance: {len(pca99.explained_variance_ratio_)}\")\n\n#X_1 = pca1.transform(X)\n#X_90 = pca90.transform(X)\n#X_95 = pca95.transform(X)\n#X_99 = pca99.transform(X)","metadata":{"id":"LBpGKOiZm-o0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"MOQYxvf6m-o1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_1.shape\n\n#plt.title(\"Most variant PCA feature\")\n#plt.scatter(X_1, y, s=2, c=\"#50BEA8\")\n#plt.ylabel(\"Temperature\")\n#plt.show()","metadata":{"id":"uqNivXLYm-o1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train test set","metadata":{"id":"RNljtJiZnrlX"}},{"cell_type":"code","source":"X = df_concat_nn_nn.drop(\"fact_time\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.900334Z","iopub.status.idle":"2022-05-10T14:24:17.900681Z","shell.execute_reply.started":"2022-05-10T14:24:17.900507Z","shell.execute_reply":"2022-05-10T14:24:17.900527Z"},"id":"XBD16csNm-o2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = X[X['fact_temperature'].isnull()]\ntrain = X[X['fact_temperature'].notnull()]\ntrain","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.901702Z","iopub.status.idle":"2022-05-10T14:24:17.902Z","shell.execute_reply.started":"2022-05-10T14:24:17.901836Z","shell.execute_reply":"2022-05-10T14:24:17.901856Z"},"id":"JYQ38KJqm-o2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n!{sys.executable} -m pip install xgboost\nimport xgboost as xgb","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.903118Z","iopub.status.idle":"2022-05-10T14:24:17.903444Z","shell.execute_reply.started":"2022-05-10T14:24:17.903253Z","shell.execute_reply":"2022-05-10T14:24:17.903273Z"},"id":"RdkaUh0tm-o2","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train.drop(\"fact_temperature\", axis=1)\ny = train[\"fact_temperature\"]\nX","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.905569Z","iopub.status.idle":"2022-05-10T14:24:17.90592Z","shell.execute_reply.started":"2022-05-10T14:24:17.905747Z","shell.execute_reply":"2022-05-10T14:24:17.905767Z"},"id":"1w_HfVcqm-o3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test of Ensamle-model\nWe did not end up using this, hence commented out.","metadata":{"id":"So2SjfaXm-o3"}},{"cell_type":"code","source":"SEED=42\n# model1 = RandomForestRegressor(\n#     n_estimators=1500,\n#     n_jobs=-1,\n#     random_state=SEED,\n#     verbose=1,\n# )\n# model2 = GradientBoostingRegressor(\n#     n_estimators=600,\n#     learning_rate=0.06,\n#     min_samples_leaf=4, \n#     max_depth=9, \n#     random_state=SEED,\n#     verbose=1,\n# )\n# model3 = lgb.LGBMRegressor(\n#     n_estimators=6000,\n#     learning_rate=0.08,\n#     num_leaves=10,\n#     random_state=SEED, \n#     seed=SEED,\n#     n_jobs=-1,\n# )\n# model4 = xgb.XGBRegressor(\n#     n_estimators=11000,\n#     learning_rate=0.01,\n#     n_jobs=-1, \n#     subsample=0.8,\n#     random_state=SEED,\n#     max_depth = 8,\n#     gamma=0.0,\n#     seed=SEED,\n#     verbosity=1,\n#     tree_method='gpu_hist'\n# )\n# model5 = CatBoostRegressor(\n#     n_estimators=5000,\n#     learning_rate=0.06322764426255192,\n#     thread_count=-1,\n#     num_leaves=24,\n#     min_child_samples=16,\n#     depth=6,\n#     task_type=\"GPU\",\n#     random_seed=SEED,\n#     silent=True,\n#     grow_policy='Lossguide',\n# )","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.907296Z","iopub.status.idle":"2022-05-10T14:24:17.908551Z","shell.execute_reply.started":"2022-05-10T14:24:17.90835Z","shell.execute_reply":"2022-05-10T14:24:17.908388Z"},"id":"W9GyetPWm-o4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#X_test = test.drop(\"fact_temperature\", axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.909427Z","iopub.status.idle":"2022-05-10T14:24:17.909989Z","shell.execute_reply.started":"2022-05-10T14:24:17.909807Z","shell.execute_reply":"2022-05-10T14:24:17.909831Z"},"id":"odT_bK3sm-o4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#from sklearn.model_selection import KFold","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.911212Z","iopub.status.idle":"2022-05-10T14:24:17.911559Z","shell.execute_reply.started":"2022-05-10T14:24:17.911351Z","shell.execute_reply":"2022-05-10T14:24:17.911406Z"},"id":"UMCMZDVkm-o5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ntrain = X.shape[0]\n# ntest = X_test.shape[0]\n# SEED = 42 # for reproducibility\n# NFOLDS = 3 # set number of folds for out-of-fold prediction\n# kf = KFold(\n#     n_splits=NFOLDS,\n#     shuffle=True,\n#     random_state=SEED\n# ) # K-Folds cross-validator\n\n# def get_oof(clf, x_train, y_train, x_test):\n#     \"\"\"\n#     Popular function on Kaggle.\n    \n#     Trains a classifier on 4/5 of the training data and\n#     predicts the rest (1/5). This procedure is repeated for all 5 folds,\n#     thus we have predictions for all training set. This prediction is one\n#     column of meta-data, later on used as a feature column by a meta-algorithm.\n#     We predict the test part and average predictions across all 5 models.\n    \n#     Keyword arguments:\n#     clf -- classifier\n#     x_train -- 4/5 of training data\n#     y_train -- corresponding labels\n#     x_test -- all test data\n    \n#     \"\"\"\n#     oof_train = np.zeros((ntrain,))\n#     oof_test = np.zeros((ntest,))\n#     oof_test_skf = np.empty((NFOLDS, ntest))\n\n#     for i, (train_index, test_index) in enumerate(kf.split(x_train)):\n#         x_tr = x_train[train_index]\n#         y_tr = y_train[train_index]\n#         x_te = x_train[test_index]\n\n#         clf.fit(x_tr, y_tr)\n\n#         oof_train[test_index] = clf.predict(x_te)\n#         oof_test_skf[i, :] = clf.predict(x_test)\n\n#     oof_test[:] = oof_test_skf.mean(axis=0)\n#     return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.912842Z","iopub.status.idle":"2022-05-10T14:24:17.913173Z","shell.execute_reply.started":"2022-05-10T14:24:17.912985Z","shell.execute_reply":"2022-05-10T14:24:17.91301Z"},"id":"gBle1hrsm-o5","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# X_train = X.values\n# X_test = X_test.values\n# y_train = y.ravel()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.914329Z","iopub.status.idle":"2022-05-10T14:24:17.914661Z","shell.execute_reply.started":"2022-05-10T14:24:17.914496Z","shell.execute_reply":"2022-05-10T14:24:17.914517Z"},"id":"VWsqo2Jrm-o6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lgb_oof_train, lgb_oof_test = get_oof(model3, X_train, y_train, X_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.915968Z","iopub.status.idle":"2022-05-10T14:24:17.916948Z","shell.execute_reply.started":"2022-05-10T14:24:17.916564Z","shell.execute_reply":"2022-05-10T14:24:17.916613Z"},"id":"8JCg_6qum-o6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# xgb_oof_train, xgb_oof_test = get_oof(model4, X_train, y_train, X_test) ","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.919242Z","iopub.status.idle":"2022-05-10T14:24:17.920472Z","shell.execute_reply.started":"2022-05-10T14:24:17.920191Z","shell.execute_reply":"2022-05-10T14:24:17.920219Z"},"id":"pCNfmthSm-o6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cat_oof_train, cat_oof_test = get_oof(model5, X_train, y_train, X_test)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.921617Z","iopub.status.idle":"2022-05-10T14:24:17.921983Z","shell.execute_reply.started":"2022-05-10T14:24:17.921787Z","shell.execute_reply":"2022-05-10T14:24:17.921808Z"},"id":"ePiHhxJtm-o7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# x_train = np.concatenate((\n#     lgb_oof_train,\n#     xgb_oof_train,\n#     cat_oof_train\n# ), axis=1)\n\n# x_test = np.concatenate((\n#     lgb_oof_test,\n#     xgb_oof_test,\n#     cat_oof_test\n# ), axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.923839Z","iopub.status.idle":"2022-05-10T14:24:17.924231Z","shell.execute_reply.started":"2022-05-10T14:24:17.924017Z","shell.execute_reply":"2022-05-10T14:24:17.924037Z"},"id":"aYU8Ci8sm-o7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.linear_model import RidgeCV","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.925695Z","iopub.status.idle":"2022-05-10T14:24:17.926063Z","shell.execute_reply.started":"2022-05-10T14:24:17.925866Z","shell.execute_reply":"2022-05-10T14:24:17.925887Z"},"id":"m6euutLLm-o8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# META_MODEL = RidgeCV(cv=5)\n# META_MODEL.fit(x_train, y_train)\n# final_predictions = META_MODEL.predict(x_test)\n\n# # final_predictions = np.average(\n# #     [\n# #      rf_oof_test,\n# #      gb_oof_test,\n# #      lgb_oof_test,\n# #      xgb_oof_test,\n# #      cat_oof_test\n# #     ],\n# #     weights = 1 / acc['RMSLE']**9,\n# #     axis=0\n# # )\n# # final_predictions = np.power(2, final_predictions)\n# final_predictions","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.927493Z","iopub.status.idle":"2022-05-10T14:24:17.927861Z","shell.execute_reply.started":"2022-05-10T14:24:17.927667Z","shell.execute_reply":"2022-05-10T14:24:17.927687Z"},"id":"FG6Dxek9m-o8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df = pd.DataFrame(data={'index': test['index'].values,\n#                                    'fact_temperature': final_predictions.squeeze()})\n\n# # Save the predictions into a csv file\n# # Notice that this file should be saved under the directory `/kaggle/working` \n# # so that you can download it later\n# submission_df.to_csv(\"submission_ensamble3.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.929181Z","iopub.status.idle":"2022-05-10T14:24:17.929567Z","shell.execute_reply.started":"2022-05-10T14:24:17.929345Z","shell.execute_reply":"2022-05-10T14:24:17.929365Z"},"id":"cecOR8e0m-o8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# kaggle competitions submit -c eurecom-aml-2022-challenge-1 -f submission_ensamble3.csv -m \"XGB CAT and LGBM with RidgeCV\"","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.930968Z","iopub.status.idle":"2022-05-10T14:24:17.931326Z","shell.execute_reply.started":"2022-05-10T14:24:17.931138Z","shell.execute_reply":"2022-05-10T14:24:17.931158Z"},"id":"HNk2n-eMm-o9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ! head -6 \"submission_ensamble2.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.933194Z","iopub.status.idle":"2022-05-10T14:24:17.933593Z","shell.execute_reply.started":"2022-05-10T14:24:17.933405Z","shell.execute_reply":"2022-05-10T14:24:17.933426Z"},"id":"VDfZoohum-o9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"id":"2F3TjPajm-o9"}},{"cell_type":"code","source":"# #xgb_model = xgb.XGBRegressor(n_jobs=-1, verbosity=2, max_depth=9, n_estimators=11000, learning_rate=0.01, tree_method=\"gpu_hist\", seed=42)\nxgb_model = xgb.XGBRegressor(n_jobs=-1, verbosity=2, n_estmators=6500, max_depth=10, learning_rate=0.14, min_child_wheight=30, tree_method=\"gpu_hist\", seed=42)\n# xgb_model = xgb.XGBRegressor(n_jobs=-1, verbosity=2, n_estmators=7000, max_depth=10, learning_rate=0.1, min_child_wheight=2, tree_method=\"gpu_hist\", seed=42)\n\nxgb_model.fit(X,y)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.93445Z","iopub.status.idle":"2022-05-10T14:24:17.934767Z","shell.execute_reply.started":"2022-05-10T14:24:17.934589Z","shell.execute_reply":"2022-05-10T14:24:17.934611Z"},"id":"L5VfAziEm-o-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Plots used to find feature importance of XGBoost","metadata":{"id":"3bug6nomm-o-"}},{"cell_type":"code","source":"# axes = plt. gca()\n# axes. xaxis. label. set_size(100)\n# axes. yaxis. label. set_size(100)\n# plt.barh(X.columns[:20], xgb_model.feature_importances_[:20])\n","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.93583Z","iopub.status.idle":"2022-05-10T14:24:17.936133Z","shell.execute_reply.started":"2022-05-10T14:24:17.935966Z","shell.execute_reply":"2022-05-10T14:24:17.935986Z"},"id":"5ecq-sp0m-o-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import plot_importance\nplt.rcParams[\"figure.figsize\"] = (200,200)\nplot_importance(xgb_model)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.938584Z","iopub.status.idle":"2022-05-10T14:24:17.938881Z","shell.execute_reply.started":"2022-05-10T14:24:17.938725Z","shell.execute_reply":"2022-05-10T14:24:17.938741Z"},"id":"Gs6joeHvm-o_","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prediction = xgb_model.predict(test.drop(\"fact_temperature\", axis=1))","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.939583Z","iopub.status.idle":"2022-05-10T14:24:17.939887Z","shell.execute_reply.started":"2022-05-10T14:24:17.939722Z","shell.execute_reply":"2022-05-10T14:24:17.939741Z"},"id":"PRQnLzaWm-o_","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(data={'index': df_test['index'].values,\n                                   'fact_temperature': prediction.squeeze()})\n\n# # Save the predictions into a csv file\n# # Notice that this file should be saved under the directory `/kaggle/working` \n# # so that you can download it later\nubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.941293Z","iopub.status.idle":"2022-05-10T14:24:17.941632Z","shell.execute_reply.started":"2022-05-10T14:24:17.941461Z","shell.execute_reply":"2022-05-10T14:24:17.94148Z"},"id":"FMNorNf0m-o_","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! head -6 \"submission.csv\"","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.944266Z","iopub.status.idle":"2022-05-10T14:24:17.944594Z","shell.execute_reply.started":"2022-05-10T14:24:17.944425Z","shell.execute_reply":"2022-05-10T14:24:17.944446Z"},"id":"XOE2QvORm-pA","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_coordinates = train[['fact_latitude','fact_longitude','lat_long_cluster']].drop_duplicates()\nte_coordinates = test[['fact_latitude','fact_longitude','lat_long_cluster']].drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.945412Z","iopub.status.idle":"2022-05-10T14:24:17.945722Z","shell.execute_reply.started":"2022-05-10T14:24:17.945547Z","shell.execute_reply":"2022-05-10T14:24:17.945567Z"},"id":"30SdcxxYm-pA","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax0, ax1) = plt.subplots(1, 2, figsize=[10, 3], dpi=200, sharex=True, sharey=True)\nax0.scatter(tr_coordinates['fact_longitude'], tr_coordinates['fact_latitude'], \n            s=1, label='Train data',c=tr_coordinates.lat_long_cluster, cmap='viridis' ) #c=tr_coordinates.fact_temperature\nax0.legend()\nax1.scatter(te_coordinates['fact_longitude'], te_coordinates['fact_latitude'], \n            s=1, label='Test data',c=te_coordinates.lat_long_cluster, cmap='viridis' )\nax1.legend()\nfig.suptitle('Distribution of train and test points in the dataset', y=1.01, fontsize=14)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.947146Z","iopub.status.idle":"2022-05-10T14:24:17.947472Z","shell.execute_reply.started":"2022-05-10T14:24:17.947282Z","shell.execute_reply":"2022-05-10T14:24:17.947301Z"},"id":"5oDzxRL1m-pA","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# KFold validation and Optuna Tuning","metadata":{"id":"cJfl03QPm-pA"}},{"cell_type":"code","source":"from copy import deepcopy\nfrom sklearn.model_selection import KFold \ndef compute_rmse(y, ypred, ystd=1.):\n    return np.mean((y - ypred)**2)**0.5 * ystd","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.94856Z","iopub.status.idle":"2022-05-10T14:24:17.948867Z","shell.execute_reply.started":"2022-05-10T14:24:17.948695Z","shell.execute_reply":"2022-05-10T14:24:17.948715Z"},"id":"k9_UQINPm-pB","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Following is a simple KFold used to check model performance locally.","metadata":{"id":"n28hHJpUm-pB"}},{"cell_type":"code","source":"#\n#Implementing cross validation\nk = 4\nkf = KFold(n_splits=k, shuffle=True, random_state=42)\n#new parametersxgb_model = xgb.XGBRegressor(n_jobs=-1, verbosity=2, max_depth=10, n_estimators=6500, learning_rate=0.14, min_child_weigth=30,tree_method=\"gpu_hist\", seed=42)\nmodel = xgb.XGBRegressor(n_jobs=-1, verbosity=2,  max_depth=10, n_estimators=6500, learning_rate=0.14, min_child_weigth=30, tree_method=\"gpu_hist\", seed=42)\ntarget = deepcopy(y.to_numpy())\nX = deepcopy(X)\nacc_score = []\n\nfor train_index , test_index in kf.split(X):\n    X_train_k , X_test_k = X.iloc[train_index,:],X.iloc[test_index,:]\n    y_train_k , y_test_k = target[train_index] , target[test_index]\n    \n    model.fit(X_train_k,y_train_k)\n  \n    pred_values = model.predict(X_test_k)\n    acc = compute_rmse(y=y_test_k, ypred=pred_values)\n    acc_score.append(acc)\n    \navg_acc_score = sum(acc_score)/k\nprint(avg_acc_score)\n#1.8631868495864443","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.950461Z","iopub.status.idle":"2022-05-10T14:24:17.950771Z","shell.execute_reply.started":"2022-05-10T14:24:17.950609Z","shell.execute_reply":"2022-05-10T14:24:17.950629Z"},"id":"8rQGi_bom-pB","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pip install optuna","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.952135Z","iopub.status.idle":"2022-05-10T14:24:17.952467Z","shell.execute_reply.started":"2022-05-10T14:24:17.95227Z","shell.execute_reply":"2022-05-10T14:24:17.95229Z"},"id":"Fvh-VAGrm-pC","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Following is the Optuna-code used to train XGBoost, in combination with KFold","metadata":{"id":"yiUXU8mUm-pC"}},{"cell_type":"code","source":"import optuna\nimport lightgbm as lgb\nfrom copy import deepcopy\nfrom sklearn.model_selection import KFold \ndef objective(trial,data=X,target=y):\n \n    #Loading the dataset\n\n    y = deepcopy(target.to_numpy())\n    X = deepcopy(data)\n    \n    #Implementing cross validation\n    k = 4\n    kf = KFold(n_splits=k, shuffle=True, random_state=42)\n\n    param = {\n        'n_estimators': trial.suggest_categorical('n_estmators', [5000,6500,7000,9000,10500]),\n        'max_depth': trial.suggest_categorical('max_depth', [8,10,12, 14]),\n        'learning_rate':trial.suggest_categorical('learning_rate', [0.06,0.1, 0.14, 0.3]),\n        #'min_samples_leaf':trial.suggest_categorical('min_samples_leaf', [1,2,4,6,9,12]),\n        #'num_leaves': trial.suggest_categorical('num_leaves', [4,17,21,24,27,50]),\n        #'boosting': trial.suggest_categorical('boosting', ['gbdt', 'dart']),\n        #'booster': trial.suggest_categorical('booster', ['gbtree', 'dart']),'reg:squaredlogerror',\n        #'objective': trial.suggest_categorical('objective', ['reg:squarederror']),\n        'min_child_wheight': trial.suggest_categorical('min_child_wheight', [0,0.5,2,6,15,30]),\n        'n_jobs': -1,\n        'seed': 42,\n        'tree_method': \"gpu_hist\",\n    }\n\n    model = xgb.XGBRegressor(**param)\n    \n    acc_score = []\n\n    for train_index , test_index in kf.split(X):\n        X_train_k , X_test_k = X.iloc[train_index,:],X.iloc[test_index,:]\n        y_train_k , y_test_k = y[train_index] , y[test_index]\n        \n        model.fit(X_train_k,y_train_k)\n      \n        pred_values = model.predict(X_test_k)\n        acc = compute_rmse(y=y_test_k, ypred=pred_values)\n        acc_score.append(acc)\n        \n    avg_acc_score = sum(acc_score)/k\n      \n    return avg_acc_score\n\noptuna.logging.enable_default_handler()\nstudy = optuna.create_study(direction='minimize')\nstudy.optimize(objective, n_trials=20)\nprint('Number of finished trials:', len(study.trials))\nprint('Best trial:', study.best_trial.params)\n\nstudy.trials_dataframe()\noptuna.visualization.plot_param_importances(study)\noptuna.visualization.plot_optimization_history(study)","metadata":{"execution":{"iopub.status.busy":"2022-05-10T14:24:17.953307Z","iopub.status.idle":"2022-05-10T14:24:17.953631Z","shell.execute_reply.started":"2022-05-10T14:24:17.953464Z","shell.execute_reply":"2022-05-10T14:24:17.953484Z"},"id":"bTU7Etfhm-pC","trusted":true},"execution_count":null,"outputs":[]}]}