{"metadata":{"kernelspec":{"display_name":"condadata","language":"python","name":"condadata"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.11"},"colab":{"name":"xgboost.ipynb","provenance":[]}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd \nimport matplotlib.pyplot as plt \nimport numpy as np \nimport seaborn as sns\n#from tqdm.notebook import tqdm \nfrom sklearn.cluster import KMeans\nimport matplotlib\nfrom matplotlib import rc\nfrom datetime import datetime\nfrom keras.models import Sequential\nfrom keras.layers import Dense\n\nrc(\"font\", **{\"family\": \"sans-serif\", \"sans-serif\": \"DejaVu Sans\"})\nrc(\"figure\", **{\"dpi\": 200})\nrc(\n    \"axes\",\n    **{\"spines.right\": False, \"spines.top\": False, \"xmargin\": 0.0, \"ymargin\": 0.05}\n)","metadata":{"_cell_guid":"994dad23-6226-4b87-a0be-56f9a378ac40","_uuid":"349a1362-e369-49ae-906d-1b5f8a48475d","execution":{"iopub.execute_input":"2022-04-20T12:06:00.664683Z","iopub.status.busy":"2022-04-20T12:06:00.663391Z","iopub.status.idle":"2022-04-20T12:06:00.671665Z","shell.execute_reply":"2022-04-20T12:06:00.670824Z","shell.execute_reply.started":"2022-04-20T12:06:00.664634Z"},"jupyter":{"outputs_hidden":false},"id":"sgMKWfRktZA1","collapsed":false},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('eurecom-aml-2022-challenge-1/public/train.csv', low_memory=True)","metadata":{"execution":{"iopub.execute_input":"2022-04-20T11:53:21.663111Z","iopub.status.busy":"2022-04-20T11:53:21.662828Z","iopub.status.idle":"2022-04-20T11:54:31.472034Z","shell.execute_reply":"2022-04-20T11:54:31.470628Z","shell.execute_reply.started":"2022-04-20T11:53:21.663072Z"},"id":"t9qYyIYUtZA7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('eurecom-aml-2022-challenge-1/public/test_feat.csv', low_memory=True)","metadata":{"execution":{"iopub.execute_input":"2022-04-20T11:54:31.474423Z","iopub.status.busy":"2022-04-20T11:54:31.474073Z","iopub.status.idle":"2022-04-20T11:54:48.565663Z","shell.execute_reply":"2022-04-20T11:54:48.564695Z","shell.execute_reply.started":"2022-04-20T11:54:31.474379Z"},"id":"gN6Ow1tttZA8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.execute_input":"2022-04-20T11:56:03.407116Z","iopub.status.busy":"2022-04-20T11:56:03.406761Z","iopub.status.idle":"2022-04-20T11:56:03.416506Z","shell.execute_reply":"2022-04-20T11:56:03.415657Z","shell.execute_reply.started":"2022-04-20T11:56:03.407079Z"},"id":"iNMmmqwHtZA9","outputId":"8090e11e-b841-49e1-ea1f-68de925d3b79"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data analysis\n\nIn this challenge, you are free (and encouraged) to explore in depth the data you have, you can run simple queries on the data, perform exploration and compute statistics.\n\n**NOTE**: finding the right question to ask is difficult! Don't be afraid to complement your analysis with your own questions. This can give you extra points!\n\n**NOTE 2**: the presentation quality is critical in any business-oriented data analysis. Take time to create few but informative plots, rather than endless tables! ","metadata":{"id":"ukcakbRhtZA-"}},{"cell_type":"code","source":"# remove variables\ndf=df.drop([\"cmc_available\",\"gfs_available\",\"gfs_soil_temperature_available\"], axis=1)\ndf_test=df_test.drop([\"cmc_available\",\"gfs_available\",\"gfs_soil_temperature_available\"], axis=1)","metadata":{"id":"-1n6F_gLtZBA"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dealing with missing values","metadata":{"id":"9t7JYSxRtZBB"}},{"cell_type":"code","source":"print(\"rate of missing values is \", ((len(df)-len(df.dropna()))/len(df))*100, \" percents\")","metadata":{"id":"hrusLy8KtZBC","outputId":"9431b1d4-cd5e-41af-e1b7-408d21e766e9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"miss=[]\nfor i in df.columns:\n    if df[str(i)].isnull().sum(axis = 0)>0:\n        miss.append(i)\n        print(i, \"count is \",df[str(i)].isnull().sum(axis = 0))","metadata":{"id":"DBs7AF1dtZBE","outputId":"8b455555-7605-4639-94e0-8196dfc2f716"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dealing with missing values, let's construct 3 datasets in data preprocessing:\n* all missing values deleted\n* rely on coordinate clusters to compute the mean to replace missing values (would be nice to cluster on more than coordinates like adding time for example to control for seasonality)\n* Delete columns with large number of missing values and either drop missing values or use coordinate clusters\n\nNote issue coming with deleting rows with missing values: 3 variables end up with no variance","metadata":{"id":"HivUyFQXtZBF"}},{"cell_type":"code","source":"std= df.dropna().std(0)\nremove=[]\nstdd= df.std(0)\nremoved=[]\nfor i in range(len(std)):\n    if std[i]==0:\n        remove.append(df.columns[i])\n    if stdd[i]==0:\n        removed.append(df.columns[i])","metadata":{"id":"RhVE92VutZBG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"no variance before removal of missing values. \",removed)\nprint(\"no variance after removal of missing values. \",remove) ","metadata":{"id":"wobtLFwutZBH","outputId":"c18dc470-3810-4c88-8e4a-eeb5ffdcd119"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df=df.drop([\"gfs_available\"],axis=1)\ndf_test=df_test.drop([\"gfs_available\"],axis=1)","metadata":{"id":"XbfU3kZJtZBH"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dealing with time","metadata":{"id":"3eofFBY_tZBI"}},{"cell_type":"code","source":"df['fact_time']=pd.to_datetime(df['fact_time'], unit='s')\ndf['fact_time'].sort_values()\ndf['month']=df['fact_time'].dt.month\ndf['hour']=df['fact_time'].dt.hour\ndf_test['fact_time']=pd.to_datetime(df_test['fact_time'], unit='s')\ndf_test['fact_time'].sort_values()\ndf_test['month']=df_test['fact_time'].dt.month\ndf_test['hour']=df_test['fact_time'].dt.hour","metadata":{"id":"-YZotnPjtZBI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hour wise\n# 24 times so one every 15°\ndef longitude_cluster(row):\n    ss=[-170,-155,-140,-125,-110,-95,-80,-65,-50,-35,-20,-5,10,25,40,55,70,85,100,115,130,145,160,175]\n    dist=180\n    for i in range(len(ss)):\n        if abs(ss[i]-row[\"fact_longitude\"])<dist:\n            dist=abs(ss[i]-row[\"fact_longitude\"])\n            index=i\n    return index\n    \n#df[\"longitude_unix\"]=\ndf['cluster_longitude']=df.apply(longitude_cluster, axis=1)\ndf_test['cluster_longitude']=df.apply(longitude_cluster, axis=1)","metadata":{"id":"O4pkbKjvtZBJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['hour']=df_test['hour']+df_test['cluster_longitude']\ndf_test['hour']=df_test['hour']%24\nhour=pd.get_dummies(df_test['hour'])\nmonth=pd.get_dummies(df_test['month'])\nhourandmonths_test=pd.concat([hour, month], axis=1)\n\ndf['hour']=df['hour']+df['cluster_longitude']\ndf['hour']=df['hour']%24\nhour=pd.get_dummies(df['hour'])\nmonth=pd.get_dummies(df['month'])\nhourandmonths=pd.concat([hour, month], axis=1)","metadata":{"id":"kzRU81bDtZBK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\nplt.title(\"Plot 1.a: Distribution of training dataset between hours of the day\")\nsns.histplot(df['hour'],bins=12)","metadata":{"id":"_SXRZ9sHtZBK","outputId":"d7c4bcfd-6ab3-4be6-c0e9-84b44ebb88d2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(12,5))\nplt.title(\"Plot 1.b: Distribution of testing dataset between hours of the day\")\nsns.histplot(df_test['hour'],bins=12)","metadata":{"id":"igl4tBritZBK","outputId":"95f68678-94ec-4bfb-c936-a5524472b56c"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dealing with coordinates","metadata":{"id":"x8WknJnctZBL"}},{"cell_type":"code","source":"tr_coordinates = df[['fact_latitude','fact_longitude']].drop_duplicates()\nte_coordinates = df_test[['fact_latitude','fact_longitude']].drop_duplicates()\nmerged = te_coordinates.merge(tr_coordinates, how='left', indicator=True)\ntropic_coordinates = merged[merged['_merge']=='left_only']\ncommon_coordinates = merged[merged['_merge']=='both']","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:03:23.224049Z","iopub.status.busy":"2022-04-20T12:03:23.223225Z","iopub.status.idle":"2022-04-20T12:03:23.500428Z","shell.execute_reply":"2022-04-20T12:03:23.499273Z","shell.execute_reply.started":"2022-04-20T12:03:23.223975Z"},"id":"7DW7DvirtZBL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clusterlatitude(n_clusters, df):\n    kmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(df['fact_longitude'].values.reshape(-1, 1))\n    centers=kmeans.cluster_centers_\n    centers=pd.DataFrame(centers)\n    centers['lat']=0\n    centers.columns=[\"long\",\"lat\"]    \n    return kmeans, centers\n    \nkmeans, center_long=clusterlatitude(20, df)","metadata":{"id":"Rnd0pnZJtZBL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clustercoordinates(n_clusters, df): \n    x=df.values\n    mean, std= x.mean(0), x.std(0)\n    x = (x - mean) / std\n    kkmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(x)\n    center=kkmeans.cluster_centers_\n    center=center*std+mean\n    center=pd.DataFrame(center)\n    return kkmeans, center\n\ndef clustercoordseason(n_clusters, df_real, df_dummy, dummyFactor): \n    x=df_real.values\n    mean, std= x.mean(0), x.std(0)\n    x = (x - mean) / std\n    xx=df_dummy.values\n    xx=dummyFactor*xx\n    xxx=np.concatenate((x, xx), axis=1)\n    kkmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(xxx)\n    center=kkmeans.cluster_centers_\n    center=center[:,:2]\n    center=center*std+mean\n    center=pd.DataFrame(center)\n    return kkmeans, center","metadata":{"id":"ZLR0BW0atZBM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def dist(df, clusters, centers):\n    summ=0\n    for i in range(len(df)):\n        c=clusters[i]\n        coord=centers.iloc[c,:]\n        dist=((df.iloc[i,0]-coord[0])**2+(df.iloc[i,1]-coord[1])**2)**0.5\n        summ=summ+dist\n    summ=summ/len(df)\n    return float(summ)","metadata":{"id":"ibW5Km9ntZBM"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"c=[25,50,75,100,125,150]\ndtrain=[]\ndtest=[]\nfor j in c:\n    kkkmeans, centers_coord=clustercoordinates(j, df[['fact_latitude','fact_longitude']])\n    pred=kkkmeans.predict(df[['fact_latitude','fact_longitude']])\n    distt=dist(df[['fact_latitude','fact_longitude']], pred, centers_coord)\n    dtrain.append(distt)\n    predtest=kkkmeans.predict(df_test[['fact_latitude','fact_longitude']])\n    disttest=dist(df_test[['fact_latitude','fact_longitude']], predtest, centers_coord)\n    dtest.append(disttest)","metadata":{"id":"4m88AJ04tZBM","outputId":"dc94203a-78e6-4ce4-8220-7dafa259f8f2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r=[]\nfor i in range(len(dtrain)):\n    k=(dtest[i]-dtrain[i])/dtrain[i]\n    r.append(k)","metadata":{"id":"wIs-cNtWtZBN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r # k=50 is a best out of the list","metadata":{"id":"OsowxGOKtZBN","outputId":"cc2d7aa9-550a-4ecd-bc47-e1d2234cd330"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kkkmeans, centers_coord=clustercoordinates(50, df[['fact_latitude','fact_longitude']])\nkkmeans, centersall=clustercoordseason(150, df[['fact_latitude','fact_longitude']], hourandmonths, 1)","metadata":{"id":"FWqzY0qHtZBN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss=[-170,-155,-140,-125,-110,-95,-80,-65,-50,-35,-20,-5,10,25,40,55,70,85,100,115,130,145,160,175]\nfig, (ax0, ax1) = plt.subplots(1, 2, figsize=[10, 3], dpi=200, sharex=True, sharey=True)\nax0.scatter(tr_coordinates['fact_longitude'], tr_coordinates['fact_latitude'], \n            s=1, c='tab:green', label='clusters')\nax0.scatter(np.array(ss), 0*np.array(ss), \n            s=6, c='tab:blue', label='Train data')\nax1.scatter(common_coordinates['fact_longitude'], common_coordinates['fact_latitude'], \n            s=1, c='tab:orange', label='In-domain test data')\nax1.scatter(tropic_coordinates['fact_longitude'], tropic_coordinates['fact_latitude'], \n            s=1, c='tab:red', label='Out-domain test data')\nax1.scatter(np.array(ss), 0*np.array(ss), \n            s=6, c='tab:blue', label='clusters')\nax0.legend(), ax1.legend()\nfig.suptitle('Plot 2.a: Distribution of train and test points in the dataset with clusters distributed evenly longitude wise', y=1.01, fontsize=14)\nplt.show()","metadata":{"id":"hkqoVxtetZBO","outputId":"9f244922-fa10-4a53-9b6d-7ae74aa0dc8c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax0, ax1) = plt.subplots(1, 2, figsize=[10, 3], dpi=200, sharex=True, sharey=True)\nax0.scatter(tr_coordinates['fact_longitude'], tr_coordinates['fact_latitude'], \n            s=1, c='tab:green', label='clusters')\nax0.scatter(center_long['long'], center_long['lat'], \n            s=6, c='tab:blue', label='Train data')\nax1.scatter(common_coordinates['fact_longitude'], common_coordinates['fact_latitude'], \n            s=1, c='tab:orange', label='In-domain test data')\nax1.scatter(tropic_coordinates['fact_longitude'], tropic_coordinates['fact_latitude'], \n            s=1, c='tab:red', label='Out-domain test data')\nax1.scatter(center_long['long'], center_long['lat'], \n            s=6, c='tab:blue', label='clusters')\nax0.legend(), ax1.legend()\nfig.suptitle('Plot 2.b: Distribution of train and test points in the dataset with clusters distributed with k-means', y=1.01, fontsize=14)\nplt.show()","metadata":{"id":"BTJJ7OSKtZBO","outputId":"b369cb07-d392-457a-db57-ea27b7861600"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax0, ax1) = plt.subplots(1, 2, figsize=[10, 3], dpi=200, sharex=True, sharey=True)\nax0.scatter(tr_coordinates['fact_longitude'], tr_coordinates['fact_latitude'], \n            s=1, c='tab:green', label='Train data')\nax0.scatter(centers_coord.iloc[:,1], centers_coord.iloc[:,0], \n            s=6, c='tab:red', label='clusters')\nax1.scatter(common_coordinates['fact_longitude'], common_coordinates['fact_latitude'], \n            s=1, c='tab:orange', label='In-domain test data')\nax1.scatter(tropic_coordinates['fact_longitude'], tropic_coordinates['fact_latitude'], \n            s=1, c='tab:red', label='Out-domain test data')\nax1.scatter(centers_coord.iloc[:,1], centers_coord.iloc[:,0], \n            s=6, c='tab:green', label='clusters')\nax0.legend(), ax1.legend()\nfig.suptitle('Plot 3:Distribution of train and test points in the dataset along with coordinate clusters', y=1.01, fontsize=14)\nplt.show()","metadata":{"id":"LlrJhmF0tZBO","outputId":"a1f82ad5-0d09-47c5-e584-095de6bc18a1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Compute the datasets for missing values","metadata":{"id":"1dhLVDWwtZBP"}},{"cell_type":"code","source":"bigmiss=[\"wrf_t2_next\", \"wrf_psfc\", \"wrf_rh2\", \"wrf_wind_u\", \"wrf_wind_v\", \"wrf_rain\", \"wrf_snow\", \"wrf_graupel\", \"wrf_t2_interpolated\", \"wrf_t2_grad\"]\ncol=[]\nfor i in df.columns:\n    col.append(i)\nnewcols=col\nnewcols.remove(\"index\")\nfor i in bigmiss:\n    newcols.remove(str(i))\nsmallmiss=miss\nfor i in bigmiss:\n    smallmiss.remove(str(i))","metadata":{"id":"usFsn4T9tZBP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prr=df[['fact_latitude','fact_longitude']]\nx=prr.values\nmean, std= x.mean(0), x.std(0)\nx = (x - mean) / std\nprr=pd.DataFrame(x)\npredcoordclusters=kkkmeans.predict(prr)\ndf[\"long_k-mean\"]=predcoordclusters\n\nprr=df_test[['fact_latitude','fact_longitude']]\nx=prr.values\nmean, std= x.mean(0), x.std(0)\nx = (x - mean) / std\nprr=pd.DataFrame(x)\npredcoordclusterstest=kkkmeans.predict(prr)\ndf_test[\"long_k-mean\"]=predcoordclusterstest","metadata":{"id":"OhO-_4twtZBP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df1=df.copy()\ndf1[\"predict\"]=predcoordclusters\ndf1=df.dropna()\n\ndf1_test=df_test.copy()\ndf1_test[\"predict\"]=predcoordclusterstest\ndf1_test=df_test.dropna()","metadata":{"id":"xQ-AusG5tZBP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"prr=df[['fact_latitude','fact_longitude']]\nx=prr.values\nmean, std= x.mean(0), x.std(0)\nx = (x - mean) / std\nprr=pd.DataFrame(x)\npr=pd.concat([prr, hourandmonths], axis=1)   \npredcoordclusters=kkmeans.predict(pr)\n\n\ndf2=df.copy()\ndf2['predict']=predcoordclusters\nstore={}\nfor i in range(df2['predict'].nunique()):  # store df per clusters in dict store\n    name_=str(i)\n    d=df2[df2['predict']==i]\n    store[name_]=d\n\nstore_means={}\nfor i in store.keys():  # store mean for each columns in each clusters in store_means dictionary\n    store_means[i]=[\"blabla\"]\n    for ii in store[i].columns:\n        store_means[i].append([ii, store[i][ii].mean()])\n\nfff=df2[df2[smallmiss].isnull().any(axis=1)]  # compute the mean for each cluster for variables with small number of missing values\nfor i in fff.index:\n    for j in smallmiss:\n        if pd.isnull(fff.loc[i,j])==True:\n            clus=df2.iloc[i,-1]\n            m=store_means[str(clus)]\n            for mm in m:\n                if mm[0]==j:\n                    df2.loc[i,j]=mm[1]\ndf2_small=df2.copy()\ndf2_small=df2_small.drop(bigmiss, axis=1)\n\nfff=df2[df2[bigmiss].isnull().any(axis=1)]# compute the mean for each cluster for variables with large number of missing values\n\nfor i in fff.index:\n    for j in bigmiss:\n        if pd.isnull(fff.loc[i,j])==True:\n            clus=df2.iloc[i,-1]\n            m=store_means[str(clus)]\n            for mm in m:\n                if mm[0]==j:\n                    df2.loc[i,j]=mm[1]\ndf2_large=df2.copy()","metadata":{"id":"CP-zmJhFtZBQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"predict\"]=predcoordclusters\ndf3=df.drop(bigmiss, axis=1)\ndf3_drop=df3.dropna()\n","metadata":{"id":"onzAH3wQtZBQ"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"datasets for missing values:\n* df1: all dropped\n* df2_small: variables with a lot of missing values dropped _ remaining completed using cluster averages\n* df2_large: all dataset completed using using cluster averages\n* df3_drop: variables with a lot of missing values dropped - remaining rows with missing values dropped","metadata":{"id":"Prui3B-4tZBQ"}},{"cell_type":"code","source":"df1.to_csv('df1.csv')\ndf2_small.to_csv('df2_small.csv')\ndf2_large.to_csv('df2_large.csv')\ndf3_drop.to_csv('df3_drop.csv')\ndf_test.to_csv('df_test.csv')","metadata":{"id":"pgA6O1WytZBR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"0Docvl03tZBR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Choose dataset and start from here","metadata":{"id":"sTugRNuMtZBR"}},{"cell_type":"code","source":"df = pd.read_csv('df2_large.csv', low_memory=True)\ndf_test = pd.read_csv('df_test.csv', low_memory=True)","metadata":{"id":"Z2q-MttStZBR"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Processing","metadata":{"id":"-Y5b_HQmtZBR"}},{"cell_type":"code","source":"def create_dummies(df):\n    hour_=pd.get_dummies(df['hour']).values\n    month_=pd.get_dummies(df['month']).values\n    dummies=np.concatenate((hour_,month_),axis=1)\n    return dummies\n\ndummies=create_dummies(df)\ndummiestest=create_dummies(df_test)","metadata":{"id":"ju3bLqfitZBR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def geo_dummy(df,df_test):\n    l=len(df)\n    tr=df['long_k-mean'].values\n    te=df_test['long_k-mean'].values\n    d=np.concatenate((tr,te),axis=0)\n    d=pd.DataFrame(d)\n    d=pd.get_dummies(d)\n    d=d.values\n    train=d[:l,:]\n    test=d[l:,:]\n    return train, test\ngeotrain, geotest=geo_dummy(df,df_test)\ndummies=np.concatenate((dummies,geotrain),axis=1)\ndummiestest=np.concatenate((dummiestest,geotest),axis=1)","metadata":{"id":"nJP_qGwWtZBS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ndef Xdatasmalltesting(df, dummies, r):\n    df[\"fact_latitude_exp\"]=df[\"fact_latitude\"]**2\n    y=df[\"fact_temperature\"]\n    df=df.drop([\"Unnamed: 0\",\"fact_time\", \"month\",\"hour\",\"cluster_longitude\",\"long_k-mean\",\"predict\",\"fact_temperature\"],axis=1)\n    x=df.values\n    Xmean, Xstd, ymean, ystd = x.mean(0), x.std(0), y.mean(), y.std()\n    x = (x - Xmean) / Xstd\n    y = (y - ymean) / ystd\n    X=np.concatenate((x,dummies),axis=1)\n    Xtr, Xval, ytr, yval = train_test_split(X, y, random_state=1, test_size=round(r*len(df)))\n    \n    return Xtr, ytr, Xval, yval, ymean, ystd, Xmean, Xstd\n\ndef Xdatatesting(test, dummiestest, Xmean, Xstd):\n    test[\"fact_latitude_exp\"]=test[\"fact_latitude\"]**2\n    test=test.drop([\"Unnamed: 0\",\"fact_time\", \"month\",\"hour\",\"cluster_longitude\",\"long_k-mean\",\"predict\"],axis=1)\n    x=test.values\n    x = (x - Xmean) / Xstd\n    Xtest=np.concatenate((x,dummiestest),axis=1)\n    \n    return Xtest\n\n","metadata":{"id":"f6Mv3O5stZBS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xtrain, ytrain, Xtest, ytest, ymean, ystd, xmean, xstd=Xdatasmalltesting(df, dummies, 0.1)\nXTEST=Xdatatesting(df_test, dummiestest, xmean, xstd)","metadata":{"id":"x1BoF1iDtZBS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XTEST.shape","metadata":{"id":"dFETr3-mtZBS","outputId":"4f98cbb1-5f67-439f-e266-3be8b0e70c98"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Xtrain.shape","metadata":{"id":"CNBHxOAytZBS","outputId":"b1efadf8-5f4c-45ff-897b-35f2bcdc4283"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#compute or not\nfrom sklearn.decomposition import PCA\npca = PCA(n_components = 0.96)\npca.fit(Xtrain)\nXtr=pca.transform(Xtrain)\nXval=pca.transform(Xtest)\nXtst=pca.transform(XTEST)","metadata":{"id":"9tsMCPHZtZBT"}},{"cell_type":"code","source":"print(ymean,ystd)","metadata":{"id":"ByMS6SZGtZBT","outputId":"1f1a2b8b-895d-4962-f206-99c1df4e2dc6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"XTEST.shape","metadata":{"id":"sAs3HvUttZBT","outputId":"6ae18bdc-6904-4691-bd4c-efa9ee458a49"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"model = Sequential()\nmodel.add(Dense(80, input_dim=53, kernel_initializer='normal',activation=\"relu\"))\nmodel.add(Dense(35, kernel_initializer='normal', activation=\"relu\"))\nmodel.add(Dense(1, kernel_initializer='normal'))\nmodel.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])\nmodel.summary()\n\nmodel.fit(Xtr,ytrain, batch_size=128, epochs=100, shuffle=True, validation_data=(Xval,ytest), verbose=True, use_multiprocessing=True)\n\nypred_tr = model.predict(Xtr)\nypred_val = model.predict(Xval)\n\nprint(f'Train RMSE: {compute_rmse(ytr, ypred_tr, ystd):.3f}')\nprint(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')","metadata":{"id":"xuhs-dVStZBT"}},{"cell_type":"markdown","source":"# Tuning","metadata":{"id":"J8a2wIeqtZBT"}},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV, GridSearchCV","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:14:55.071372Z","iopub.status.busy":"2022-04-20T12:14:55.071037Z","iopub.status.idle":"2022-04-20T12:14:55.07656Z","shell.execute_reply":"2022-04-20T12:14:55.07556Z","shell.execute_reply.started":"2022-04-20T12:14:55.07134Z"},"id":"z-6J499QtZBT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBRegressor\n\nmodel = XGBRegressor()\n\nparameters = {'n_estimators': [30], 'max_depth': [20,30], 'eta': [0.1], 'subsample': [0.7]}","metadata":{"id":"HE31QjQRtZBU"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"grid = GridSearchCV(estimator=model, param_grid = parameters, cv = 2, n_jobs=-1)\ngrid.fit(Xtrain, ytrain)\n\nprint(\" Results from Grid Search \" )\nprint(\"\\n The best estimator across ALL searched params:\\n\", grid.best_estimator_)\nprint(\"\\n The best score across ALL searched params:\\n\", grid.best_score_)\nprint(\"\\n The best parameters across ALL searched params:\\n\", grid.best_params_)","metadata":{"id":"A1O0HksGtZBU","outputId":"9efc4db6-3db5-4db5-8c0a-421918bce3b0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compute_rmse(y, ypred, ystd=1.):\n    return np.mean((y - ypred)**2)**0.5 * ystd\n    \nyreg_tr = grid.predict(Xtrain)\nyreg_val= grid.predict(Xtest)\n\nprint(f'Train RMSE: {compute_rmse(ytrain, yreg_tr, ystd):.10f}')\nprint(f'Valid RMSE: {compute_rmse(ytest, yreg_val, ystd):.10f}')","metadata":{"id":"vwizfAkWtZBU","outputId":"181fced0-cf67-45c8-8d2d-db41e0552735"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model Evaluation\n\nSome form of pre-evaluation will inevitably be required in the preceding sections in order to both select an appropriate model and configure its parameters appropriately. In this final section, you may evaluate other aspects of the model such as:\n\n-    Assessing the running time of your model;\n-    Determining whether some aspects can be parallelised;\n-    Training the model with smaller subsets of the data.\n-    etc.\n\nRemember, the goal of this challenge is to construct a model for predicting the temperature around the globe.\n\n","metadata":{"id":"1xqgEmnstZBU"}},{"cell_type":"code","source":"ypred_te = grid.predict(XTEST) * ystd + ymean  # Remember to un-standardize the predictions","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:16:31.298063Z","iopub.status.busy":"2022-04-20T12:16:31.29766Z","iopub.status.idle":"2022-04-20T12:16:31.326329Z","shell.execute_reply":"2022-04-20T12:16:31.32505Z","shell.execute_reply.started":"2022-04-20T12:16:31.298019Z"},"id":"9cZVZL8XtZBU"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Submission\nYour submission is a CSV file containing your final model's predictions on the given test data. This file should contain a header and have the following format:\n\n```\nindex,fact_temperature\n1993574,3.9865149124872303\n1993575,18.165092058370533\n1993576,16.53315442160854\n1993577,8.377598784006866\n...\n```\n\nA leaderboard for this challenge will be ranked using the root mean squared error between the predicted values and the observed arrival delays. However, you can use other metrics for regression tasks in your presentation notebook to evaluate many aspects of your model, including quantification of the uncertainty in the predictions.\n\nBelow is an example of creating a submission file.\n","metadata":{"id":"kRichlfetZBV"}},{"cell_type":"code","source":"submission_df = pd.DataFrame(data={'index': df_test['index'].values,\n                                   'fact_temperature': ypred_te.squeeze()})\n\n# Save the predictions into a csv file\n# Notice that this file should be saved under the directory `/kaggle/working` \n# so that you can download it later\nsubmission_df.to_csv(\"kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:17:06.770835Z","iopub.status.busy":"2022-04-20T12:17:06.770444Z","iopub.status.idle":"2022-04-20T12:17:08.111242Z","shell.execute_reply":"2022-04-20T12:17:08.110326Z","shell.execute_reply.started":"2022-04-20T12:17:06.770786Z"},"id":"cZ2HvnzvtZBV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the submission file\n! head -6 \"kaggle/working/submission.csv\"","metadata":{"execution":{"iopub.execute_input":"2022-04-20T12:17:42.391091Z","iopub.status.busy":"2022-04-20T12:17:42.390138Z","iopub.status.idle":"2022-04-20T12:17:43.274662Z","shell.execute_reply":"2022-04-20T12:17:43.273299Z","shell.execute_reply.started":"2022-04-20T12:17:42.391019Z"},"id":"_95zSR4otZBV","outputId":"04990c52-2310-44db-e977-381f1ffbca86"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"N7_7F19jtZBV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"xPRMQvf2tZBV"},"execution_count":null,"outputs":[]}]}