{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:50:55.986807Z","iopub.execute_input":"2022-05-11T23:50:55.987564Z","iopub.status.idle":"2022-05-11T23:50:56.024963Z","shell.execute_reply.started":"2022-05-11T23:50:55.98747Z","shell.execute_reply":"2022-05-11T23:50:56.024117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt \nimport seaborn as sns\nfrom tqdm.notebook import tqdm \n\nimport matplotlib\nfrom matplotlib import rc\n\nrc(\"font\", **{\"family\": \"sans-serif\", \"sans-serif\": \"DejaVu Sans\"})\nrc(\"figure\", **{\"dpi\": 200})\nrc(\n    \"axes\",\n    **{\"spines.right\": False, \"spines.top\": False, \"xmargin\": 0.0, \"ymargin\": 0.05}\n)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:50:56.02672Z","iopub.execute_input":"2022-05-11T23:50:56.027265Z","iopub.status.idle":"2022-05-11T23:50:56.957932Z","shell.execute_reply.started":"2022-05-11T23:50:56.027225Z","shell.execute_reply":"2022-05-11T23:50:56.957221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/train.csv', low_memory=True)\ndf_test = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/test_feat.csv', low_memory=True)\ndf.head()\ndf.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T23:50:56.959219Z","iopub.execute_input":"2022-05-11T23:50:56.95945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"DATA ANALYSIS : ","metadata":{}},{"cell_type":"code","source":"##What treshold take to reduce the number of features, regarding the correlation with fact_temperature ?\n\n#import matplotlib.pyplot as plt\n\n#df__=df.sample(1000)\n#X=[n*0.01 for n in range(100,0,-1)]\n#Y=[len(corr_df(df__,x).columns) for x in X]\n#plt.figure()\n#plt.xlabel('Correlation treshold')\n#plt.ylabel('Number of features')\n#plt.title('What treshold take to reduce the number of features, regarding the correlation with fact_temperature ?')\n#plt.plot(X,Y)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##What treshold take to reduce the number of features, regarding the correlation between features?\n\n#import matplotlib.pyplot as plt\n\n#df__=df.sample(1000)\n#X=[n*0.01 for n in range(100,0,-1)]\n#Z=[len(corr_df(df__,x).columns) for x in X]\n#plt.figure()\n#plt.xlabel('Correlation treshold')\n#plt.ylabel('Number of features')\n#plt.title('What treshold take to reduce the number of features, regarding the correlation between features?')\n#plt.plot(X,Z)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df.plot.scatter(x='fact_longitude',y='fact_latitude',c='climate_temperature')\n#df.plot.scatter(x='fact_longitude',y='fact_latitude',c='fact_temperature')\n#ax0.legend(), ax1.legend()\n#fig.suptitle('Distribution of fact and climate temperature of the training dataset', y=1.01, fontsize=14)\n#plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Adding a month and an hemisphere feature\n\n#import datetime\n\n#df_temp=df\n#df_temp['fact_time']=df_temp['fact_time'].apply(datetime.date.fromtimestamp)\n\n#df['month']=0\n#df['hemisphere']=0\n#for i in range (df['month'].count()): \n#    df['month'].iloc[i]=df_temp['fact_time'].iloc[i].month\n#    if df['fact_latitude'].iloc[i]<0: \n#        df['hemisphere'].iloc[i]=1","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to do this process with 10% of the data for rough faster testing\n#df_real=df\n#df=df.drop(index=range(1800000))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#corr_df = df.corr(method='pearson')\n#corr_df.style.background_gradient(cmap='coolwarm')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_coordinates = df[['fact_latitude','fact_longitude']].drop_duplicates()\nte_coordinates = df_test[['fact_latitude','fact_longitude']].drop_duplicates()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged = te_coordinates.merge(tr_coordinates, how='left', indicator=True)\ntropic_coordinates = merged[merged['_merge']=='left_only']\ncommon_coordinates = merged[merged['_merge']=='both']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#dropping attributes ending with 'available' because they are not usefull anymore\nto_drop =['cmc_available','gfs_soil_temperature_available','wrf_available','gfs_available']\ndf=df.drop(columns=to_drop, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# handling the missing data and replace missing values with nan from numpy and replace with mean of all the other values\nfrom sklearn.impute import SimpleImputer\narr_df=df.values\ndf_ind=df.index\ndf_col=df.columns\nimputer = SimpleImputer(missing_values=np.nan, strategy='mean') \nn_col = arr_df.shape[1]\nimputer = imputer.fit(arr_df[:,1:n_col-1])\narr_df[:,1:n_col-1] = imputer.transform(arr_df[:,1:n_col-1])\n\ndf= pd.DataFrame(arr_df, columns = df_col, index = df_ind)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"DATA PRE-PROCESSING","metadata":{}},{"cell_type":"code","source":"df.dropna(inplace=True)\n#print(df.shape[0])\n#print(df.shape[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#corrdf = df.corr(method='pearson')\n#corrdf.style.background_gradient(cmap='coolwarm')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr_fact_temp(df, corr_val):\n    x=df\n    # Creates Correlation Matrix and Instantiates\n    corr_matrix = x.corr()\n    iters = range(len(corr_matrix.columns)-1)\n    drop_cols = []\n    corr_arr=corr_matrix.values\n    # Iterates through Correlation Matrix Table to find correlated columns\n    for i in iters:\n        val = corr_arr[i,-1]\n        if abs(val) <= corr_val:\n            drop_cols.append(i)\n\n    drop_cols.sort()\n    col_temp=[]\n    #correlated columns\n    for i in range(len(drop_cols)):\n        col_temp.append(df.columns[drop_cols[i]])\n    return col_temp","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_temp = corr_fact_temp(df, 0.4)\n\nprint(len(col_temp))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_temp=df.drop(columns=col_temp, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#corr_df1 = df_temp.corr(method='pearson')\n#corr_df1.style.background_gradient(cmap='coolwarm')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr_df(df, corr_val):\n    '''\n    Obj: Drops features that are strongly correlated to other features.\n    Inputs:\n          df: features df (x)\n          corr_val: Columns are dropped relative to the corr_val input \n    Output: df that only includes uncorrelated features\n    '''\n    #we need to drop the targeted value first \n    x=df.drop('fact_temperature', axis=1, inplace = False)\n    \n    # Creates Correlation Matrix and Instantiates\n    corr_matrix = x.corr()\n    iters = range(len(corr_matrix.columns))\n    drop_cols = []\n    corr_arr=corr_matrix.values\n    # Iterates through Correlation Matrix Table to find correlated columns\n    for i in iters:\n        for j in range(i):\n            val = corr_arr[i,j]\n            if abs(val) >= corr_val:\n                drop_cols.append(i)\n\n    drop_cols.sort()\n    #correlated columns\n    col=[]\n    for i in range(len(drop_cols)):\n        col.append(df.columns[drop_cols[i]])\n    return col","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col = corr_df(df_temp, 0.9)\n\nprint(len(col))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_=df_temp.drop(columns=col, axis=1)\n#print(df_.shape[0])\n#print(df_.shape[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df_.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#corr_df2 = df_.corr(method='pearson')\n#corr_df2.style.background_gradient(cmap='coolwarm')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = df_.dropna().iloc[:, 1:-1].values\ny = df_.dropna().iloc[:, -1].values\nprint(df_)\n\nXmean, Xstd, ymean, ystd = X.mean(0), X.std(0), y.mean(), y.std()\nX = (X - Xmean) / Xstd\ny = (y - ymean) / ystd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MODEL SELECTION","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nXtr, Xval, ytr, yval = train_test_split(X, y, random_state=1, test_size=10000)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import Lasso\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.tree import DecisionTreeRegressor\nfrom sklearn.gaussian_process import GaussianProcessRegressor\nfrom sklearn.gaussian_process.kernels import DotProduct, WhiteKernel\nfrom sklearn.neighbors import KNeighborsRegressor\nfrom sklearn.svm import SVR\nfrom sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor, AdaBoostRegressor\n\n\nlist_of_models =[\"lin_reg\",\"lasso_reg\",\"reg_tree\",\"gaussian_process_regressor\",\"knn_reg\",\"svr\",\"random_forest\",\"gradient_boosting_reg\",\"ada_boosting_reg\"]\n\nchosen_model = list_of_models[4]\n\ndef compute_rmse(y, ypred, ystd=1.):\n    return np.mean((y - ypred)**2)**0.5 * ystd\n\n\nif chosen_model == \"lin_reg\":\n    model = LinearRegression()\nelif chosen_model == \"lasso_reg\":\n    model = Lasso(alpha=0.005)\nelif chosen_model == \"reg_tree\":\n    model = DecisionTreeRegressor(max_depth=9)\nelif chosen_model == \"gaussian_process_regressor\":\n    kernel = DotProduct() + WhiteKernel()\n    model = GaussianProcessRegressor(kernel=kernel, random_state=0)\nelif chosen_model == \"knn_reg\":\n    model = KNeighborsRegressor(n_neighbors=6, weights='distance')\nelif chosen_model == \"svr\":\n    model = SVR(kernel=\"poly\", C=100, gamma=\"auto\", degree=3, epsilon=0.1, coef0=1)\nelif chosen_model == \"random_forest\":\n    model = RandomForestRegressor()\nelif chosen_model == \"gradient_boosting_reg\":\n    model = GradientBoostingRegressor()\nelif chosen_model == \"ada_boosting_reg\":\n    model = AdaBoostRegressor()\n\n\nmodel.fit(Xtr, ytr)\nypred_tr = model.predict(Xtr)\nypred_val = model.predict(Xval)\n    \nprint(chosen_model)\nprint(f'Train RMSE: {compute_rmse(ytr, ypred_tr, ystd):.3f}')\nprint(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')\nprint()\n    \n    ","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"PARAMETER OPTIMISATION","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import RandomizedSearchCV, GridSearchCV","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\n\ndef get_cv_idx(n, test_size=0.2, n_splits=2):\n    train_idx, test_idx = [], []\n    for _ in range(n_splits):\n        idx = np.random.permutation(n)\n        train_size = int(n * (1 - test_size)) if isinstance(test_size, float) else n - test_size\n        train_idx.append(idx[:train_size])\n        test_idx.append(idx[train_size:])\n    return train_idx, test_idx","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_idx, cv_idx = get_cv_idx(len(Xtr), test_size=10000, n_splits=10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_of_models =[\"lin_reg\",\"lasso_reg\",\"reg_tree\",\"gaussian_process_regressor\",\"knn_reg\",\"svr\",\"random_forest\",\"gradient_boosting_reg\",\"ada_boosting_reg\"]\nmodels_test_hyp = [\"lasso_reg\",\"reg_tree\",\"knn_reg\",\"random_forest\",\"gradient_boosting_reg\"]\n\n\n\n\nparam_grid = {\n        \"n_neighbors\": [ 4, 6, 8, 10]\n    }\n\n\n\n\n\nsearch = GridSearchCV(model, \n                      param_grid,\n                      n_jobs=-1, \n                      verbose=1,\n                      cv=zip(train_idx, cv_idx), \n                      scoring='neg_root_mean_squared_error').fit(Xtr, ytr)\nprint('Done!')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Best parameters set found on cv set:\")\nprint(search.best_params_)\nprint()\nprint(\"Grid scores on cv set:\")\nmeans = search.cv_results_[\"mean_test_score\"]\nstds = search.cv_results_[\"std_test_score\"]\nfor mean, std, params in zip(means, stds, search.cv_results_[\"params\"]):\n    print(\"%0.3f (+/-%0.03f) for %r\" % (-mean * ystd, (std * ystd) * 2, params))\nprint()\nprint(\"Error on the validation set\")\nypred_val = search.predict(Xval)\nprint(f'Valid RMSE: {compute_rmse(yval, ypred_val, ystd):.3f}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"MODEL EVALUATION","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/test_feat.csv', low_memory=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"col_selection =df_.columns\n\ncol_selection=col_selection[:-1]\n\n##df_test.head()\n\nXte = df_test[col_selection].iloc[:, 1:].values\nXte = (Xte - Xmean) / Xstd","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ypred_te = search.predict(Xte) * ystd + ymean  # Remember to un-standardize the predictions\ndf_test.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"SUBMISSION","metadata":{}},{"cell_type":"code","source":"submission_df = pd.DataFrame(data={'index': df_test['index'].values,\n                                   'fact_temperature': ypred_te.squeeze()})\n\n# Save the predictions into a csv file\n# Notice that this file should be saved under the directory `/kaggle/working` \n# so that you can download it later\nsubmission_df.to_csv(\"/kaggle/working/submission2.csv\", index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the submission file\n! head -6 \"/kaggle/working/submission2.csv\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}