{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd \nimport matplotlib.pyplot as plt \nimport numpy as np \nimport seaborn as sns\n\nfrom tqdm.notebook import tqdm \n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import LogisticRegression, Ridge\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler, RobustScaler\nfrom sklearn.feature_selection import VarianceThreshold, chi2, SelectKBest\nfrom sklearn.feature_selection import SelectFromModel, RFECV\n\nfrom sklearn.linear_model import Lasso\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.tree import DecisionTreeRegressor\n\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nimport matplotlib\nfrom matplotlib import rc","metadata":{"_uuid":"349a1362-e369-49ae-906d-1b5f8a48475d","_cell_guid":"994dad23-6226-4b87-a0be-56f9a378ac40","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-05-11T20:03:06.969135Z","iopub.execute_input":"2022-05-11T20:03:06.969375Z","iopub.status.idle":"2022-05-11T20:03:06.977742Z","shell.execute_reply.started":"2022-05-11T20:03:06.969347Z","shell.execute_reply":"2022-05-11T20:03:06.976640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/train.csv', low_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:03:14.070358Z","iopub.execute_input":"2022-05-11T20:03:14.070726Z","iopub.status.idle":"2022-05-11T20:04:30.848808Z","shell.execute_reply.started":"2022-05-11T20:03:14.070689Z","shell.execute_reply":"2022-05-11T20:04:30.847704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/test_feat.csv', low_memory=True)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:04:36.029227Z","iopub.execute_input":"2022-05-11T20:04:36.029653Z","iopub.status.idle":"2022-05-11T20:04:56.665103Z","shell.execute_reply.started":"2022-05-11T20:04:36.029596Z","shell.execute_reply":"2022-05-11T20:04:56.664135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Discovery of the dataset","metadata":{}},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T21:03:00.691271Z","iopub.execute_input":"2022-05-11T21:03:00.691654Z","iopub.status.idle":"2022-05-11T21:03:00.732669Z","shell.execute_reply.started":"2022-05-11T21:03:00.691619Z","shell.execute_reply":"2022-05-11T21:03:00.731699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T21:03:03.468005Z","iopub.execute_input":"2022-05-11T21:03:03.468371Z","iopub.status.idle":"2022-05-11T21:03:13.160418Z","shell.execute_reply.started":"2022-05-11T21:03:03.468326Z","shell.execute_reply":"2022-05-11T21:03:13.159503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tr_coordinates = df[['fact_latitude','fact_longitude']].drop_duplicates()\nte_coordinates = df_test[['fact_latitude','fact_longitude']].drop_duplicates()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:01:58.857192Z","iopub.execute_input":"2022-05-11T13:01:58.857464Z","iopub.status.idle":"2022-05-11T13:01:59.030235Z","shell.execute_reply.started":"2022-05-11T13:01:58.857435Z","shell.execute_reply":"2022-05-11T13:01:59.029416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged = te_coordinates.merge(tr_coordinates, how='left', indicator=True)\ntropic_coordinates = merged[merged['_merge']=='left_only']\ncommon_coordinates = merged[merged['_merge']=='both']","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:01:59.031529Z","iopub.execute_input":"2022-05-11T13:01:59.031741Z","iopub.status.idle":"2022-05-11T13:01:59.071482Z","shell.execute_reply.started":"2022-05-11T13:01:59.031716Z","shell.execute_reply":"2022-05-11T13:01:59.070555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, (ax0, ax1) = plt.subplots(1, 2, figsize=[10, 3], dpi=200, sharex=True, sharey=True)\nax0.scatter(tr_coordinates['fact_longitude'], tr_coordinates['fact_latitude'], \n            s=1, c='tab:green', label='Train data')\nax1.scatter(common_coordinates['fact_longitude'], common_coordinates['fact_latitude'], \n            s=1, c='tab:orange', label='In-domain test data')\nax1.scatter(tropic_coordinates['fact_longitude'], tropic_coordinates['fact_latitude'], \n            s=1, c='tab:red', label='Out-domain test data')\nax0.legend(), ax1.legend()\nfig.suptitle('Distribution of train and test points in the dataset', y=1.01, fontsize=14)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:01:59.073205Z","iopub.execute_input":"2022-05-11T13:01:59.074187Z","iopub.status.idle":"2022-05-11T13:02:00.046449Z","shell.execute_reply.started":"2022-05-11T13:01:59.074109Z","shell.execute_reply":"2022-05-11T13:02:00.045212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (df.dtypes)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:02:00.04853Z","iopub.execute_input":"2022-05-11T13:02:00.049198Z","iopub.status.idle":"2022-05-11T13:02:00.05824Z","shell.execute_reply.started":"2022-05-11T13:02:00.049142Z","shell.execute_reply":"2022-05-11T13:02:00.057183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualization of the output values**","metadata":{}},{"cell_type":"code","source":"df.fact_temperature.value_counts().count()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:02:00.06272Z","iopub.execute_input":"2022-05-11T13:02:00.06318Z","iopub.status.idle":"2022-05-11T13:02:00.097023Z","shell.execute_reply.started":"2022-05-11T13:02:00.063078Z","shell.execute_reply":"2022-05-11T13:02:00.096058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print (df['fact_temperature'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:02:00.098534Z","iopub.execute_input":"2022-05-11T13:02:00.098875Z","iopub.status.idle":"2022-05-11T13:02:00.141712Z","shell.execute_reply.started":"2022-05-11T13:02:00.098829Z","shell.execute_reply":"2022-05-11T13:02:00.140558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"One useless column :","metadata":{}},{"cell_type":"code","source":"print (df['gfs_available'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:02:00.14342Z","iopub.execute_input":"2022-05-11T13:02:00.143979Z","iopub.status.idle":"2022-05-11T13:02:00.174455Z","shell.execute_reply.started":"2022-05-11T13:02:00.143922Z","shell.execute_reply":"2022-05-11T13:02:00.173491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Pre-processing","metadata":{}},{"cell_type":"code","source":"a = np.array(df['fact_time']).copy()\na = pd.to_datetime(a,unit='s')\ndf['year'] = a.year\ndf['hour'] = a.hour\ndf['day'] = a.day\ndf['month'] = a.month\ndf= df.drop(['fact_time'],axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:06:37.649023Z","iopub.execute_input":"2022-05-11T20:06:37.649411Z","iopub.status.idle":"2022-05-11T20:06:39.525668Z","shell.execute_reply.started":"2022-05-11T20:06:37.649371Z","shell.execute_reply":"2022-05-11T20:06:39.524684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:02:01.673971Z","iopub.execute_input":"2022-05-11T13:02:01.674204Z","iopub.status.idle":"2022-05-11T13:02:01.702446Z","shell.execute_reply.started":"2022-05-11T13:02:01.674178Z","shell.execute_reply":"2022-05-11T13:02:01.701498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Correlation****","metadata":{}},{"cell_type":"markdown","source":"We want to see the correlation matrix between the features","metadata":{}},{"cell_type":"code","source":"df.corr(method='spearman').style.format(\"{:.2}\").background_gradient(cmap=plt.get_cmap('coolwarm'))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T13:02:01.703963Z","iopub.execute_input":"2022-05-11T13:02:01.704209Z","iopub.status.idle":"2022-05-11T14:00:35.366696Z","shell.execute_reply.started":"2022-05-11T13:02:01.70418Z","shell.execute_reply":"2022-05-11T14:00:35.364277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's' print the correlation value between the features gfs_temp_25000->975000. print the correlation value between the features gfs_temp_25000->975000.","metadata":{}},{"cell_type":"markdown","source":"We have seen with the corrolation matrix that the high correlation value are obtained with climate_temperature, cmc_0_0_6_2, cmc_0_1_0_0, cmc_0_3_5_500, cmc-0-3-5-7000, gfs_2m_dewpoint, gfs_predictable_xater, gfs_soil_temp, gfs_temp_25000 ----> 975000, wrf_t2_next, wrf_t2_interpolated and gfs_2m_dewpoint_next.","metadata":{}},{"cell_type":"code","source":"df[['gfs_temperature_25000','gfs_temperature_30000','gfs_temperature_35000','gfs_temperature_40000','gfs_temperature_45000','gfs_temperature_50000','gfs_temperature_55000','gfs_temperature_60000','gfs_temperature_65000','gfs_temperature_70000','gfs_temperature_75000','gfs_temperature_80000','gfs_temperature_85000','gfs_temperature_90000','gfs_temperature_92500','gfs_temperature_95000','gfs_temperature_97500']].corr(method='spearman').style.format(\"{:.2}\").background_gradient(cmap=plt.get_cmap('coolwarm'))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T14:00:35.37151Z","iopub.execute_input":"2022-05-11T14:00:35.371853Z","iopub.status.idle":"2022-05-11T14:00:52.037397Z","shell.execute_reply.started":"2022-05-11T14:00:35.371819Z","shell.execute_reply":"2022-05-11T14:00:52.036151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see here that all those features are higly correlated. We are going to keep in one data set only the one who has the higher correlation will all the features: gfs_temperature_4000. The correlation with the other is between 0.79 and 1.","metadata":{}},{"cell_type":"markdown","source":"Let's' print the correlation value between the features cmc_0_1... ","metadata":{}},{"cell_type":"code","source":"df[['cmc_0_0_6_2','cmc_0_0_7_1000','cmc_0_0_7_2','cmc_0_0_7_500','cmc_0_0_7_700','cmc_0_0_7_850','cmc_0_0_7_925','cmc_0_1_0_0','cmc_0_1_11_0','cmc_0_1_65_0','cmc_0_1_66_0','cmc_0_1_67_0','cmc_0_1_68_0','cmc_0_1_7_0','cmc_0_2_2_10','cmc_0_2_2_1000','cmc_0_2_2_500','cmc_0_2_2_700','cmc_0_2_2_850','cmc_0_2_2_925','cmc_0_2_3_10','cmc_0_2_3_1000','cmc_0_2_3_500','cmc_0_2_3_700','cmc_0_2_3_850','cmc_0_2_3_925','cmc_0_3_0_0','cmc_0_3_0_0_next','cmc_0_3_1_0','cmc_0_3_5_1000','cmc_0_3_5_500','cmc_0_3_5_700','cmc_0_3_5_850','cmc_0_3_5_925','cmc_0_6_1_0']].corr(method='spearman').style.format(\"{:.2}\").background_gradient(cmap=plt.get_cmap('coolwarm'))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T14:00:52.039299Z","iopub.execute_input":"2022-05-11T14:00:52.03993Z","iopub.status.idle":"2022-05-11T14:01:41.218826Z","shell.execute_reply.started":"2022-05-11T14:00:52.039877Z","shell.execute_reply":"2022-05-11T14:01:41.21787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There is poor corelation between those variables.","metadata":{}},{"cell_type":"code","source":"col_selection = [\n    'index', 'year', 'hour', 'day','month','fact_latitude', 'fact_longitude', #parameters\n    'climate_temperature', 'cmc_0_0_6_2', 'cmc_0_1_0_0', 'cmc_0_3_5_500',\n    'cmc_0_3_5_700', 'gfs_2m_dewpoint', 'gfs_precipitable_water', 'gfs_soil_temperature',\n    'gfs_temperature_40000', 'wrf_t2_next', 'wrf_t2_interpolated','gfs_2m_dewpoint_next',\n    'fact_temperature' #output\n]","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:06:45.328868Z","iopub.execute_input":"2022-05-11T20:06:45.329212Z","iopub.status.idle":"2022-05-11T20:06:45.335734Z","shell.execute_reply.started":"2022-05-11T20:06:45.329176Z","shell.execute_reply":"2022-05-11T20:06:45.334487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Processs****","metadata":{}},{"cell_type":"code","source":"def dataset_process(dataset): \n    \n    dataset = df.drop(['index'],axis=1)\n    #delete all the rows that have missed data \n\n    index_with_nan = dataset.index[df.isnull().any(axis=1)]\n    print(index_with_nan.shape)\n    dataset.drop(index_with_nan,0, inplace=True)\n\n    #split the data into training and validation\n\n    n = dataset.shape[0]\n    X_train, X_test, y_train, y_test = train_test_split(dataset.drop(['fact_temperature'], axis=1), dataset['fact_temperature'], test_size=int(n*0.2))\n\n    #normalisation thanks to sklearn transformers \n\n    scaler = RobustScaler()\n    X_train_norm = scaler.fit_transform(X_train) \n    X_test_norm = scaler.transform(X_test)\n    \n    return X_train, X_test, y_train, y_test, X_train_norm, X_test_norm, scaler\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:06:54.289634Z","iopub.execute_input":"2022-05-11T20:06:54.290753Z","iopub.status.idle":"2022-05-11T20:06:54.299149Z","shell.execute_reply.started":"2022-05-11T20:06:54.290689Z","shell.execute_reply":"2022-05-11T20:06:54.298039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Variance threshold:****","metadata":{}},{"cell_type":"code","source":"X_train, X_test, y_train, y_test, X_train_norm, X_test_norm, scaler = dataset_process(df)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:50:52.105773Z","iopub.execute_input":"2022-05-11T20:50:52.107015Z","iopub.status.idle":"2022-05-11T20:51:16.753560Z","shell.execute_reply.started":"2022-05-11T20:50:52.106954Z","shell.execute_reply":"2022-05-11T20:51:16.752387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train_norm.var(axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:18:36.090561Z","iopub.execute_input":"2022-05-11T18:18:36.090956Z","iopub.status.idle":"2022-05-11T18:18:37.110440Z","shell.execute_reply.started":"2022-05-11T18:18:36.090915Z","shell.execute_reply":"2022-05-11T18:18:37.109448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that some features have a very low variance we decided to add a threshold We can see that some features have a very low variance.","metadata":{}},{"cell_type":"code","source":"selector = VarianceThreshold(threshold=0.2)\nX_train_norm_var = selector.fit_transform(X_train_norm)\nX_test_norm_var = selector.transform(X_test_norm) \nselector.get_support()\nX_train_norm_var.shape","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:07:25.141835Z","iopub.execute_input":"2022-05-11T20:07:25.142290Z","iopub.status.idle":"2022-05-11T20:07:27.622471Z","shell.execute_reply.started":"2022-05-11T20:07:25.142236Z","shell.execute_reply":"2022-05-11T20:07:27.621535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Only the features which have a high correlation with the output****","metadata":{}},{"cell_type":"code","source":"X_train2, X_test2, y_train2, y_test2, X_train_norm2, X_test_norm2, scaler2 = dataset_process(df[col_selection])","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:19:01.448908Z","iopub.execute_input":"2022-05-11T18:19:01.449525Z","iopub.status.idle":"2022-05-11T18:19:20.335031Z","shell.execute_reply.started":"2022-05-11T18:19:01.449479Z","shell.execute_reply":"2022-05-11T18:19:20.334086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Process with PCA**","metadata":{}},{"cell_type":"code","source":"from sklearn.decomposition import PCA","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:19:27.448945Z","iopub.execute_input":"2022-05-11T18:19:27.449219Z","iopub.status.idle":"2022-05-11T18:19:27.454332Z","shell.execute_reply.started":"2022-05-11T18:19:27.449189Z","shell.execute_reply":"2022-05-11T18:19:27.453301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train3, X_test3, y_train3, y_test3, X_train_norm3, X_test_norm3, scaler3 = X_train, X_test, y_train, y_test, X_train_norm, X_test_norm","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:19:29.749050Z","iopub.execute_input":"2022-05-11T18:19:29.749476Z","iopub.status.idle":"2022-05-11T18:19:29.754007Z","shell.execute_reply.started":"2022-05-11T18:19:29.749444Z","shell.execute_reply":"2022-05-11T18:19:29.753117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pca = PCA(0.99)\nprincipalComponents = pca.fit_transform(X_train_norm3)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:19:31.867972Z","iopub.execute_input":"2022-05-11T18:19:31.868260Z","iopub.status.idle":"2022-05-11T18:19:54.917900Z","shell.execute_reply.started":"2022-05-11T18:19:31.868225Z","shell.execute_reply":"2022-05-11T18:19:54.916756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_norm3 = pca.transform(X_test3)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:19:57.987906Z","iopub.execute_input":"2022-05-11T18:19:57.988203Z","iopub.status.idle":"2022-05-11T18:19:58.444897Z","shell.execute_reply.started":"2022-05-11T18:19:57.988175Z","shell.execute_reply":"2022-05-11T18:19:58.444002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_test_norm3.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"At this step we only have 95 features.","metadata":{}},{"cell_type":"code","source":"X1.var(axis=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Model Selection\n","metadata":{}},{"cell_type":"code","source":"def compute_rmse(y, ypred, ystd=1.):\n    return np.mean((y - ypred)**2)**0.5 * ystd","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:07:47.409933Z","iopub.execute_input":"2022-05-11T20:07:47.410239Z","iopub.status.idle":"2022-05-11T20:07:47.415407Z","shell.execute_reply.started":"2022-05-11T20:07:47.410209Z","shell.execute_reply":"2022-05-11T20:07:47.414735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Lasso**","metadata":{}},{"cell_type":"markdown","source":"With every feature","metadata":{}},{"cell_type":"code","source":"model1 = Lasso(alpha=0.1)\nmodel1.fit(X_train_norm, y_train)\ny_pred = model1.predict(X_test_norm)\ny_pred2= model1.predict(X_train_norm)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With the features with high variance","metadata":{}},{"cell_type":"code","source":"model1 = Lasso(alpha=0.1)\nmodel1.fit(X_train_norm_var, y_train)\ny_pred = model1.predict(X_test_norm_var)\ny_pred2= model1.predict(X_train_norm_var)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Linear regression**","metadata":{}},{"cell_type":"markdown","source":"With every features:","metadata":{}},{"cell_type":"code","source":"model1 = LinearRegression().fit(X_train_norm, y_train)\ny_pred = model1.predict(X_test_norm)\ny_pred2= model1.predict(X_train_norm)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:20:25.648596Z","iopub.execute_input":"2022-05-11T18:20:25.649339Z","iopub.status.idle":"2022-05-11T18:20:38.275024Z","shell.execute_reply.started":"2022-05-11T18:20:25.649260Z","shell.execute_reply":"2022-05-11T18:20:38.274012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With only the high variance features:","metadata":{}},{"cell_type":"code","source":"model1 = LinearRegression().fit(X_train_norm_var, y_train)\ny_pred = model1.predict(X_test_norm_var)\ny_pred2= model1.predict(X_train_norm_var)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:20:42.437475Z","iopub.execute_input":"2022-05-11T18:20:42.438179Z","iopub.status.idle":"2022-05-11T18:20:52.693088Z","shell.execute_reply.started":"2022-05-11T18:20:42.438140Z","shell.execute_reply":"2022-05-11T18:20:52.692052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Decision tree****","metadata":{}},{"cell_type":"markdown","source":"With all the features","metadata":{}},{"cell_type":"code","source":"# import the regressor\nfrom sklearn.tree import DecisionTreeRegressor\n\n# create a regressor object\nregressor = DecisionTreeRegressor() \n\n# fit the regressor with X and Y data\nregressor.fit(X_train_norm, y_train)\ny_pred = model1.predict(X_test_norm)\ny_pred2= model1.predict(X_train_norm)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:27:11.599679Z","iopub.execute_input":"2022-05-11T18:27:11.600137Z","iopub.status.idle":"2022-05-11T18:33:17.349443Z","shell.execute_reply.started":"2022-05-11T18:27:11.600085Z","shell.execute_reply":"2022-05-11T18:33:17.348196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With only the feature with high variance","metadata":{}},{"cell_type":"code","source":"# create a regressor object\nregressor = DecisionTreeRegressor()\n\n# fit the regressor with X and Y data\nregressor.fit(X_train_norm_var, y_train)\ny_pred = regressor.predict(X_test_norm_var)\ny_pred2= regressor.predict(X_train_norm_var)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Random forest****","metadata":{}},{"cell_type":"markdown","source":"With all the features ","metadata":{}},{"cell_type":"code","source":"rf = RandomForestRegressor(n_estimators = 300, max_features = 'sqrt', max_depth = 5, random_state = 18).fit(X_train_norm, y_train)\ny_pred = rf.predict(X_test_norm)\ny_pred2= rf.predict(X_train_norm)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T19:12:17.696762Z","iopub.execute_input":"2022-05-11T19:12:17.697753Z","iopub.status.idle":"2022-05-11T19:24:44.182490Z","shell.execute_reply.started":"2022-05-11T19:12:17.697666Z","shell.execute_reply":"2022-05-11T19:24:44.181222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With only the features with high variance","metadata":{}},{"cell_type":"code","source":"rf = RandomForestRegressor(n_estimators = 300, max_features = 'sqrt', max_depth = 5, random_state = 18).fit(X_train_norm_var, y_train)\ny_pred = rf.predict(X_test_norm_var)\ny_pred2= rf.predict(X_train_norm_var)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****Neural Network****","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nfrom keras import backend as K","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:08:14.018590Z","iopub.execute_input":"2022-05-11T20:08:14.020017Z","iopub.status.idle":"2022-05-11T20:08:14.025231Z","shell.execute_reply.started":"2022-05-11T20:08:14.019955Z","shell.execute_reply":"2022-05-11T20:08:14.024070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With the features with high variance","metadata":{}},{"cell_type":"code","source":"def root_mean_squared_error(y_true, y_pred):\n        return K.sqrt(K.mean(K.square(y_pred - y_true)))\n    \n    \n\nmodel = keras.Sequential()\nmodel.add(layers.Dense(66, activation=\"relu\"))\nmodel.add(layers.Dense(33, activation=\"relu\"))\nmodel.add(layers.Dense(1, activation=\"linear\"))\nmodel.compile(optimizer='adam', loss=root_mean_squared_error)\n\nhistory = model.fit(X_train_norm_var, y_train, batch_size = 32, epochs = 20, validation_data=(X_test_norm_var, y_test))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:08:19.756200Z","iopub.execute_input":"2022-05-11T20:08:19.756588Z","iopub.status.idle":"2022-05-11T20:49:53.257699Z","shell.execute_reply.started":"2022-05-11T20:08:19.756543Z","shell.execute_reply":"2022-05-11T20:49:53.255538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:50:02.741116Z","iopub.execute_input":"2022-05-11T20:50:02.742041Z","iopub.status.idle":"2022-05-11T20:50:02.994804Z","shell.execute_reply.started":"2022-05-11T20:50:02.741993Z","shell.execute_reply":"2022-05-11T20:50:02.994104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Another type of NN**","metadata":{}},{"cell_type":"markdown","source":"With all the features","metadata":{}},{"cell_type":"code","source":"from sklearn.neural_network import MLPRegressor\nregr = MLPRegressor(random_state=1, max_iter=20).fit(X_train_norm, y_train)\ny_pred = regr.predict(X_test_norm)\ny_pred2= regr.predict(X_train_norm)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T18:52:12.052802Z","iopub.execute_input":"2022-05-11T18:52:12.054544Z","iopub.status.idle":"2022-05-11T18:59:09.743095Z","shell.execute_reply.started":"2022-05-11T18:52:12.054482Z","shell.execute_reply":"2022-05-11T18:59:09.742026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Only with the features with high variance","metadata":{}},{"cell_type":"code","source":"from sklearn.neural_network import MLPRegressor\nregr2 = MLPRegressor(random_state=1, max_iter=20).fit(X_train_norm_var, y_train)\ny_pred = regr2.predict(X_test_norm_var)\ny_pred2= regr2.predict(X_train_norm_var)\nprint('rmse train : '+str(compute_rmse(y_pred2, y_train)))\nprint('rmse val : '+str(compute_rmse(y_test, y_pred)))","metadata":{"execution":{"iopub.status.busy":"2022-05-11T19:01:25.249770Z","iopub.execute_input":"2022-05-11T19:01:25.250169Z","iopub.status.idle":"2022-05-11T19:07:37.439960Z","shell.execute_reply.started":"2022-05-11T19:01:25.250130Z","shell.execute_reply":"2022-05-11T19:07:37.438756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"df_test1 = df_test.copy()\na = np.array(df_test1['fact_time']).copy()\na = pd.to_datetime(a,unit='s')\ndf_test1['year'] = a.year\ndf_test1['hour'] = a.hour\ndf_test1['day'] = a.day\ndf_test1['month'] = a.month\ndf_test1= df_test1.drop(['fact_time'],axis=1)\ndf_test1 = df_test1.drop(['index'],axis=1)\n\ninput_test = scaler.transform(df_test1)\ninput_test = selector.transform(input_test)\ny_pred=model.predict(input_test)   # here we were testing for Neural Network using keras and only the features with high correlation","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:52:45.738000Z","iopub.execute_input":"2022-05-11T20:52:45.738744Z","iopub.status.idle":"2022-05-11T20:53:06.689943Z","shell.execute_reply.started":"2022-05-11T20:52:45.738696Z","shell.execute_reply":"2022-05-11T20:53:06.689000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame(data={'index': df_test['index'].values,\n                                   'fact_temperature': y_pred.squeeze()})\nsubmission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T20:53:12.250428Z","iopub.execute_input":"2022-05-11T20:53:12.250886Z","iopub.status.idle":"2022-05-11T20:53:13.782395Z","shell.execute_reply.started":"2022-05-11T20:53:12.250840Z","shell.execute_reply":"2022-05-11T20:53:13.781261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check the submission file\n! head -6 \"/kaggle/working/submission.csv\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}