{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

AML Challenge 1

\n\n

CORDINA Emmanuel
RADJOU Aditya

","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:20:44.710539Z","iopub.execute_input":"2022-05-11T09:20:44.710875Z","iopub.status.idle":"2022-05-11T09:20:44.716677Z","shell.execute_reply.started":"2022-05-11T09:20:44.710838Z","shell.execute_reply":"2022-05-11T09:20:44.715613Z"}}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-11T09:33:00.970752Z","iopub.execute_input":"2022-05-11T09:33:00.971093Z","iopub.status.idle":"2022-05-11T09:33:00.983660Z","shell.execute_reply.started":"2022-05-11T09:33:00.971060Z","shell.execute_reply":"2022-05-11T09:33:00.982716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import all the necessary modules \n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import mean_squared_error\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:33:00.988558Z","iopub.execute_input":"2022-05-11T09:33:00.989431Z","iopub.status.idle":"2022-05-11T09:33:00.998569Z","shell.execute_reply.started":"2022-05-11T09:33:00.989394Z","shell.execute_reply":"2022-05-11T09:33:00.997356Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Load data\n\ndf_train = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/train.csv')\ndf_test = pd.read_csv('../input/eurecom-aml-2022-challenge-1/public/test_feat.csv')","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:33:01.007087Z","iopub.execute_input":"2022-05-11T09:33:01.007329Z","iopub.status.idle":"2022-05-11T09:33:59.933706Z","shell.execute_reply.started":"2022-05-11T09:33:01.007299Z","shell.execute_reply":"2022-05-11T09:33:59.932940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Step 1: Pre-processing

","metadata":{}},{"cell_type":"code","source":"# Replacing the missing values with the mean value\n\nfor i in df_train.columns:\n if df_train[i].dtype == 'object':\n df_train[i] = df_train[i].fillna(df_train[i].mode().iloc[0])\n if (df_train[i].dtype == 'int' or df_train[i].dtype == 'float'):\n df_train[i] =df_train[i].fillna(np.mean(df_train[i]))\n\nfor i in df_test.columns:\n if df_test[i].dtype == 'object':\n df_test[i] = df_test[i].fillna(df_test[i].mode().iloc[0])\n if (df_test[i].dtype == 'int' or df_test[i].dtype == 'float'):\n df_test[i] = df_test[i].fillna(np.mean(df_test[i]))\n\nX = np.array(df_train[df_train.columns[:-1]])\ny = np.array(df_train[\"fact_temperature\"])\n\n# Normalisation\n\nXmean, Xstd, ymean, ystd = X.mean(0), X.std(0), y.mean(), y.std()\n\n#Delete columns where the standard deviation is 0, they are useless because constant\n\nzeros=[]\nfor i in range(len(Xstd)) :\n if Xstd[i]==0 :\n zeros.append(i)\n\nX=np.delete(X,zeros,1) \nXmean=np.delete(Xmean,zeros,0)\nXstd=np.delete(Xstd,zeros,0)\nfeatures = df_train.columns.values\nfor i in range(len(zeros)):\n features = np.delete(features,zeros[i])\n\nX = (X-Xmean)/Xstd\ny = (y-ymean)/ystd\n\n# Remove outliers\n\noutliers=[]\nlen_with_outliers=len(y)\nfor i in range(len_with_outliers) :\n if abs(y[i])>4 :\n outliers.append(i)\ny=np.delete(y,outliers, 0)\nX=np.delete(X,outliers,0)\nprint(len(outliers),\"entries removed out of\",len_with_outliers)","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:33:59.935562Z","iopub.execute_input":"2022-05-11T09:33:59.935873Z","iopub.status.idle":"2022-05-11T09:34:10.927988Z","shell.execute_reply.started":"2022-05-11T09:33:59.935830Z","shell.execute_reply":"2022-05-11T09:34:10.926962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Step 2: Feature selection

","metadata":{}},{"cell_type":"code","source":"# Correlation between fact_temperature and the other features\n\ndata = pd.DataFrame(X,columns=features[:-1])\ndata[\"fact_temperature\"] = y\n\ncor = data.corr()[\"fact_temperature\"]\ncorrelation_values = cor.values\nabs_cor = cor.abs()\n\nfeatures_preselected = abs_cor.sort_values(ascending=False)\ncorrelation_values_abs = features_preselected.values","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:34:10.929367Z","iopub.execute_input":"2022-05-11T09:34:10.929973Z","iopub.status.idle":"2022-05-11T09:35:12.252990Z","shell.execute_reply.started":"2022-05-11T09:34:10.929927Z","shell.execute_reply":"2022-05-11T09:35:12.252306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Plot the results\n\nplt.figure(figsize=(6,4))\nplt.plot(features,correlation_values_abs)\nplt.axhline(y = 0.72, color = 'r', linestyle = '-')\nplt.title(\"Correlation coefficient (absolute value) between all features and 'fact_temperature'\")\nplt.xlabel(\"feature\")\nplt.ylabel(\"Correlation coefficient (absolute value)\")\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-11T09:35:12.255187Z","iopub.execute_input":"2022-05-11T09:35:12.255714Z","iopub.status.idle":"2022-05-11T09:35:13.941023Z","shell.execute_reply.started":"2022-05-11T09:35:12.255666Z","shell.execute_reply":"2022-05-11T09:35:13.940056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features_preselected = features.index.values.tolist()\nfeatures_preselected ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## Covariate shift\n\ndf_train['origin'] = 0\ndf_test['origin'] = 1\n\ntraining = df_train.drop('fact_temperature', axis=1, inplace=False)\nN = 1993574\n\ntraining = training.sample(int(N/100))\ntesting = df_test.sample(int(N/100))\n\ncombi = training.append(df_test)\ncombi.dropna(inplace=True)\ny = combi['origin']\ncombibi = combi.drop('origin', axis=1)\n\nmodel = RandomForestClassifier(n_estimators = 50, n_jobs=-1, max_depth = 5,min_samples_leaf = 5)\ndrop_list = []\nfor i in combibi.columns:\n score = np.mean(cross_val_score(model,pd.DataFrame(combibi[i]),y,cv=2,n_jobs=-1,scoring='roc_auc'))\n drop_list.append([i,score])\n print(i,score)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_list=np.array(drop_list)\ndrop_list=drop_list[drop_list[:,1].argsort()]\ndrop_list=drop_list[::-1]\nfeatures_selected=features_preselected[:5]\nfor feature in features_preselected[5:]:\n if feature not in drop_list[:15,0]:\n features_selected.append(feature)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_data=df_train[features_selected]\nX = np.array(training_data[features_selected[:-1]])\ny = np.array(training_data[features_selected[-1]])","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Using Skicit-learn to split data into training and testing sets\nfrom sklearn.model_selection import train_test_split\n# Split the data into training and testing sets\ntrain_features, test_features, train_labels, test_labels = train_test_split(X, y, test_size = 0.25)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Dummy Model\nmean=train_labels.mean()\nmean","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Instantiate model with 1000 decision trees\nrf = RandomForestRegressor(n_estimators=50,bootstrap=True,n_jobs=-1,random_state=42,verbose =1)\n# Train the model on training data\nrf.fit(train_features, train_labels);","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Use the forest's predict method on the test data\npredictions = rf.predict(test_features)\ndummy_predictions=mean*np.ones(len(test_labels))\n# Calculate the absolute errors\nerrors = mean_squared_error(predictions, test_labels)\ndummy_error=mean_squared_error(dummy_predictions,test_labels)\n# Print out the mse\nprint('MSE:', errors)\nprint('RMSE:', np.sqrt(errors))\nprint('Dummy MSE:',dummy_error)\nprint('Dummy RMSE:',np.sqrt(dummy_error))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test.dropna()\ntest_features_2 = np.array(df_test[features_selected[:-1]])\npredictions = rf.predict(test_features_2)*ystd+ymean\nsubmission_df = pd.DataFrame(data={'index': df_test['index'].values,\n 'fact_temperature': predictions.squeeze()})\nsubmission_df","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv(\"/kaggle/working/submission.csv\", index=False)","metadata":{},"execution_count":null,"outputs":[]}]}