{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input/emsi-5iir-ds'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2023-01-08T21:21:37.799027Z","iopub.execute_input":"2023-01-08T21:21:37.799381Z","iopub.status.idle":"2023-01-08T21:21:37.805983Z","shell.execute_reply.started":"2023-01-08T21:21:37.799349Z","shell.execute_reply":"2023-01-08T21:21:37.805144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Beginning**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.linear_model import LogisticRegression","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:37.807513Z","iopub.execute_input":"2023-01-08T21:21:37.807774Z","iopub.status.idle":"2023-01-08T21:21:37.821410Z","shell.execute_reply.started":"2023-01-08T21:21:37.807743Z","shell.execute_reply":"2023-01-08T21:21:37.820712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reading Data**","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:37.822316Z","iopub.execute_input":"2023-01-08T21:21:37.822843Z","iopub.status.idle":"2023-01-08T21:21:43.211686Z","shell.execute_reply.started":"2023-01-08T21:21:37.822817Z","shell.execute_reply":"2023-01-08T21:21:43.210476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:43.214059Z","iopub.execute_input":"2023-01-08T21:21:43.214392Z","iopub.status.idle":"2023-01-08T21:21:43.350526Z","shell.execute_reply.started":"2023-01-08T21:21:43.214361Z","shell.execute_reply":"2023-01-08T21:21:43.349598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:43.351559Z","iopub.execute_input":"2023-01-08T21:21:43.351848Z","iopub.status.idle":"2023-01-08T21:21:43.446354Z","shell.execute_reply.started":"2023-01-08T21:21:43.351789Z","shell.execute_reply":"2023-01-08T21:21:43.445508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:43.447560Z","iopub.execute_input":"2023-01-08T21:21:43.447823Z","iopub.status.idle":"2023-01-08T21:21:44.189534Z","shell.execute_reply.started":"2023-01-08T21:21:43.447799Z","shell.execute_reply":"2023-01-08T21:21:44.188425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['f_27'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:44.190500Z","iopub.execute_input":"2023-01-08T21:21:44.190724Z","iopub.status.idle":"2023-01-08T21:21:44.995798Z","shell.execute_reply.started":"2023-01-08T21:21:44.190702Z","shell.execute_reply":"2023-01-08T21:21:44.994849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### No one-hot encoding, too many unique values","metadata":{}},{"cell_type":"code","source":"ordinal_mapping = {c: i for i, c in enumerate('ABCDEFGHIJKLMNOPQRSTUVWXYZ')}\n\nordinal_mapping['?'] = 26","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:44.996948Z","iopub.execute_input":"2023-01-08T21:21:44.997294Z","iopub.status.idle":"2023-01-08T21:21:45.002125Z","shell.execute_reply.started":"2023-01-08T21:21:44.997261Z","shell.execute_reply":"2023-01-08T21:21:45.001119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## testing with 30_000 rows only for faster benchmarking","metadata":{}},{"cell_type":"code","source":"col = \"f_27\"\n\n# train_df = train_df.head(30_000)\n\n# test_df = test_df.head(30_000)\n\nenc_df = []\n\nfor df in [train_df, test_df]:\n    col_enum = df[col].apply(lambda x: pd.Series([ ordinal_mapping[c] for c in x]))\n\n    col_enum\n\n\n\n    # Rename the columns\n    col_enum.columns = [f'{col}_enum_{i}' for i in range(col_enum.shape[1])]\n\n    # Concatenate the original dataframe with the new columns\n    tmp = pd.concat([df, col_enum], axis=1)\n    \n    tmp['unique'] = df['f_27'].apply(lambda x: len(set(x)))\n    \n    tmp = tmp.drop(labels=[\"f_27\"], axis=1)\n    \n    enc_df.append(tmp)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:21:45.003492Z","iopub.execute_input":"2023-01-08T21:21:45.004076Z","iopub.status.idle":"2023-01-08T21:26:37.314097Z","shell.execute_reply.started":"2023-01-08T21:21:45.004041Z","shell.execute_reply":"2023-01-08T21:26:37.313346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##  Unique feature describes the unique values present in the f_27 column","metadata":{}},{"cell_type":"code","source":"enc_train_df, enc_test_df = enc_df","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.316948Z","iopub.execute_input":"2023-01-08T21:26:37.317358Z","iopub.status.idle":"2023-01-08T21:26:37.320890Z","shell.execute_reply.started":"2023-01-08T21:26:37.317333Z","shell.execute_reply":"2023-01-08T21:26:37.320258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enc_train_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.321758Z","iopub.execute_input":"2023-01-08T21:26:37.322740Z","iopub.status.idle":"2023-01-08T21:26:37.345698Z","shell.execute_reply.started":"2023-01-08T21:26:37.322714Z","shell.execute_reply":"2023-01-08T21:26:37.344742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"enc_test_df.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.346706Z","iopub.execute_input":"2023-01-08T21:26:37.346938Z","iopub.status.idle":"2023-01-08T21:26:37.368830Z","shell.execute_reply.started":"2023-01-08T21:26:37.346916Z","shell.execute_reply":"2023-01-08T21:26:37.368031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.372638Z","iopub.execute_input":"2023-01-08T21:26:37.372955Z","iopub.status.idle":"2023-01-08T21:26:37.380986Z","shell.execute_reply.started":"2023-01-08T21:26:37.372931Z","shell.execute_reply":"2023-01-08T21:26:37.380407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Given the test.csv and the train.csv i assume it's supervised learning","metadata":{}},{"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(enc_train_df.drop(columns=['target']), enc_train_df['target'], test_size=0.2, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.382007Z","iopub.execute_input":"2023-01-08T21:26:37.382470Z","iopub.status.idle":"2023-01-08T21:26:37.752900Z","shell.execute_reply.started":"2023-01-08T21:26:37.382426Z","shell.execute_reply":"2023-01-08T21:26:37.752238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.753968Z","iopub.execute_input":"2023-01-08T21:26:37.754213Z","iopub.status.idle":"2023-01-08T21:26:37.816397Z","shell.execute_reply.started":"2023-01-08T21:26:37.754189Z","shell.execute_reply":"2023-01-08T21:26:37.815501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_valid","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.817671Z","iopub.execute_input":"2023-01-08T21:26:37.817959Z","iopub.status.idle":"2023-01-08T21:26:37.849500Z","shell.execute_reply.started":"2023-01-08T21:26:37.817934Z","shell.execute_reply":"2023-01-08T21:26:37.848597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## The f_27 can't be one-hot encoded; too many unique values","metadata":{}},{"cell_type":"markdown","source":"# Understanding n_estimators impact on the model's performance\n\n### benchmarking:","metadata":{}},{"cell_type":"code","source":"# from sklearn.model_selection import cross_val_score\n\n# n_estimators_values = [10, 50, 100, 200, 500]\n\n# scores = []\n\n# for n_estimators in n_estimators_values:\n#     model = RandomForestClassifier(n_estimators=n_estimators)\n    \n#     score = cross_val_score(model, X_train, y_train, cv=5).mean()\n    \n#     scores.append(score)\n    \n# best_n_estimators = n_estimators_values[scores.index(max(scores))]\n\n# best_n_estimators","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2023-01-08T21:26:37.850845Z","iopub.execute_input":"2023-01-08T21:26:37.851214Z","iopub.status.idle":"2023-01-08T21:26:37.855031Z","shell.execute_reply.started":"2023-01-08T21:26:37.851188Z","shell.execute_reply":"2023-01-08T21:26:37.854321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = RandomForestClassifier(n_estimators=100, random_state=42)\n\n# Fit the model to the training data\nmodel.fit(X_train, y_train)\n\n# Use the model to make predictions on the test data\ny_pred = model.predict(X_valid)\n\n# Calculate the accuracy of the model\n# accuracy = model.score(X_valid, y_valid)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:26:37.855917Z","iopub.execute_input":"2023-01-08T21:26:37.856468Z","iopub.status.idle":"2023-01-08T21:36:17.548383Z","shell.execute_reply.started":"2023-01-08T21:26:37.856428Z","shell.execute_reply":"2023-01-08T21:36:17.547398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_proba_pred = model.predict_proba(X_valid)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:17.549586Z","iopub.execute_input":"2023-01-08T21:36:17.549925Z","iopub.status.idle":"2023-01-08T21:36:25.805532Z","shell.execute_reply.started":"2023-01-08T21:36:17.549893Z","shell.execute_reply":"2023-01-08T21:36:25.803744Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_proba_pred","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.807240Z","iopub.execute_input":"2023-01-08T21:36:25.808138Z","iopub.status.idle":"2023-01-08T21:36:25.815662Z","shell.execute_reply.started":"2023-01-08T21:36:25.808105Z","shell.execute_reply":"2023-01-08T21:36:25.814158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfor_proba_submission = model.predict_proba(enc_test_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:50:51.806876Z","iopub.execute_input":"2023-01-08T21:50:51.807308Z","iopub.status.idle":"2023-01-08T21:51:22.831892Z","shell.execute_reply.started":"2023-01-08T21:50:51.807273Z","shell.execute_reply":"2023-01-08T21:51:22.830932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfor_proba_submission","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:51:22.833904Z","iopub.execute_input":"2023-01-08T21:51:22.834340Z","iopub.status.idle":"2023-01-08T21:51:22.842862Z","shell.execute_reply.started":"2023-01-08T21:51:22.834299Z","shell.execute_reply":"2023-01-08T21:51:22.841673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfor_proba_submission_df = pd.DataFrame({'id': enc_test_df[\"id\"], 'target': [x[1] for x in rfor_proba_submission]})","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:51:35.225045Z","iopub.execute_input":"2023-01-08T21:51:35.225911Z","iopub.status.idle":"2023-01-08T21:51:35.462674Z","shell.execute_reply.started":"2023-01-08T21:51:35.225878Z","shell.execute_reply":"2023-01-08T21:51:35.461900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfor_proba_submission_df","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:51:47.120070Z","iopub.execute_input":"2023-01-08T21:51:47.120679Z","iopub.status.idle":"2023-01-08T21:51:47.132879Z","shell.execute_reply.started":"2023-01-08T21:51:47.120644Z","shell.execute_reply":"2023-01-08T21:51:47.131712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rfor_proba_submission_df.to_csv(\"sub_rfor_proba.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.907212Z","iopub.status.idle":"2023-01-08T21:36:25.907717Z","shell.execute_reply.started":"2023-01-08T21:36:25.907430Z","shell.execute_reply":"2023-01-08T21:36:25.907450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_valid","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.909625Z","iopub.status.idle":"2023-01-08T21:36:25.910084Z","shell.execute_reply.started":"2023-01-08T21:36:25.909850Z","shell.execute_reply":"2023-01-08T21:36:25.909872Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, roc_auc_score\n\naccuracy_score = accuracy_score(y_valid, y_pred)\nroc_auc_score = roc_auc_score(y_valid, y_pred)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.911831Z","iopub.status.idle":"2023-01-08T21:36:25.912269Z","shell.execute_reply.started":"2023-01-08T21:36:25.912043Z","shell.execute_reply":"2023-01-08T21:36:25.912064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.913353Z","iopub.status.idle":"2023-01-08T21:36:25.913813Z","shell.execute_reply.started":"2023-01-08T21:36:25.913580Z","shell.execute_reply":"2023-01-08T21:36:25.913600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roc_auc_score","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.915328Z","iopub.status.idle":"2023-01-08T21:36:25.915768Z","shell.execute_reply.started":"2023-01-08T21:36:25.915552Z","shell.execute_reply":"2023-01-08T21:36:25.915572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgboost_model = xgb.XGBClassifier(objective='binary:logistic',seed=42,n_estimators=500)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.919120Z","iopub.status.idle":"2023-01-08T21:36:25.919591Z","shell.execute_reply.started":"2023-01-08T21:36:25.919334Z","shell.execute_reply":"2023-01-08T21:36:25.919355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fit the model to the training data\nxgboost_model.fit(X_train, y_train)\n\n# Make predictions on the test set\npredictions = xgboost_model.predict(X_valid)\n\nprint(predictions)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.920941Z","iopub.status.idle":"2023-01-08T21:36:25.921362Z","shell.execute_reply.started":"2023-01-08T21:36:25.921140Z","shell.execute_reply":"2023-01-08T21:36:25.921161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score\n\naccuracy_score = accuracy_score(y_valid, predictions)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.922982Z","iopub.status.idle":"2023-01-08T21:36:25.923413Z","shell.execute_reply.started":"2023-01-08T21:36:25.923184Z","shell.execute_reply":"2023-01-08T21:36:25.923204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy_score","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.925225Z","iopub.status.idle":"2023-01-08T21:36:25.925731Z","shell.execute_reply.started":"2023-01-08T21:36:25.925495Z","shell.execute_reply":"2023-01-08T21:36:25.925523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\nroc_auc = roc_auc_score(y_valid, predictions)\nprint(\"ROC AUC: \", roc_auc)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.926796Z","iopub.status.idle":"2023-01-08T21:36:25.927200Z","shell.execute_reply.started":"2023-01-08T21:36:25.926990Z","shell.execute_reply":"2023-01-08T21:36:25.927009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = xgboost_model.predict(enc_test_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.928661Z","iopub.status.idle":"2023-01-08T21:36:25.929069Z","shell.execute_reply.started":"2023-01-08T21:36:25.928856Z","shell.execute_reply":"2023-01-08T21:36:25.928876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.930315Z","iopub.status.idle":"2023-01-08T21:36:25.930998Z","shell.execute_reply.started":"2023-01-08T21:36:25.930779Z","shell.execute_reply":"2023-01-08T21:36:25.930801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Probability p_0 et p_1","metadata":{}},{"cell_type":"code","source":"probs = xgboost_model.predict_proba(enc_test_df)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.932150Z","iopub.status.idle":"2023-01-08T21:36:25.932581Z","shell.execute_reply.started":"2023-01-08T21:36:25.932341Z","shell.execute_reply":"2023-01-08T21:36:25.932360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Take only p_1","metadata":{}},{"cell_type":"code","source":"probas = [p[1] for p in probs]","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.933847Z","iopub.status.idle":"2023-01-08T21:36:25.934252Z","shell.execute_reply.started":"2023-01-08T21:36:25.934041Z","shell.execute_reply":"2023-01-08T21:36:25.934061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"probas[:10]","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.935934Z","iopub.status.idle":"2023-01-08T21:36:25.936366Z","shell.execute_reply.started":"2023-01-08T21:36:25.936144Z","shell.execute_reply":"2023-01-08T21:36:25.936164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.DataFrame({'id': enc_test_df[\"id\"], 'target': probas})","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.937822Z","iopub.status.idle":"2023-01-08T21:36:25.938251Z","shell.execute_reply.started":"2023-01-08T21:36:25.938017Z","shell.execute_reply":"2023-01-08T21:36:25.938039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.939831Z","iopub.status.idle":"2023-01-08T21:36:25.940258Z","shell.execute_reply.started":"2023-01-08T21:36:25.940028Z","shell.execute_reply":"2023-01-08T21:36:25.940050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!head submission.csv","metadata":{"execution":{"iopub.status.busy":"2023-01-08T21:36:25.942180Z","iopub.status.idle":"2023-01-08T21:36:25.942643Z","shell.execute_reply.started":"2023-01-08T21:36:25.942386Z","shell.execute_reply":"2023-01-08T21:36:25.942406Z"},"trusted":true},"execution_count":null,"outputs":[]}]}