{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to loadil\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # librairie de corrélation\nimport optuna\nimport gc\n#import matplotlib as mpl\n#import matplotlib.pyplot as plt\n#import random\n#import tensorflow as tf\n\n\n#import tensorflow.keras as keras\n#from tensorflow.keras.models import Sequential\n#from tensorflow.keras.layers import Dense, Dropout, BatchNormalization\n#from tensorflow.keras import backend as K\n#from tensorflow.keras.utils import to_categorical\n#from sklearn.model_selection import train_test_split\n#from sklearn.metrics import confusion_matrix\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-08T22:37:43.280808Z","iopub.execute_input":"2023-01-08T22:37:43.281444Z","iopub.status.idle":"2023-01-08T22:37:44.501474Z","shell.execute_reply.started":"2023-01-08T22:37:43.281353Z","shell.execute_reply":"2023-01-08T22:37:44.500185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Méthodes de Traitement de la Data","metadata":{}},{"cell_type":"markdown","source":"**Visualiser les colonnes du fichier Train.csv séparés par une virgule**","metadata":{}},{"cell_type":"code","source":"open('/kaggle/input/emsi-5iir-ds/train.csv').readlines()[0]\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:44.503320Z","iopub.execute_input":"2023-01-08T22:37:44.503629Z","iopub.status.idle":"2023-01-08T22:37:48.987450Z","shell.execute_reply.started":"2023-01-08T22:37:44.503600Z","shell.execute_reply":"2023-01-08T22:37:48.986322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser les  premières valeurs des colonnes du fichier Train.csv séparés par une virgule**\n","metadata":{}},{"cell_type":"code","source":"open('/kaggle/input/emsi-5iir-ds/train.csv').readlines()[1]\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:48.988648Z","iopub.execute_input":"2023-01-08T22:37:48.988985Z","iopub.status.idle":"2023-01-08T22:37:49.647946Z","shell.execute_reply.started":"2023-01-08T22:37:48.988956Z","shell.execute_reply":"2023-01-08T22:37:49.646766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser toutes les colonnes et valeurs du fichier train.csv**","metadata":{}},{"cell_type":"code","source":"pd.read_csv('/kaggle/input/emsi-5iir-ds/train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:49.651226Z","iopub.execute_input":"2023-01-08T22:37:49.651719Z","iopub.status.idle":"2023-01-08T22:37:54.518870Z","shell.execute_reply.started":"2023-01-08T22:37:49.651676Z","shell.execute_reply":"2023-01-08T22:37:54.517784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser 10 colonnes aléatoires et leurs valeurs du fichier Train.csv**","metadata":{}},{"cell_type":"code","source":"df= pd.read_csv('/kaggle/input/emsi-5iir-ds/train.csv')\ndf.sample(10)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:54.520415Z","iopub.execute_input":"2023-01-08T22:37:54.521049Z","iopub.status.idle":"2023-01-08T22:37:59.185648Z","shell.execute_reply.started":"2023-01-08T22:37:54.521006Z","shell.execute_reply":"2023-01-08T22:37:59.184837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser les 5 premières colonnes et leurs valeurs**","metadata":{}},{"cell_type":"code","source":"df.head(5)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:59.187432Z","iopub.execute_input":"2023-01-08T22:37:59.187745Z","iopub.status.idle":"2023-01-08T22:37:59.213726Z","shell.execute_reply.started":"2023-01-08T22:37:59.187716Z","shell.execute_reply":"2023-01-08T22:37:59.212303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser l'intitulé des différentes colonnes**\n","metadata":{}},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:59.215335Z","iopub.execute_input":"2023-01-08T22:37:59.215782Z","iopub.status.idle":"2023-01-08T22:37:59.226102Z","shell.execute_reply.started":"2023-01-08T22:37:59.215718Z","shell.execute_reply":"2023-01-08T22:37:59.225107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Pour accéder à une valeur précise, il suffit de tapper la commande iloc**","metadata":{}},{"cell_type":"code","source":"df.iloc[10]","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:59.227993Z","iopub.execute_input":"2023-01-08T22:37:59.228844Z","iopub.status.idle":"2023-01-08T22:37:59.239034Z","shell.execute_reply.started":"2023-01-08T22:37:59.228802Z","shell.execute_reply":"2023-01-08T22:37:59.237644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Chargement de la Data**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")\n\ntrain.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:37:59.241143Z","iopub.execute_input":"2023-01-08T22:37:59.241553Z","iopub.status.idle":"2023-01-08T22:38:11.463895Z","shell.execute_reply.started":"2023-01-08T22:37:59.241513Z","shell.execute_reply":"2023-01-08T22:38:11.462622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Information sur le fichier Train**","metadata":{}},{"cell_type":"code","source":"train.info()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:11.468703Z","iopub.execute_input":"2023-01-08T22:38:11.469056Z","iopub.status.idle":"2023-01-08T22:38:11.584900Z","shell.execute_reply.started":"2023-01-08T22:38:11.469026Z","shell.execute_reply":"2023-01-08T22:38:11.583511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Informations sur le fichier test**","metadata":{}},{"cell_type":"code","source":"test.info()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:11.586599Z","iopub.execute_input":"2023-01-08T22:38:11.587072Z","iopub.status.idle":"2023-01-08T22:38:11.672460Z","shell.execute_reply.started":"2023-01-08T22:38:11.587025Z","shell.execute_reply":"2023-01-08T22:38:11.671649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Vérifier les colonnes manquantes**","metadata":{}},{"cell_type":"code","source":"train.isna().sum().any()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:11.673805Z","iopub.execute_input":"2023-01-08T22:38:11.674880Z","iopub.status.idle":"2023-01-08T22:38:11.767630Z","shell.execute_reply.started":"2023-01-08T22:38:11.674845Z","shell.execute_reply":"2023-01-08T22:38:11.766454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum().any()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:11.769187Z","iopub.execute_input":"2023-01-08T22:38:11.769644Z","iopub.status.idle":"2023-01-08T22:38:11.863257Z","shell.execute_reply.started":"2023-01-08T22:38:11.769606Z","shell.execute_reply":"2023-01-08T22:38:11.861994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualisation des données du fichier test.csv**","metadata":{}},{"cell_type":"code","source":"test.describe()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:11.865191Z","iopub.execute_input":"2023-01-08T22:38:11.865978Z","iopub.status.idle":"2023-01-08T22:38:12.754149Z","shell.execute_reply.started":"2023-01-08T22:38:11.865941Z","shell.execute_reply":"2023-01-08T22:38:12.752908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser le nombre de lignes dans le DataFrame**","metadata":{}},{"cell_type":"code","source":"len(df)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:12.755796Z","iopub.execute_input":"2023-01-08T22:38:12.756180Z","iopub.status.idle":"2023-01-08T22:38:12.763129Z","shell.execute_reply.started":"2023-01-08T22:38:12.756147Z","shell.execute_reply":"2023-01-08T22:38:12.762019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualisation de la Data via des Histogrammes","metadata":{}},{"cell_type":"markdown","source":"**Visualiser la fréquence des valeurs via un histogramme (Distribution de la fréquence des valeurs)**","metadata":{}},{"cell_type":"code","source":"df.f_01.plot.hist()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:12.765227Z","iopub.execute_input":"2023-01-08T22:38:12.765676Z","iopub.status.idle":"2023-01-08T22:38:13.168845Z","shell.execute_reply.started":"2023-01-08T22:38:12.765622Z","shell.execute_reply":"2023-01-08T22:38:13.167663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Ajout de la résolution des barres du histogramme pour une meilleure précision**","metadata":{}},{"cell_type":"code","source":"df.f_01.plot.hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:13.170390Z","iopub.execute_input":"2023-01-08T22:38:13.170779Z","iopub.status.idle":"2023-01-08T22:38:13.598874Z","shell.execute_reply.started":"2023-01-08T22:38:13.170734Z","shell.execute_reply":"2023-01-08T22:38:13.597683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualisation des histogrammes des différents Features**","metadata":{}},{"cell_type":"code","source":"# float columns\ndf.hist(column=[\n    'f_00', 'f_01', \n    'f_02', 'f_03', \n    'f_04', 'f_05', \n    'f_06', 'f_19', \n    'f_20', 'f_21', \n    'f_22', 'f_23', \n    'f_24', 'f_25', \n    'f_26', 'f_28'\n], figsize=(20,15), bins=25);","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:13.600654Z","iopub.execute_input":"2023-01-08T22:38:13.601459Z","iopub.status.idle":"2023-01-08T22:38:16.915390Z","shell.execute_reply.started":"2023-01-08T22:38:13.601408Z","shell.execute_reply":"2023-01-08T22:38:16.914211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Le nombre de lignes dans le Dataframe**","metadata":{}},{"cell_type":"code","source":"len(df)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:16.916657Z","iopub.execute_input":"2023-01-08T22:38:16.917072Z","iopub.status.idle":"2023-01-08T22:38:16.924881Z","shell.execute_reply.started":"2023-01-08T22:38:16.917033Z","shell.execute_reply":"2023-01-08T22:38:16.923712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Traitement de la corrélation entre les valeurs","metadata":{}},{"cell_type":"markdown","source":"**Afficher la corrélation entre les différentes colonnes**","metadata":{}},{"cell_type":"code","source":"df.corr()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:16.926586Z","iopub.execute_input":"2023-01-08T22:38:16.927036Z","iopub.status.idle":"2023-01-08T22:38:19.609906Z","shell.execute_reply.started":"2023-01-08T22:38:16.926994Z","shell.execute_reply":"2023-01-08T22:38:19.608675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2ème méthode de corrélation: Afficher la corrélation entre champs sous forme de graphe (Autrement dit: Plus la couleur est orange clair, plus la valeur de corrélation est plus grande, ce qui veut dire que les valeurs affectées à 1 sont linéairement corrélées à 100%**","metadata":{}},{"cell_type":"code","source":"sns.heatmap(df.corr())","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:19.611636Z","iopub.execute_input":"2023-01-08T22:38:19.612125Z","iopub.status.idle":"2023-01-08T22:38:22.675448Z","shell.execute_reply.started":"2023-01-08T22:38:19.612079Z","shell.execute_reply":"2023-01-08T22:38:22.674327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Afficher les diférentes corrélations avec la colonne f_01, et la plus grande valeur de corrélation persiste avec la colonne f_28**","metadata":{}},{"cell_type":"code","source":"df.corr()['f_01']","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:22.676816Z","iopub.execute_input":"2023-01-08T22:38:22.677140Z","iopub.status.idle":"2023-01-08T22:38:25.310006Z","shell.execute_reply.started":"2023-01-08T22:38:22.677110Z","shell.execute_reply":"2023-01-08T22:38:25.308832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Les valeurs de corrélation avec f_01 qui ont valeur supérieur à 0.001**","metadata":{}},{"cell_type":"code","source":"corr= df.corr()['f_01']\ncorr[corr > 0.001]","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:25.311641Z","iopub.execute_input":"2023-01-08T22:38:25.312786Z","iopub.status.idle":"2023-01-08T22:38:27.952944Z","shell.execute_reply.started":"2023-01-08T22:38:25.312714Z","shell.execute_reply":"2023-01-08T22:38:27.952108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Les colonnes corrélées avec f_01 qui ont valeur supérieur à 0.001**\n","metadata":{}},{"cell_type":"code","source":"corr= df.corr()['f_01']\ncorr[corr > 0.001].index","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:27.954323Z","iopub.execute_input":"2023-01-08T22:38:27.957607Z","iopub.status.idle":"2023-01-08T22:38:30.593216Z","shell.execute_reply.started":"2023-01-08T22:38:27.957571Z","shell.execute_reply":"2023-01-08T22:38:30.592043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Afficher les corrélations qui ont la plus grande valeur, ce qui permet une meilleure visibilité (colonnes avec un très grand signal), Dans cet exemple, on peut remarquer que la colonne f_01 est fortement corrélée avec l'autre colonne f_01 (voir couleur en blanc clair) et moyennement corrélée avec le f_28**","metadata":{}},{"cell_type":"code","source":"corr_mat= df.corr()\ncorr= corr_mat['f_01']\ntop_corr_features = corr[corr.abs() > 0.001].index\nsns.heatmap(corr_mat.loc[top_corr_features, top_corr_features])","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:30.594473Z","iopub.execute_input":"2023-01-08T22:38:30.594920Z","iopub.status.idle":"2023-01-08T22:38:33.608877Z","shell.execute_reply.started":"2023-01-08T22:38:30.594888Z","shell.execute_reply":"2023-01-08T22:38:33.607799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Afficher les valeurs des corrélations supérieurs à 0.001 d'une manière ascendante (du plus grand au plus petit)\nNB: La corrélation avec f_01 est 1 du fait qu'elle est corrélée avec elle-même et on remarque aussi que f_28 est fortement corrélé avec f_01**","metadata":{}},{"cell_type":"code","source":"corr[corr.abs() > 0.001].sort_values(ascending= False)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:33.610358Z","iopub.execute_input":"2023-01-08T22:38:33.610831Z","iopub.status.idle":"2023-01-08T22:38:33.620736Z","shell.execute_reply.started":"2023-01-08T22:38:33.610788Z","shell.execute_reply":"2023-01-08T22:38:33.619812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Cette méthode permet aussi de visualiser la forte corrélation entre f_01 et f_28, on a utilisé la valeur afin de mieux visualiser la répartition des points d'une manière transparente**","metadata":{}},{"cell_type":"code","source":"df.plot.scatter(x='f_01', y='f_28', alpha=0.25)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:33.622226Z","iopub.execute_input":"2023-01-08T22:38:33.622919Z","iopub.status.idle":"2023-01-08T22:38:37.027222Z","shell.execute_reply.started":"2023-01-08T22:38:33.622887Z","shell.execute_reply":"2023-01-08T22:38:37.026157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.target.plot.hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:37.033463Z","iopub.execute_input":"2023-01-08T22:38:37.033801Z","iopub.status.idle":"2023-01-08T22:38:37.503232Z","shell.execute_reply.started":"2023-01-08T22:38:37.033771Z","shell.execute_reply":"2023-01-08T22:38:37.502072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualisation de la corrélation entre les deux colonnes f_26 et f_01**","metadata":{}},{"cell_type":"code","source":"df.plot.scatter(x='f_26', y='f_01')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:37.504483Z","iopub.execute_input":"2023-01-08T22:38:37.504987Z","iopub.status.idle":"2023-01-08T22:38:40.714737Z","shell.execute_reply.started":"2023-01-08T22:38:37.504956Z","shell.execute_reply":"2023-01-08T22:38:40.713636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Autre manière de visualisation de la corrélation avec la distrbution de chaque feature avec une densité en Hexadécimale afin de mieux visualiser les points de corrélation**","metadata":{}},{"cell_type":"code","source":"sns.jointplot(df.f_26, df.f_01, kind='hex')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:40.716141Z","iopub.execute_input":"2023-01-08T22:38:40.716475Z","iopub.status.idle":"2023-01-08T22:38:43.341405Z","shell.execute_reply.started":"2023-01-08T22:38:40.716444Z","shell.execute_reply":"2023-01-08T22:38:43.340119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-Processing","metadata":{}},{"cell_type":"markdown","source":"**A partir de cet Histogramme, on peut remarquer qu'il y a des points extrêmes qui peuvent influencer notre modèle et qui sont dans les alentoures de la valeur 4 (valeurs anormales)**","metadata":{}},{"cell_type":"code","source":"df.f_01.plot.hist(bins=50)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:43.342718Z","iopub.execute_input":"2023-01-08T22:38:43.343058Z","iopub.status.idle":"2023-01-08T22:38:43.793501Z","shell.execute_reply.started":"2023-01-08T22:38:43.343029Z","shell.execute_reply":"2023-01-08T22:38:43.792104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**On a rajouté une fonction logarithme afin de rapprocher au mieux les points extrêmes**","metadata":{}},{"cell_type":"code","source":"df.f_01.apply(np.log).plot.hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:43.795198Z","iopub.execute_input":"2023-01-08T22:38:43.795527Z","iopub.status.idle":"2023-01-08T22:38:44.294282Z","shell.execute_reply.started":"2023-01-08T22:38:43.795498Z","shell.execute_reply":"2023-01-08T22:38:44.293211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Permet d'afficher que les valeurs qui ne sont pas numériques**","metadata":{}},{"cell_type":"code","source":"df.select_dtypes(include=['object'])","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:44.296047Z","iopub.execute_input":"2023-01-08T22:38:44.297039Z","iopub.status.idle":"2023-01-08T22:38:44.325276Z","shell.execute_reply.started":"2023-01-08T22:38:44.296995Z","shell.execute_reply":"2023-01-08T22:38:44.324065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**La commande suivante permet de visualiser les valeurs non manquantes (False pour les valeurs manquantes)**","metadata":{}},{"cell_type":"code","source":"df.isna()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:44.326612Z","iopub.execute_input":"2023-01-08T22:38:44.326967Z","iopub.status.idle":"2023-01-08T22:38:44.458972Z","shell.execute_reply.started":"2023-01-08T22:38:44.326937Z","shell.execute_reply":"2023-01-08T22:38:44.457814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**A partir de la, on remarque qu'il n' y a pas de valeurs inconnus vu que les valeurs sont de l'ordre de 0. Dans le cas échéant, il faut calculer la médiane de chaque valeur**","metadata":{}},{"cell_type":"code","source":"df.isna().sum(axis=0).sort_values(ascending= False)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:44.460337Z","iopub.execute_input":"2023-01-08T22:38:44.461374Z","iopub.status.idle":"2023-01-08T22:38:44.556169Z","shell.execute_reply.started":"2023-01-08T22:38:44.461321Z","shell.execute_reply":"2023-01-08T22:38:44.554885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# The string feature f_27\n\n","metadata":{}},{"cell_type":"markdown","source":"**f_27 est une fonctionnalité de chaîne qui ne peut pas être utilisée telle quelle. Découvrons comment en tirer quelque chose d'utile !\nNous vérifions d'abord que la chaîne a toujours une longueur de 10**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")\ntrain.f_27.str.len().min(), train.f_27.str.len().max(), test.f_27.str.len().min(), test.f_27.str.len().max()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:44.557636Z","iopub.execute_input":"2023-01-08T22:38:44.557986Z","iopub.status.idle":"2023-01-08T22:38:53.358221Z","shell.execute_reply.started":"2023-01-08T22:38:44.557956Z","shell.execute_reply":"2023-01-08T22:38:53.356948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n**Les échantillons de train contiennent 741 354 valeurs différentes, c'est-à-dire que la plupart des chaînes sont différentes. La chaîne la plus fréquente, 'BBBBBBCJBC' n'apparaît que 12 fois :**","metadata":{}},{"cell_type":"code","source":"train.f_27.value_counts()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:53.359998Z","iopub.execute_input":"2023-01-08T22:38:53.362141Z","iopub.status.idle":"2023-01-08T22:38:54.254366Z","shell.execute_reply.started":"2023-01-08T22:38:53.362092Z","shell.execute_reply":"2023-01-08T22:38:54.253217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Il est important de comprendre si les chaînes f_27 dans le test sont les mêmes. Malheureusement, le test contient 1181880 - 741354 = 440526 chaînes qui ne se reproduisent pas.**\n\n**NB : Nous ne devons pas utiliser cette chaîne comme caractéristique catégorique dans un classificateur. Sinon, le modèle apprend à s'appuyer sur des chaînes qui n'apparaissent jamais dans les données de test.**","metadata":{}},{"cell_type":"code","source":"pd.concat([train, test]).f_27.value_counts()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:54.255980Z","iopub.execute_input":"2023-01-08T22:38:54.256715Z","iopub.status.idle":"2023-01-08T22:38:55.977684Z","shell.execute_reply.started":"2023-01-08T22:38:54.256669Z","shell.execute_reply":"2023-01-08T22:38:55.976501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Dans l'étape suivante, nous examinons les distributions des lettres à chacune des dix positions de la chaîne. On voit que les positions 0, 2 et 5 sont binaires ; les autres positions ont plus de valeurs. Chaque position donne des informations sur la cible (la moyenne de la cible dépend de la valeur de la caractéristique).**","metadata":{}},{"cell_type":"code","source":"for i in range(10):\n    print(f'Position {i}')\n    tg = train.groupby(train.f_27.str.get(i))\n    temp = pd.DataFrame({'size': tg.size(), 'probability': tg.target.mean().round(2)})\n    print(temp)\n    print()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:38:55.979685Z","iopub.execute_input":"2023-01-08T22:38:55.980473Z","iopub.status.idle":"2023-01-08T22:39:01.517802Z","shell.execute_reply.started":"2023-01-08T22:38:55.980420Z","shell.execute_reply":"2023-01-08T22:39:01.516629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# Feature Engineering","metadata":{}},{"cell_type":"markdown","source":"**Nous allons lire les données et appliquer une ingénierie minimale des fonctionnalités : nous allons diviser uniquement la chaîne f_27 en dix fonctionnalités distinctes et nous allons compter les caractères uniques dans la chaîne.**","metadata":{}},{"cell_type":"code","source":"for df in [train, test]:\n    for i in range(10):\n        df[f'ch{i}'] = df.f_27.str.get(i).apply(ord) - ord('A')\n    df[\"unique_characters\"] = df.f_27.apply(lambda s: len(set(s)))\nfeatures = [f for f in test.columns if f != 'id' and f != 'f_27']\ntest[features].head(2)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:01.519225Z","iopub.execute_input":"2023-01-08T22:39:01.519628Z","iopub.status.idle":"2023-01-08T22:39:15.874070Z","shell.execute_reply.started":"2023-01-08T22:39:01.519598Z","shell.execute_reply":"2023-01-08T22:39:15.873037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Modelisation","metadata":{}},{"cell_type":"markdown","source":"**Visualisation d'un dataframe avec 3 lignes**","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:15.875639Z","iopub.execute_input":"2023-01-08T22:39:15.876113Z","iopub.status.idle":"2023-01-08T22:39:15.995742Z","shell.execute_reply.started":"2023-01-08T22:39:15.876069Z","shell.execute_reply":"2023-01-08T22:39:15.994821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train[\"target\"]\nX = train.drop(columns=[\"id\",\"f_27\", \"target\"])\n\ntest = test.drop(columns=[\"id\",\"f_27\"])\n\nX.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:15.997120Z","iopub.execute_input":"2023-01-08T22:39:15.997546Z","iopub.status.idle":"2023-01-08T22:39:16.301701Z","shell.execute_reply.started":"2023-01-08T22:39:15.997503Z","shell.execute_reply":"2023-01-08T22:39:16.300559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=0.33, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:16.303802Z","iopub.execute_input":"2023-01-08T22:39:16.304279Z","iopub.status.idle":"2023-01-08T22:39:16.861456Z","shell.execute_reply.started":"2023-01-08T22:39:16.304234Z","shell.execute_reply":"2023-01-08T22:39:16.860277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\n\nimport time","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:16.862798Z","iopub.execute_input":"2023-01-08T22:39:16.863131Z","iopub.status.idle":"2023-01-08T22:39:17.698748Z","shell.execute_reply.started":"2023-01-08T22:39:16.863101Z","shell.execute_reply":"2023-01-08T22:39:17.697573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optuna.logging.set_verbosity(optuna.logging.WARNING)\n\ndef objectivesLGBM(trial):\n    model = LGBMClassifier()\n    model.fit(X,y)\n\n    return model.score(X,y)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:17.700662Z","iopub.execute_input":"2023-01-08T22:39:17.701050Z","iopub.status.idle":"2023-01-08T22:39:17.706413Z","shell.execute_reply.started":"2023-01-08T22:39:17.701018Z","shell.execute_reply":"2023-01-08T22:39:17.705062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LGBMClassifier()\nmodel.fit(X, y)\n\nprint(\"Training score :\", model.score(X, y))\n\npred_y_test = model.predict(X_test)\nprint(\"Roc auc score  :\", roc_auc_score(y_test, pred_y_test))","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:17.708366Z","iopub.execute_input":"2023-01-08T22:39:17.708939Z","iopub.status.idle":"2023-01-08T22:39:38.848827Z","shell.execute_reply.started":"2023-01-08T22:39:17.708898Z","shell.execute_reply":"2023-01-08T22:39:38.847557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n# sorted(zip(clf.feature_importances_, X.columns), reverse=True)\nfeature_imp = pd.DataFrame(sorted(zip(model.feature_importances_,X.columns)), columns=['Value','Feature'])\n\nplt.figure(figsize=(20, 10))\nsns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.show()\nplt.savefig('lgbm_importances-01.png')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:38.850194Z","iopub.execute_input":"2023-01-08T22:39:38.850565Z","iopub.status.idle":"2023-01-08T22:39:39.723669Z","shell.execute_reply.started":"2023-01-08T22:39:38.850534Z","shell.execute_reply":"2023-01-08T22:39:39.722494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_imp = feature_imp.sort_values(by = \"Value\", ascending=False)\nfeature_imp","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:39.725224Z","iopub.execute_input":"2023-01-08T22:39:39.725545Z","iopub.status.idle":"2023-01-08T22:39:39.738809Z","shell.execute_reply.started":"2023-01-08T22:39:39.725517Z","shell.execute_reply":"2023-01-08T22:39:39.737693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = feature_imp[:6]['Feature'].tolist()\nselected_features","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:39.740319Z","iopub.execute_input":"2023-01-08T22:39:39.740794Z","iopub.status.idle":"2023-01-08T22:39:39.749467Z","shell.execute_reply.started":"2023-01-08T22:39:39.740731Z","shell.execute_reply":"2023-01-08T22:39:39.748291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"kf = StratifiedKFold(n_splits=14, shuffle=True, random_state = 0)\ny_predict = []\n\nfor fold, (train_index, val_index) in enumerate(kf.split(X, y)):\n        print('*'*14, f\" Fold {fold} \", '*'*14, '\\n')\n        \n        # Split Data\n        X_train = X.loc[train_index]\n        X_val = X.loc[val_index]\n\n        y_train = y.loc[train_index]\n        y_val = y.loc[val_index]\n        \n        # Create Model here\n        model = LGBMClassifier()\n        \n        # Fit Model\n        model.fit(X_train,y_train)\n        \n          # Make X_val prediction\n        y_pred = model.predict_proba(X_val)[:,1]\n        \n        # Make Test prediction\n        y_predict.append(model.predict_proba(test)[:,1])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:39:39.751358Z","iopub.execute_input":"2023-01-08T22:39:39.752103Z","iopub.status.idle":"2023-01-08T22:44:13.082899Z","shell.execute_reply.started":"2023-01-08T22:39:39.752058Z","shell.execute_reply":"2023-01-08T22:44:13.081931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.sample(3)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:13.084581Z","iopub.execute_input":"2023-01-08T22:44:13.085380Z","iopub.status.idle":"2023-01-08T22:44:13.134858Z","shell.execute_reply.started":"2023-01-08T22:44:13.085334Z","shell.execute_reply":"2023-01-08T22:44:13.133648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TRAIN DATA","metadata":{}},{"cell_type":"markdown","source":"**Examinons d'abord les valeurs statistiques de chaque fonctionnalité.**","metadata":{}},{"cell_type":"code","source":"\n\ntrain.describe().T.style.bar(subset=['mean'], color='#205ff2')\\\n                            .background_gradient(subset=['std'], cmap='Reds')\\\n                            .background_gradient(subset=['50%'], cmap='coolwarm')\n\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:13.136582Z","iopub.execute_input":"2023-01-08T22:44:13.137021Z","iopub.status.idle":"2023-01-08T22:44:14.701966Z","shell.execute_reply.started":"2023-01-08T22:44:13.136983Z","shell.execute_reply":"2023-01-08T22:44:14.700806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Test Data**","metadata":{}},{"cell_type":"code","source":"test.describe().T.style.bar(subset=['mean'], color='#205ff2')\\\n                            .background_gradient(subset=['std'], cmap='Reds')\\\n                            .background_gradient(subset=['50%'], cmap='coolwarm')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:14.703366Z","iopub.execute_input":"2023-01-08T22:44:14.703666Z","iopub.status.idle":"2023-01-08T22:44:15.816809Z","shell.execute_reply.started":"2023-01-08T22:44:14.703639Z","shell.execute_reply":"2023-01-08T22:44:15.815383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train[\"target\"]\nX = train.drop(columns=[\"id\",\"f_27\", \"target\"])\n\ntest = test.drop(columns=[\"id\",\"f_27\"])\n\nX.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:15.818478Z","iopub.execute_input":"2023-01-08T22:44:15.818831Z","iopub.status.idle":"2023-01-08T22:44:16.092430Z","shell.execute_reply.started":"2023-01-08T22:44:15.818798Z","shell.execute_reply":"2023-01-08T22:44:16.090912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=0.33, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:16.093889Z","iopub.status.idle":"2023-01-08T22:44:16.094385Z","shell.execute_reply.started":"2023-01-08T22:44:16.094183Z","shell.execute_reply":"2023-01-08T22:44:16.094203Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LGBMClassifier()\n\nmodel.fit(X, y)\n\ny_predict.append(model.predict_proba(test)[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:16.096219Z","iopub.status.idle":"2023-01-08T22:44:16.096623Z","shell.execute_reply.started":"2023-01-08T22:44:16.096432Z","shell.execute_reply":"2023-01-08T22:44:16.096451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model, X, y, test, X_test, y_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:16.097728Z","iopub.status.idle":"2023-01-08T22:44:16.098160Z","shell.execute_reply.started":"2023-01-08T22:44:16.097956Z","shell.execute_reply":"2023-01-08T22:44:16.097975Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.array(y_predict).mean(axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:16.099493Z","iopub.status.idle":"2023-01-08T22:44:16.099889Z","shell.execute_reply.started":"2023-01-08T22:44:16.099670Z","shell.execute_reply":"2023-01-08T22:44:16.099687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/sample_submission.csv\")\nsubmission.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:16.101814Z","iopub.status.idle":"2023-01-08T22:44:16.102185Z","shell.execute_reply.started":"2023-01-08T22:44:16.101996Z","shell.execute_reply":"2023-01-08T22:44:16.102012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['target'] = np.array(y_predict).mean(axis=0)\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-08T22:44:16.104052Z","iopub.status.idle":"2023-01-08T22:44:16.104458Z","shell.execute_reply.started":"2023-01-08T22:44:16.104274Z","shell.execute_reply":"2023-01-08T22:44:16.104291Z"},"trusted":true},"execution_count":null,"outputs":[]}]}