{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to loadil\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns # librairie de corrélation\n#import matplotlib as mpl\n#import matplotlib.pyplot as plt\n#import random\n#import tensorflow as tf\n\n\n#import tensorflow.keras as keras\n#from tensorflow.keras.models import Sequential\n#from tensorflow.keras.layers import Dense, Dropout, BatchNormalization\n#from tensorflow.keras import backend as K\n#from tensorflow.keras.utils import to_categorical\n#from sklearn.model_selection import train_test_split\n#from sklearn.metrics import confusion_matrix\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":1.097375,"end_time":"2023-01-08T19:43:36.963702","exception":false,"start_time":"2023-01-08T19:43:35.866327","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:33.609035Z","iopub.execute_input":"2023-01-08T20:22:33.609455Z","iopub.status.idle":"2023-01-08T20:22:33.618859Z","shell.execute_reply.started":"2023-01-08T20:22:33.609420Z","shell.execute_reply":"2023-01-08T20:22:33.617780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Méthodes de Traitement de la Data","metadata":{"papermill":{"duration":0.016522,"end_time":"2023-01-08T19:43:36.998588","exception":false,"start_time":"2023-01-08T19:43:36.982066","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Visualiser les colonnes du fichier Train.csv séparés par une virgule**","metadata":{"papermill":{"duration":0.016861,"end_time":"2023-01-08T19:43:37.032897","exception":false,"start_time":"2023-01-08T19:43:37.016036","status":"completed"},"tags":[]}},{"cell_type":"code","source":"open('/kaggle/input/emsi-5iir-ds/train.csv').readlines()[0]\n","metadata":{"papermill":{"duration":4.233808,"end_time":"2023-01-08T19:43:41.283511","exception":false,"start_time":"2023-01-08T19:43:37.049703","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:33.847325Z","iopub.execute_input":"2023-01-08T20:22:33.847746Z","iopub.status.idle":"2023-01-08T20:22:34.481801Z","shell.execute_reply.started":"2023-01-08T20:22:33.847708Z","shell.execute_reply":"2023-01-08T20:22:34.480515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser les  premières valeurs des colonnes du fichier Train.csv séparés par une virgule**\n","metadata":{"papermill":{"duration":0.016754,"end_time":"2023-01-08T19:43:41.317395","exception":false,"start_time":"2023-01-08T19:43:41.300641","status":"completed"},"tags":[]}},{"cell_type":"code","source":"open('/kaggle/input/emsi-5iir-ds/train.csv').readlines()[1]\n","metadata":{"papermill":{"duration":0.666549,"end_time":"2023-01-08T19:43:42.000669","exception":false,"start_time":"2023-01-08T19:43:41.334120","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:34.483984Z","iopub.execute_input":"2023-01-08T20:22:34.484525Z","iopub.status.idle":"2023-01-08T20:22:35.108045Z","shell.execute_reply.started":"2023-01-08T20:22:34.484474Z","shell.execute_reply":"2023-01-08T20:22:35.106869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser toutes les colonnes et valeurs du fichier train.csv**","metadata":{"papermill":{"duration":0.016745,"end_time":"2023-01-08T19:43:42.034438","exception":false,"start_time":"2023-01-08T19:43:42.017693","status":"completed"},"tags":[]}},{"cell_type":"code","source":"pd.read_csv('/kaggle/input/emsi-5iir-ds/train.csv')","metadata":{"papermill":{"duration":4.9314,"end_time":"2023-01-08T19:43:46.982599","exception":false,"start_time":"2023-01-08T19:43:42.051199","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:35.109908Z","iopub.execute_input":"2023-01-08T20:22:35.110527Z","iopub.status.idle":"2023-01-08T20:22:40.538593Z","shell.execute_reply.started":"2023-01-08T20:22:35.110491Z","shell.execute_reply":"2023-01-08T20:22:40.537429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser 10 colonnes aléatoires et leurs valeurs du fichier Train.csv**","metadata":{"papermill":{"duration":0.01769,"end_time":"2023-01-08T19:43:47.018396","exception":false,"start_time":"2023-01-08T19:43:47.000706","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df= pd.read_csv('/kaggle/input/emsi-5iir-ds/train.csv')\ndf.sample(10)","metadata":{"papermill":{"duration":4.6482,"end_time":"2023-01-08T19:43:51.684289","exception":false,"start_time":"2023-01-08T19:43:47.036089","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:40.541898Z","iopub.execute_input":"2023-01-08T20:22:40.542578Z","iopub.status.idle":"2023-01-08T20:22:45.697375Z","shell.execute_reply.started":"2023-01-08T20:22:40.542529Z","shell.execute_reply":"2023-01-08T20:22:45.696228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser les 5 premières colonnes et leurs valeurs**","metadata":{"papermill":{"duration":0.017568,"end_time":"2023-01-08T19:43:51.720425","exception":false,"start_time":"2023-01-08T19:43:51.702857","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.head(5)","metadata":{"papermill":{"duration":0.046783,"end_time":"2023-01-08T19:43:51.785258","exception":false,"start_time":"2023-01-08T19:43:51.738475","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:45.698510Z","iopub.execute_input":"2023-01-08T20:22:45.698845Z","iopub.status.idle":"2023-01-08T20:22:45.725772Z","shell.execute_reply.started":"2023-01-08T20:22:45.698817Z","shell.execute_reply":"2023-01-08T20:22:45.724456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser l'intitulé des différentes colonnes**\n","metadata":{"papermill":{"duration":0.018305,"end_time":"2023-01-08T19:43:51.822397","exception":false,"start_time":"2023-01-08T19:43:51.804092","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.columns","metadata":{"papermill":{"duration":0.028895,"end_time":"2023-01-08T19:43:51.869822","exception":false,"start_time":"2023-01-08T19:43:51.840927","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:45.727331Z","iopub.execute_input":"2023-01-08T20:22:45.728135Z","iopub.status.idle":"2023-01-08T20:22:45.739324Z","shell.execute_reply.started":"2023-01-08T20:22:45.728087Z","shell.execute_reply":"2023-01-08T20:22:45.738255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Pour accéder à une valeur précise, il suffit de tapper la commande iloc**","metadata":{"papermill":{"duration":0.018811,"end_time":"2023-01-08T19:43:51.907634","exception":false,"start_time":"2023-01-08T19:43:51.888823","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.iloc[10]","metadata":{"papermill":{"duration":0.030658,"end_time":"2023-01-08T19:43:51.956958","exception":false,"start_time":"2023-01-08T19:43:51.926300","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:45.740984Z","iopub.execute_input":"2023-01-08T20:22:45.741310Z","iopub.status.idle":"2023-01-08T20:22:45.754876Z","shell.execute_reply.started":"2023-01-08T20:22:45.741283Z","shell.execute_reply":"2023-01-08T20:22:45.753990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Chargement de la Data**","metadata":{"papermill":{"duration":0.017965,"end_time":"2023-01-08T19:43:51.993540","exception":false,"start_time":"2023-01-08T19:43:51.975575","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")\n\ntrain.shape, test.shape","metadata":{"papermill":{"duration":11.405212,"end_time":"2023-01-08T19:44:03.418020","exception":false,"start_time":"2023-01-08T19:43:52.012808","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:45.756214Z","iopub.execute_input":"2023-01-08T20:22:45.757350Z","iopub.status.idle":"2023-01-08T20:22:54.763008Z","shell.execute_reply.started":"2023-01-08T20:22:45.757311Z","shell.execute_reply":"2023-01-08T20:22:54.761820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Information sur le fichier Train**","metadata":{"papermill":{"duration":0.018571,"end_time":"2023-01-08T19:44:03.456080","exception":false,"start_time":"2023-01-08T19:44:03.437509","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train.info()\n","metadata":{"papermill":{"duration":0.136191,"end_time":"2023-01-08T19:44:03.611089","exception":false,"start_time":"2023-01-08T19:44:03.474898","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:54.764443Z","iopub.execute_input":"2023-01-08T20:22:54.765760Z","iopub.status.idle":"2023-01-08T20:22:54.870659Z","shell.execute_reply.started":"2023-01-08T20:22:54.765712Z","shell.execute_reply":"2023-01-08T20:22:54.869511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Informations sur le fichier test**","metadata":{"papermill":{"duration":0.018967,"end_time":"2023-01-08T19:44:03.650533","exception":false,"start_time":"2023-01-08T19:44:03.631566","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test.info()\n","metadata":{"papermill":{"duration":0.105312,"end_time":"2023-01-08T19:44:03.774968","exception":false,"start_time":"2023-01-08T19:44:03.669656","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:54.875022Z","iopub.execute_input":"2023-01-08T20:22:54.875416Z","iopub.status.idle":"2023-01-08T20:22:54.958903Z","shell.execute_reply.started":"2023-01-08T20:22:54.875380Z","shell.execute_reply":"2023-01-08T20:22:54.957720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Vérifier les colonnes manquantes**","metadata":{"papermill":{"duration":0.018956,"end_time":"2023-01-08T19:44:03.813331","exception":false,"start_time":"2023-01-08T19:44:03.794375","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train.isna().sum().any()\n","metadata":{"papermill":{"duration":0.11526,"end_time":"2023-01-08T19:44:03.948437","exception":false,"start_time":"2023-01-08T19:44:03.833177","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:54.960901Z","iopub.execute_input":"2023-01-08T20:22:54.961377Z","iopub.status.idle":"2023-01-08T20:22:55.055118Z","shell.execute_reply.started":"2023-01-08T20:22:54.961331Z","shell.execute_reply":"2023-01-08T20:22:55.053997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum().any()\n","metadata":{"papermill":{"duration":0.114454,"end_time":"2023-01-08T19:44:04.082001","exception":false,"start_time":"2023-01-08T19:44:03.967547","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:55.057651Z","iopub.execute_input":"2023-01-08T20:22:55.057993Z","iopub.status.idle":"2023-01-08T20:22:55.151497Z","shell.execute_reply.started":"2023-01-08T20:22:55.057964Z","shell.execute_reply":"2023-01-08T20:22:55.150338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualisation des données du fichier test.csv**","metadata":{"papermill":{"duration":0.018878,"end_time":"2023-01-08T19:44:04.119940","exception":false,"start_time":"2023-01-08T19:44:04.101062","status":"completed"},"tags":[]}},{"cell_type":"code","source":"test.describe()\n","metadata":{"papermill":{"duration":0.897185,"end_time":"2023-01-08T19:44:05.036313","exception":false,"start_time":"2023-01-08T19:44:04.139128","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:55.153212Z","iopub.execute_input":"2023-01-08T20:22:55.153715Z","iopub.status.idle":"2023-01-08T20:22:56.057592Z","shell.execute_reply.started":"2023-01-08T20:22:55.153668Z","shell.execute_reply":"2023-01-08T20:22:56.056669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Visualiser le nombre de lignes dans le DataFrame**","metadata":{"papermill":{"duration":0.019095,"end_time":"2023-01-08T19:44:05.075259","exception":false,"start_time":"2023-01-08T19:44:05.056164","status":"completed"},"tags":[]}},{"cell_type":"code","source":"len(df)","metadata":{"papermill":{"duration":0.029809,"end_time":"2023-01-08T19:44:05.124464","exception":false,"start_time":"2023-01-08T19:44:05.094655","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:56.058929Z","iopub.execute_input":"2023-01-08T20:22:56.059276Z","iopub.status.idle":"2023-01-08T20:22:56.066651Z","shell.execute_reply.started":"2023-01-08T20:22:56.059244Z","shell.execute_reply":"2023-01-08T20:22:56.065574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Visualisation de la Data via des Histogrammes","metadata":{"papermill":{"duration":0.019281,"end_time":"2023-01-08T19:44:05.163442","exception":false,"start_time":"2023-01-08T19:44:05.144161","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Visualiser la fréquence des valeurs via un histogramme (Distribution de la fréquence des valeurs)**","metadata":{"papermill":{"duration":0.019249,"end_time":"2023-01-08T19:44:05.202268","exception":false,"start_time":"2023-01-08T19:44:05.183019","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.f_01.plot.hist()","metadata":{"papermill":{"duration":0.390717,"end_time":"2023-01-08T19:44:05.612536","exception":false,"start_time":"2023-01-08T19:44:05.221819","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:56.068615Z","iopub.execute_input":"2023-01-08T20:22:56.069645Z","iopub.status.idle":"2023-01-08T20:22:56.476415Z","shell.execute_reply.started":"2023-01-08T20:22:56.069595Z","shell.execute_reply":"2023-01-08T20:22:56.475341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Ajout de la résolution des barres du histogramme pour une meilleure précision**","metadata":{"papermill":{"duration":0.019532,"end_time":"2023-01-08T19:44:05.652144","exception":false,"start_time":"2023-01-08T19:44:05.632612","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.f_01.plot.hist(bins=50)","metadata":{"papermill":{"duration":0.454696,"end_time":"2023-01-08T19:44:06.126741","exception":false,"start_time":"2023-01-08T19:44:05.672045","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:56.477951Z","iopub.execute_input":"2023-01-08T20:22:56.478402Z","iopub.status.idle":"2023-01-08T20:22:56.922652Z","shell.execute_reply.started":"2023-01-08T20:22:56.478355Z","shell.execute_reply":"2023-01-08T20:22:56.921326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Le nombre de lignes dans le Dataframe**","metadata":{"papermill":{"duration":0.020993,"end_time":"2023-01-08T19:44:09.593472","exception":false,"start_time":"2023-01-08T19:44:09.572479","status":"completed"},"tags":[]}},{"cell_type":"code","source":"len(df)","metadata":{"papermill":{"duration":0.030652,"end_time":"2023-01-08T19:44:09.645345","exception":false,"start_time":"2023-01-08T19:44:09.614693","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:56.924458Z","iopub.execute_input":"2023-01-08T20:22:56.924822Z","iopub.status.idle":"2023-01-08T20:22:56.932458Z","shell.execute_reply.started":"2023-01-08T20:22:56.924790Z","shell.execute_reply":"2023-01-08T20:22:56.931185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Traitement de la corrélation entre les valeurs","metadata":{"papermill":{"duration":0.021008,"end_time":"2023-01-08T19:44:09.687649","exception":false,"start_time":"2023-01-08T19:44:09.666641","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Afficher la corrélation entre les différentes colonnes**","metadata":{"papermill":{"duration":0.021025,"end_time":"2023-01-08T19:44:09.729963","exception":false,"start_time":"2023-01-08T19:44:09.708938","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.corr()","metadata":{"papermill":{"duration":2.743314,"end_time":"2023-01-08T19:44:12.494586","exception":false,"start_time":"2023-01-08T19:44:09.751272","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:56.934214Z","iopub.execute_input":"2023-01-08T20:22:56.934900Z","iopub.status.idle":"2023-01-08T20:22:59.635991Z","shell.execute_reply.started":"2023-01-08T20:22:56.934861Z","shell.execute_reply":"2023-01-08T20:22:59.634896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Afficher les diférentes corrélations avec la colonne f_01, et la plus grande valeur de corrélation persiste avec la colonne f_28**","metadata":{"papermill":{"duration":0.022802,"end_time":"2023-01-08T19:44:15.713022","exception":false,"start_time":"2023-01-08T19:44:15.690220","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.corr()['f_01']","metadata":{"papermill":{"duration":2.679562,"end_time":"2023-01-08T19:44:18.416712","exception":false,"start_time":"2023-01-08T19:44:15.737150","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:22:59.637296Z","iopub.execute_input":"2023-01-08T20:22:59.637633Z","iopub.status.idle":"2023-01-08T20:23:02.296430Z","shell.execute_reply.started":"2023-01-08T20:22:59.637604Z","shell.execute_reply":"2023-01-08T20:23:02.295045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Les valeurs de corrélation avec f_01 qui ont valeur supérieur à 0.001**","metadata":{"papermill":{"duration":0.02269,"end_time":"2023-01-08T19:44:18.462876","exception":false,"start_time":"2023-01-08T19:44:18.440186","status":"completed"},"tags":[]}},{"cell_type":"code","source":"corr= df.corr()['f_01']\ncorr[corr > 0.001]","metadata":{"papermill":{"duration":2.691258,"end_time":"2023-01-08T19:44:21.177245","exception":false,"start_time":"2023-01-08T19:44:18.485987","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:23:02.298602Z","iopub.execute_input":"2023-01-08T20:23:02.299068Z","iopub.status.idle":"2023-01-08T20:23:04.969123Z","shell.execute_reply.started":"2023-01-08T20:23:02.299025Z","shell.execute_reply":"2023-01-08T20:23:04.967931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Les colonnes corrélées avec f_01 qui ont valeur supérieur à 0.001**\n","metadata":{"papermill":{"duration":0.022995,"end_time":"2023-01-08T19:44:21.223601","exception":false,"start_time":"2023-01-08T19:44:21.200606","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**Afficher les corrélations qui ont la plus grande valeur, ce qui permet une meilleure visibilité (colonnes avec un très grand signal), Dans cet exemple, on peut remarquer que la colonne f_01 est fortement corrélée avec l'autre colonne f_01 (voir couleur en blanc clair) et moyennement corrélée avec le f_28**","metadata":{"papermill":{"duration":0.023771,"end_time":"2023-01-08T19:44:24.029687","exception":false,"start_time":"2023-01-08T19:44:24.005916","status":"completed"},"tags":[]}},{"cell_type":"code","source":"corr_mat= df.corr()\ncorr= corr_mat['f_01']\ntop_corr_features = corr[corr.abs() > 0.001].index\nsns.heatmap(corr_mat.loc[top_corr_features, top_corr_features])","metadata":{"papermill":{"duration":3.027311,"end_time":"2023-01-08T19:44:27.080953","exception":false,"start_time":"2023-01-08T19:44:24.053642","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:23:04.970853Z","iopub.execute_input":"2023-01-08T20:23:04.971181Z","iopub.status.idle":"2023-01-08T20:23:07.996495Z","shell.execute_reply.started":"2023-01-08T20:23:04.971152Z","shell.execute_reply":"2023-01-08T20:23:07.995624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Afficher les valeurs des corrélations supérieurs à 0.001 d'une manière ascendante (du plus grand au plus petit)\nNB: La corrélation avec f_01 est 1 du fait qu'elle est corrélée avec elle-même et on remarque aussi que f_28 est fortement corrélé avec f_01**","metadata":{"papermill":{"duration":0.024143,"end_time":"2023-01-08T19:44:27.129340","exception":false,"start_time":"2023-01-08T19:44:27.105197","status":"completed"},"tags":[]}},{"cell_type":"code","source":"corr[corr.abs() > 0.001].sort_values(ascending= False)\n","metadata":{"papermill":{"duration":0.035976,"end_time":"2023-01-08T19:44:27.190254","exception":false,"start_time":"2023-01-08T19:44:27.154278","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:23:07.997989Z","iopub.execute_input":"2023-01-08T20:23:08.001980Z","iopub.status.idle":"2023-01-08T20:23:08.012417Z","shell.execute_reply.started":"2023-01-08T20:23:08.001933Z","shell.execute_reply":"2023-01-08T20:23:08.010935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.target.plot.hist(bins=100)","metadata":{"papermill":{"duration":0.509388,"end_time":"2023-01-08T19:44:31.295968","exception":false,"start_time":"2023-01-08T19:44:30.786580","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:23:08.014216Z","iopub.execute_input":"2023-01-08T20:23:08.014742Z","iopub.status.idle":"2023-01-08T20:23:08.522495Z","shell.execute_reply.started":"2023-01-08T20:23:08.014697Z","shell.execute_reply":"2023-01-08T20:23:08.521658Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pre-Processing","metadata":{"papermill":{"duration":0.026016,"end_time":"2023-01-08T19:44:37.458575","exception":false,"start_time":"2023-01-08T19:44:37.432559","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"**A partir de cet Histogramme, on peut remarquer qu'il y a des points extrêmes qui peuvent influencer notre modèle et qui sont dans les alentoures de la valeur 4 (valeurs anormales)**","metadata":{"papermill":{"duration":0.025813,"end_time":"2023-01-08T19:44:37.510584","exception":false,"start_time":"2023-01-08T19:44:37.484771","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.f_01.plot.hist(bins=50)","metadata":{"papermill":{"duration":0.472533,"end_time":"2023-01-08T19:44:38.010092","exception":false,"start_time":"2023-01-08T19:44:37.537559","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:23:08.523937Z","iopub.execute_input":"2023-01-08T20:23:08.524643Z","iopub.status.idle":"2023-01-08T20:23:08.969319Z","shell.execute_reply.started":"2023-01-08T20:23:08.524598Z","shell.execute_reply":"2023-01-08T20:23:08.968086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**On a rajouté une fonction logarithme afin de rapprocher au mieux les points extrêmes**","metadata":{"papermill":{"duration":0.026571,"end_time":"2023-01-08T19:44:38.063587","exception":false,"start_time":"2023-01-08T19:44:38.037016","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.select_dtypes(include=['object'])","metadata":{"papermill":{"duration":0.056496,"end_time":"2023-01-08T19:44:38.719863","exception":false,"start_time":"2023-01-08T19:44:38.663367","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:23:08.970734Z","iopub.execute_input":"2023-01-08T20:23:08.971085Z","iopub.status.idle":"2023-01-08T20:23:08.997595Z","shell.execute_reply.started":"2023-01-08T20:23:08.971055Z","shell.execute_reply":"2023-01-08T20:23:08.996333Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**La commande suivante permet de visualiser les valeurs non manquantes (False pour les valeurs manquantes)**","metadata":{"papermill":{"duration":0.02698,"end_time":"2023-01-08T19:44:38.773718","exception":false,"start_time":"2023-01-08T19:44:38.746738","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.isna()\n","metadata":{"papermill":{"duration":0.163675,"end_time":"2023-01-08T19:44:38.964429","exception":false,"start_time":"2023-01-08T19:44:38.800754","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:23:08.998982Z","iopub.execute_input":"2023-01-08T20:23:08.999420Z","iopub.status.idle":"2023-01-08T20:23:09.133713Z","shell.execute_reply.started":"2023-01-08T20:23:08.999386Z","shell.execute_reply":"2023-01-08T20:23:09.132596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**A partir de la, on remarque qu'il n' y a pas de valeurs inconnus vu que les valeurs sont de l'ordre de 0. Dans le cas échéant, il faut calculer la médiane de chaque valeur**","metadata":{"papermill":{"duration":0.027369,"end_time":"2023-01-08T19:44:39.019354","exception":false,"start_time":"2023-01-08T19:44:38.991985","status":"completed"},"tags":[]}},{"cell_type":"code","source":"df.isna().sum(axis=0).sort_values(ascending= False)\n","metadata":{"papermill":{"duration":0.12764,"end_time":"2023-01-08T19:44:39.174318","exception":false,"start_time":"2023-01-08T19:44:39.046678","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2023-01-08T20:29:07.543413Z","iopub.execute_input":"2023-01-08T20:29:07.544732Z","iopub.status.idle":"2023-01-08T20:29:07.639092Z","shell.execute_reply.started":"2023-01-08T20:29:07.544678Z","shell.execute_reply":"2023-01-08T20:29:07.637778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{"papermill":{"duration":0.031213,"end_time":"2023-01-08T19:45:15.676182","exception":false,"start_time":"2023-01-08T19:45:15.644969","status":"completed"},"tags":[]}},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/sample_submission.csv\")\nsubmission.shape\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:31:00.058065Z","iopub.execute_input":"2023-01-08T20:31:00.058511Z","iopub.status.idle":"2023-01-08T20:31:00.167975Z","shell.execute_reply.started":"2023-01-08T20:31:00.058471Z","shell.execute_reply":"2023-01-08T20:31:00.166749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:31:38.258401Z","iopub.execute_input":"2023-01-08T20:31:38.259542Z","iopub.status.idle":"2023-01-08T20:31:39.215879Z","shell.execute_reply.started":"2023-01-08T20:31:38.259498Z","shell.execute_reply":"2023-01-08T20:31:39.214598Z"},"trusted":true},"execution_count":null,"outputs":[]}]}