{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-01T17:47:06.681239Z","iopub.execute_input":"2023-01-01T17:47:06.681781Z","iopub.status.idle":"2023-01-01T17:47:06.717369Z","shell.execute_reply.started":"2023-01-01T17:47:06.681682Z","shell.execute_reply":"2023-01-01T17:47:06.716428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:49:18.920202Z","iopub.execute_input":"2023-01-01T17:49:18.920681Z","iopub.status.idle":"2023-01-01T17:49:37.454956Z","shell.execute_reply.started":"2023-01-01T17:49:18.920646Z","shell.execute_reply":"2023-01-01T17:49:37.453773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"tab for train : \",train.shape , \", tab for test :\" ,test.shape  )","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:51:37.562092Z","iopub.execute_input":"2023-01-01T17:51:37.562579Z","iopub.status.idle":"2023-01-01T17:51:37.569079Z","shell.execute_reply.started":"2023-01-01T17:51:37.562543Z","shell.execute_reply":"2023-01-01T17:51:37.567881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we will move forward to visualization for the dataset","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:54:40.320126Z","iopub.execute_input":"2023-01-01T17:54:40.320967Z","iopub.status.idle":"2023-01-01T17:54:41.612507Z","shell.execute_reply.started":"2023-01-01T17:54:40.320900Z","shell.execute_reply":"2023-01-01T17:54:41.611228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:55:18.524562Z","iopub.execute_input":"2023-01-01T17:55:18.525003Z","iopub.status.idle":"2023-01-01T17:55:19.458454Z","shell.execute_reply.started":"2023-01-01T17:55:18.524971Z","shell.execute_reply":"2023-01-01T17:55:19.457224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we gonna see the type of each column","metadata":{}},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:57:04.061477Z","iopub.execute_input":"2023-01-01T17:57:04.061899Z","iopub.status.idle":"2023-01-01T17:57:04.172654Z","shell.execute_reply.started":"2023-01-01T17:57:04.061867Z","shell.execute_reply":"2023-01-01T17:57:04.171423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:57:12.652205Z","iopub.execute_input":"2023-01-01T17:57:12.652705Z","iopub.status.idle":"2023-01-01T17:57:12.745073Z","shell.execute_reply.started":"2023-01-01T17:57:12.652666Z","shell.execute_reply":"2023-01-01T17:57:12.743818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T17:59:49.392955Z","iopub.execute_input":"2023-01-01T17:59:49.393364Z","iopub.status.idle":"2023-01-01T17:59:49.423403Z","shell.execute_reply.started":"2023-01-01T17:59:49.393317Z","shell.execute_reply":"2023-01-01T17:59:49.422114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum().any()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:00:15.708712Z","iopub.execute_input":"2023-01-01T18:00:15.709158Z","iopub.status.idle":"2023-01-01T18:00:15.804477Z","shell.execute_reply.started":"2023-01-01T18:00:15.709126Z","shell.execute_reply":"2023-01-01T18:00:15.803135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:00:54.753115Z","iopub.execute_input":"2023-01-01T18:00:54.753449Z","iopub.status.idle":"2023-01-01T18:00:54.782618Z","shell.execute_reply.started":"2023-01-01T18:00:54.753419Z","shell.execute_reply":"2023-01-01T18:00:54.781307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isna().sum().any()","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:01:10.946489Z","iopub.execute_input":"2023-01-01T18:01:10.946874Z","iopub.status.idle":"2023-01-01T18:01:11.021566Z","shell.execute_reply.started":"2023-01-01T18:01:10.946845Z","shell.execute_reply":"2023-01-01T18:01:11.020386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ML Model to test and train our Dataset","metadata":{}},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:07:47.591176Z","iopub.execute_input":"2023-01-01T18:07:47.591674Z","iopub.status.idle":"2023-01-01T18:07:48.069718Z","shell.execute_reply.started":"2023-01-01T18:07:47.591640Z","shell.execute_reply":"2023-01-01T18:07:48.068457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train[\"target\"]\nX = train.drop(columns=[\"id\",\"f_27\", \"target\"])\n\ntest = test.drop(columns=[\"id\",\"f_27\"])\n\nX.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:13:26.787682Z","iopub.execute_input":"2023-01-01T18:13:26.788129Z","iopub.status.idle":"2023-01-01T18:13:26.913001Z","shell.execute_reply.started":"2023-01-01T18:13:26.788093Z","shell.execute_reply":"2023-01-01T18:13:26.911485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=0.33, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:18:57.371551Z","iopub.execute_input":"2023-01-01T18:18:57.372006Z","iopub.status.idle":"2023-01-01T18:18:57.813493Z","shell.execute_reply.started":"2023-01-01T18:18:57.371971Z","shell.execute_reply":"2023-01-01T18:18:57.812582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\n\nimport time\n\nmodel = LGBMClassifier()\nmodel.fit(X, y)\n\nprint(\"Training score :\", model.score(X, y))\n\npred_y_test = model.predict(X_test)\nprint(\"Roc auc score  :\", roc_auc_score(y_test, pred_y_test))","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:22:26.740935Z","iopub.execute_input":"2023-01-01T18:22:26.741374Z","iopub.status.idle":"2023-01-01T18:22:48.971562Z","shell.execute_reply.started":"2023-01-01T18:22:26.741318Z","shell.execute_reply":"2023-01-01T18:22:48.970073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\n\nwarnings.simplefilter(\"ignore\", category=FutureWarning)\n\n# Create a data frame with the feature importances and the corresponding column names\nfeature_imp = pd.DataFrame(sorted(zip(model.feature_importances_, X.columns)), columns=[\"Value\", \"Feature\"])\n\n# Set the figure size\nplt.figure(figsize=(20, 10))\n\n# Plot the feature importances using a barplot\nsns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False))\n\n# Add a title to the plot\nplt.title(\"LightGBM Features (avg over folds)\")\n\n# Tighten the layout\nplt.tight_layout()\n\n# Show the plot\nplt.show()\n\n# Save the plot to a file\nplt.savefig(\"lgbm_importances-01.png\")\n","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:27:39.667808Z","iopub.execute_input":"2023-01-01T18:27:39.668993Z","iopub.status.idle":"2023-01-01T18:27:40.455644Z","shell.execute_reply.started":"2023-01-01T18:27:39.668952Z","shell.execute_reply":"2023-01-01T18:27:40.454522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_imp = feature_imp.sort_values(by=\"Value\", ascending=False)\nprint(feature_imp)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:28:53.087422Z","iopub.execute_input":"2023-01-01T18:28:53.087852Z","iopub.status.idle":"2023-01-01T18:28:53.097799Z","shell.execute_reply.started":"2023-01-01T18:28:53.087821Z","shell.execute_reply":"2023-01-01T18:28:53.096273Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the top 6 features\nselected_features = feature_imp.iloc[:6, 1]\n\n# Convert the series to a list\nselected_features = selected_features.tolist()\n\nprint(selected_features)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:30:51.389885Z","iopub.execute_input":"2023-01-01T18:30:51.390637Z","iopub.status.idle":"2023-01-01T18:30:51.397563Z","shell.execute_reply.started":"2023-01-01T18:30:51.390601Z","shell.execute_reply":"2023-01-01T18:30:51.396409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:32:07.198543Z","iopub.execute_input":"2023-01-01T18:32:07.200012Z","iopub.status.idle":"2023-01-01T18:32:07.208252Z","shell.execute_reply.started":"2023-01-01T18:32:07.199965Z","shell.execute_reply":"2023-01-01T18:32:07.206385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"z = StratifiedKFold(n_splits=14, shuffle=True, random_state=0)\ny_predict = []\n\n# Iterate over the folds\nfor i, (train_index, val_index) in enumerate(z.split(X, y)):\n   \n    # Split the data into training and validation sets\n\n    X_train, X_val = X.iloc[train_index], X.iloc[val_index]\n    y_train, y_val = y.iloc[train_index], y.iloc[val_index]\n\n    # Create the model\n    model = LGBMClassifier()\n\n    # Fit the model\n    model.fit(X_train, y_train)\n\n    # Make predictions on the validation set\n    y_pred = model.predict_proba(X_val)[:, 1]\n\n    # Make predictions on the test set\n    y_predict.append(model.predict_proba(test)[:, 1])\n\n    # Evaluate the model\n    print(\"Training score:\", model.score(X_train, y_train))\n    print(\"ROC AUC score:\", roc_auc_score(y_val, y_pred), \"\\n\")","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:40:07.538303Z","iopub.execute_input":"2023-01-01T18:40:07.538754Z","iopub.status.idle":"2023-01-01T18:44:23.584466Z","shell.execute_reply.started":"2023-01-01T18:40:07.538721Z","shell.execute_reply":"2023-01-01T18:44:23.582799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We gonna apply this model for the submission.csv","metadata":{}},{"cell_type":"code","source":"# Create the model\nmodel = LGBMClassifier()\n\n# Fit the model to the training data\nmodel.fit(X, y)\n\n# Make predictions on the test set\ny_predict.append(model.predict_proba(test)[:, 1])\n","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:46:57.679850Z","iopub.execute_input":"2023-01-01T18:46:57.680421Z","iopub.status.idle":"2023-01-01T18:47:13.596170Z","shell.execute_reply.started":"2023-01-01T18:46:57.680382Z","shell.execute_reply":"2023-01-01T18:47:13.595234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Take the mean of the predictions along the rows (axis=0)\nmean_predictions = np.mean(y_predict, axis=0)\n\nprint(mean_predictions)","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:48:31.025555Z","iopub.execute_input":"2023-01-01T18:48:31.025955Z","iopub.status.idle":"2023-01-01T18:48:31.064889Z","shell.execute_reply.started":"2023-01-01T18:48:31.025925Z","shell.execute_reply":"2023-01-01T18:48:31.063443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will read the submission Dataset and apply the model trained before and get a prediction ","metadata":{}},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/sample_submission.csv\")\nsubmission.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:50:20.712222Z","iopub.execute_input":"2023-01-01T18:50:20.713279Z","iopub.status.idle":"2023-01-01T18:50:20.984624Z","shell.execute_reply.started":"2023-01-01T18:50:20.713234Z","shell.execute_reply":"2023-01-01T18:50:20.983376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['target'] = np.array(y_predict).mean(axis=0)\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-01T18:52:02.119221Z","iopub.execute_input":"2023-01-01T18:52:02.119789Z","iopub.status.idle":"2023-01-01T18:52:03.774137Z","shell.execute_reply.started":"2023-01-01T18:52:02.119741Z","shell.execute_reply":"2023-01-01T18:52:03.773088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}