{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport optuna\nimport gc\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-08T20:08:40.238248Z","iopub.execute_input":"2023-01-08T20:08:40.239208Z","iopub.status.idle":"2023-01-08T20:08:40.270600Z","shell.execute_reply.started":"2023-01-08T20:08:40.239158Z","shell.execute_reply":"2023-01-08T20:08:40.269341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:08:42.242365Z","iopub.execute_input":"2023-01-08T20:08:42.243198Z","iopub.status.idle":"2023-01-08T20:08:51.060197Z","shell.execute_reply.started":"2023-01-08T20:08:42.243124Z","shell.execute_reply":"2023-01-08T20:08:51.059118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:08:54.354107Z","iopub.execute_input":"2023-01-08T20:08:54.354989Z","iopub.status.idle":"2023-01-08T20:08:54.362557Z","shell.execute_reply.started":"2023-01-08T20:08:54.354944Z","shell.execute_reply":"2023-01-08T20:08:54.361239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:08:55.933687Z","iopub.execute_input":"2023-01-08T20:08:55.934122Z","iopub.status.idle":"2023-01-08T20:08:57.238605Z","shell.execute_reply.started":"2023-01-08T20:08:55.934088Z","shell.execute_reply":"2023-01-08T20:08:57.237237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:51:02.147986Z","iopub.execute_input":"2023-01-08T19:51:02.148454Z","iopub.status.idle":"2023-01-08T19:51:03.150263Z","shell.execute_reply.started":"2023-01-08T19:51:02.148414Z","shell.execute_reply":"2023-01-08T19:51:03.148935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:51:05.487709Z","iopub.execute_input":"2023-01-08T19:51:05.488128Z","iopub.status.idle":"2023-01-08T19:51:05.599763Z","shell.execute_reply.started":"2023-01-08T19:51:05.488091Z","shell.execute_reply":"2023-01-08T19:51:05.598366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:51:07.927465Z","iopub.execute_input":"2023-01-08T19:51:07.927872Z","iopub.status.idle":"2023-01-08T19:51:08.034751Z","shell.execute_reply.started":"2023-01-08T19:51:07.927837Z","shell.execute_reply":"2023-01-08T19:51:08.033804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"train.isna().sum().any()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:51:10.224943Z","iopub.execute_input":"2023-01-08T19:51:10.225942Z","iopub.status.idle":"2023-01-08T19:51:10.323618Z","shell.execute_reply.started":"2023-01-08T19:51:10.225888Z","shell.execute_reply":"2023-01-08T19:51:10.322470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.isna().sum().any()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:51:11.999976Z","iopub.execute_input":"2023-01-08T19:51:12.000856Z","iopub.status.idle":"2023-01-08T19:51:12.076982Z","shell.execute_reply.started":"2023-01-08T19:51:12.000811Z","shell.execute_reply":"2023-01-08T19:51:12.075859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for df in [train, test]:\n    for i in range(10):\n        df[f'ch{i}'] = df.f_27.str.get(i).apply(ord) - ord('A') #hna kanjibo aski ydal koula letter ou kan9so mnha laski dyal A\n    df[\"unique_characters\"] = df.f_27.apply(lambda s: len(set(s))) #elemina les doublans ou 7thom f unique chars\nfeatures = [f for f in test.columns if f != 'id' and f != 'f_27']\ntrain[features].head()\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:09:04.229388Z","iopub.execute_input":"2023-01-08T20:09:04.229830Z","iopub.status.idle":"2023-01-08T20:09:19.032113Z","shell.execute_reply.started":"2023-01-08T20:09:04.229793Z","shell.execute_reply":"2023-01-08T20:09:19.029168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(features)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:51:32.545006Z","iopub.execute_input":"2023-01-08T19:51:32.546311Z","iopub.status.idle":"2023-01-08T19:51:32.551071Z","shell.execute_reply.started":"2023-01-08T19:51:32.546259Z","shell.execute_reply":"2023-01-08T19:51:32.550226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, accuracy_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:10:17.983964Z","iopub.execute_input":"2023-01-08T20:10:17.984395Z","iopub.status.idle":"2023-01-08T20:10:17.990397Z","shell.execute_reply.started":"2023-01-08T20:10:17.984357Z","shell.execute_reply":"2023-01-08T20:10:17.989079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train[\"target\"]\nX = train.drop(columns=[\"id\",\"f_27\", \"target\"])\n\ntest = test.drop(columns=[\"id\",\"f_27\"])\n\nX.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:10:20.808113Z","iopub.execute_input":"2023-01-08T20:10:20.808952Z","iopub.status.idle":"2023-01-08T20:10:21.151002Z","shell.execute_reply.started":"2023-01-08T20:10:20.808908Z","shell.execute_reply":"2023-01-08T20:10:21.150034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train, X_test, y_train, y_test= train_test_split(X, y, test_size=.3, random_state=5)\n","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:10:23.111343Z","iopub.execute_input":"2023-01-08T20:10:23.111764Z","iopub.status.idle":"2023-01-08T20:10:23.668434Z","shell.execute_reply.started":"2023-01-08T20:10:23.111729Z","shell.execute_reply":"2023-01-08T20:10:23.667467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\n\nimport time","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:10:25.954328Z","iopub.execute_input":"2023-01-08T20:10:25.954748Z","iopub.status.idle":"2023-01-08T20:10:25.960259Z","shell.execute_reply.started":"2023-01-08T20:10:25.954711Z","shell.execute_reply":"2023-01-08T20:10:25.958925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"optuna.logging.set_verbosity(optuna.logging.WARNING)\n\ndef objectivesLGBM(trial):\n    model = LGBMClassifier()\n    model.fit(X,y)\n\n    return model.score(X,y)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:10:27.577660Z","iopub.execute_input":"2023-01-08T20:10:27.578438Z","iopub.status.idle":"2023-01-08T20:10:27.584469Z","shell.execute_reply.started":"2023-01-08T20:10:27.578394Z","shell.execute_reply":"2023-01-08T20:10:27.583269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LGBMClassifier()\nmodel.fit(X, y)\n\nprint(\"Training score :\", model.score(X, y))\n\npred_y_test = model.predict(X_test)\nprint(\"Roc auc score  :\", roc_auc_score(y_test, pred_y_test))","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:10:29.505785Z","iopub.execute_input":"2023-01-08T20:10:29.506564Z","iopub.status.idle":"2023-01-08T20:10:51.067109Z","shell.execute_reply.started":"2023-01-08T20:10:29.506518Z","shell.execute_reply":"2023-01-08T20:10:51.065866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\n\n# sorted(zip(clf.feature_importances_, X.columns), reverse=True)\nfeature_imp = pd.DataFrame(sorted(zip(model.feature_importances_,X.columns)), columns=['Value','Feature'])\n\nplt.figure(figsize=(20, 10))\nsns.barplot(x=\"Value\", y=\"Feature\", data=feature_imp.sort_values(by=\"Value\", ascending=False))\nplt.title('LightGBM Features (avg over folds)')\nplt.tight_layout()\nplt.show()\nplt.savefig('lgbm_importances-01.png')","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:11:03.027990Z","iopub.execute_input":"2023-01-08T20:11:03.028463Z","iopub.status.idle":"2023-01-08T20:11:03.699007Z","shell.execute_reply.started":"2023-01-08T20:11:03.028423Z","shell.execute_reply":"2023-01-08T20:11:03.697778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feature_imp = feature_imp.sort_values(by = \"Value\", ascending=False)\nfeature_imp","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:11:10.498467Z","iopub.execute_input":"2023-01-08T20:11:10.498883Z","iopub.status.idle":"2023-01-08T20:11:10.515327Z","shell.execute_reply.started":"2023-01-08T20:11:10.498848Z","shell.execute_reply":"2023-01-08T20:11:10.513773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_features = feature_imp[:6]['Feature'].tolist()\nselected_features","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:11:13.378562Z","iopub.execute_input":"2023-01-08T20:11:13.378973Z","iopub.status.idle":"2023-01-08T20:11:13.387005Z","shell.execute_reply.started":"2023-01-08T20:11:13.378939Z","shell.execute_reply":"2023-01-08T20:11:13.385930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:11:14.923867Z","iopub.execute_input":"2023-01-08T20:11:14.925082Z","iopub.status.idle":"2023-01-08T20:11:14.932345Z","shell.execute_reply.started":"2023-01-08T20:11:14.925040Z","shell.execute_reply":"2023-01-08T20:11:14.931132Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_basics_features(data, features):\n    \n    for feature in features:\n        \"\"\"\n        new_feature_name = str(feature) + '_min'\n        data[new_feature_name] = data[feature].min()\n        \n        new_feature_name = str(feature) + '_max'\n        data[new_feature_name] = data[feature].max()\n        \"\"\"\n        \n        new_feature_name = str(feature) + '_mean'\n        data[new_feature_name] = data[feature].mean()\n        \n        new_feature_name = str(feature) + '_std'\n        data[new_feature_name] = data[feature].std()\n        \n        new_feature_name = str(feature) + '_max_min'\n        data[new_feature_name] = data[feature].max() - data[feature].min()\n    \n    return data","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:11:17.052628Z","iopub.execute_input":"2023-01-08T20:11:17.053042Z","iopub.status.idle":"2023-01-08T20:11:17.061304Z","shell.execute_reply.started":"2023-01-08T20:11:17.053008Z","shell.execute_reply":"2023-01-08T20:11:17.059843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = add_basics_features(X, selected_features)\ntest = add_basics_features(test, selected_features)\n\nX.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:11:20.262176Z","iopub.execute_input":"2023-01-08T20:11:20.262573Z","iopub.status.idle":"2023-01-08T20:11:20.500498Z","shell.execute_reply.started":"2023-01-08T20:11:20.262539Z","shell.execute_reply":"2023-01-08T20:11:20.499113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nkf = StratifiedKFold(n_splits=10, shuffle=True, random_state = 0)\ny_predict = []\n\nfor fold, (train_index, val_index) in enumerate(kf.split(X, y)):\n        print('*'*50, f\" Fold {fold} \", '*'*50, '\\n')\n        \n        # Split Data\n        X_train = X.loc[train_index]\n        X_val = X.loc[val_index]\n\n        y_train = y.loc[train_index]\n        y_val = y.loc[val_index]\n        \n        # Create Model here\n        model = LGBMClassifier()\n        \n        # Fit Model\n        model.fit(X_train,y_train)\n\n        # Make X_val prediction\n        y_pred = model.predict_proba(X_val)[:,1]\n        \n        # Make Test prediction\n        y_predict.append(model.predict_proba(test)[:,1])\n\n        # Evaluate Model\n        print(\"Training score :\", model.score(X_train, y_train))\n        print(\"Roc auc score  :\", roc_auc_score(y_val, y_pred), '\\n')\n        \n        # Free the memory\n        del X_train, y_train, model, X_val, y_val, y_pred\n        gc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:11:22.066437Z","iopub.execute_input":"2023-01-08T20:11:22.066843Z","iopub.status.idle":"2023-01-08T20:15:18.531362Z","shell.execute_reply.started":"2023-01-08T20:11:22.066809Z","shell.execute_reply":"2023-01-08T20:15:18.530072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LGBMClassifier()\n\nmodel.fit(X, y)\n\ny_predict.append(model.predict_proba(test)[:,1])","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:13:57.380120Z","iopub.status.idle":"2023-01-08T19:13:57.380874Z","shell.execute_reply.started":"2023-01-08T19:13:57.380570Z","shell.execute_reply":"2023-01-08T19:13:57.380599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model, X, y, test, X_test, y_test\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:13:57.382625Z","iopub.status.idle":"2023-01-08T19:13:57.383734Z","shell.execute_reply.started":"2023-01-08T19:13:57.383508Z","shell.execute_reply":"2023-01-08T19:13:57.383531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.array(y_predict).mean(axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-01-08T19:13:57.385250Z","iopub.status.idle":"2023-01-08T19:13:57.386020Z","shell.execute_reply.started":"2023-01-08T19:13:57.385804Z","shell.execute_reply":"2023-01-08T19:13:57.385827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv(\"/kaggle/input/emsi-5iir-ds/sample_submission.csv\")\nsubmission.shape","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:15:35.644860Z","iopub.execute_input":"2023-01-08T20:15:35.645337Z","iopub.status.idle":"2023-01-08T20:15:35.764071Z","shell.execute_reply.started":"2023-01-08T20:15:35.645294Z","shell.execute_reply":"2023-01-08T20:15:35.762673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['target'] = np.array(y_predict).mean(axis=0)\nsubmission.to_csv('submission_v4.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2023-01-08T20:15:37.699441Z","iopub.execute_input":"2023-01-08T20:15:37.699845Z","iopub.status.idle":"2023-01-08T20:15:39.336866Z","shell.execute_reply.started":"2023-01-08T20:15:37.699812Z","shell.execute_reply":"2023-01-08T20:15:39.335713Z"},"trusted":true},"execution_count":null,"outputs":[]}]}