{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-02-20T15:13:05.039743Z","iopub.execute_input":"2023-02-20T15:13:05.040845Z","iopub.status.idle":"2023-02-20T15:13:05.077728Z","shell.execute_reply.started":"2023-02-20T15:13:05.040716Z","shell.execute_reply":"2023-02-20T15:13:05.076719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfrom datetime import datetime\nfrom IPython.display import Audio, display\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom category_encoders.leave_one_out import LeaveOneOutEncoder\nimport sklearn\nfrom sklearn.manifold import TSNE\nfrom sklearn.utils import shuffle\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.preprocessing import OrdinalEncoder, OneHotEncoder, StandardScaler, PolynomialFeatures\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import train_test_split, cross_val_predict, KFold, StratifiedKFold, RepeatedStratifiedKFold, RandomizedSearchCV\nfrom sklearn.linear_model import LogisticRegression, Lasso, LogisticRegressionCV, LassoCV, ElasticNetCV, RidgeCV, LarsCV, Ridge\nfrom sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, RandomForestRegressor, ExtraTreesRegressor\nimport matplotlib.pyplot as plt\nimport matplotlib as mpl\nfrom xgboost import XGBClassifier, XGBRegressor\nfrom catboost import CatBoostClassifier, CatBoostRegressor\nfrom lightgbm.sklearn import LGBMClassifier, LGBMRegressor\nimport tensorflow as tf\nimport tensorflow.keras.layers as layers\nfrom scipy import special\n\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:13:05.079727Z","iopub.execute_input":"2023-02-20T15:13:05.080376Z","iopub.status.idle":"2023-02-20T15:13:14.201076Z","shell.execute_reply.started":"2023-02-20T15:13:05.080341Z","shell.execute_reply":"2023-02-20T15:13:14.200097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv(\"/kaggle/input/deepshare-playground-series2/train_data.csv\")\ntest=pd.read_csv(\"/kaggle/input/deepshare-playground-series2/test_data.csv\")\nsub=pd.read_csv(\"/kaggle/input/deepshare-playground-series2/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:13:14.202844Z","iopub.execute_input":"2023-02-20T15:13:14.203663Z","iopub.status.idle":"2023-02-20T15:13:54.941749Z","shell.execute_reply.started":"2023-02-20T15:13:14.203624Z","shell.execute_reply":"2023-02-20T15:13:54.940270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.concat([train,test],axis=0).reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:13:54.944765Z","iopub.execute_input":"2023-02-20T15:13:54.945521Z","iopub.status.idle":"2023-02-20T15:13:59.351782Z","shell.execute_reply.started":"2023-02-20T15:13:54.945454Z","shell.execute_reply":"2023-02-20T15:13:59.350782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\ndf[\"user_tag\"]=df[\"user_tag\"].apply(lambda x:x[2:-2].split(\"', '\")[0:-1])\ndf[\"user_tag\"]=df[\"user_tag\"].apply(lambda x:[int(i) for i in x])\ndf[\"user_tag\"]=df[\"user_tag\"].apply(lambda x:len(x))","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:13:59.353145Z","iopub.execute_input":"2023-02-20T15:13:59.354146Z","iopub.status.idle":"2023-02-20T15:14:55.880556Z","shell.execute_reply.started":"2023-02-20T15:13:59.354107Z","shell.execute_reply":"2023-02-20T15:14:55.879574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols  =  [f'user_fea{i}' for i in range(6,9)] \ncate_cols = ['user_id','item_id']+ [f'user_fea{i}' for i in range(1,6)] + ['item_fea1']+['user_fea9']","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:14:55.881988Z","iopub.execute_input":"2023-02-20T15:14:55.882370Z","iopub.status.idle":"2023-02-20T15:14:55.889485Z","shell.execute_reply.started":"2023-02-20T15:14:55.882328Z","shell.execute_reply":"2023-02-20T15:14:55.887408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in tqdm(num_cols):\n    df[col] = df[col].fillna(0)\n    df[col] = (df[col] - df[col].mean()) / df[col].std()","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:14:55.891229Z","iopub.execute_input":"2023-02-20T15:14:55.892012Z","iopub.status.idle":"2023-02-20T15:14:56.463698Z","shell.execute_reply.started":"2023-02-20T15:14:55.891975Z","shell.execute_reply":"2023-02-20T15:14:56.462638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in tqdm(cate_cols):\n    df[col] = df[col].fillna('-1')\n    map_dict = dict(zip(df[col].unique(), range(df[col].nunique())))\n    \n    df[col] = df[col].map(map_dict)\n    df[f'{col}_count'] = df[col].map(df[col].value_counts())","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:14:56.465327Z","iopub.execute_input":"2023-02-20T15:14:56.465727Z","iopub.status.idle":"2023-02-20T15:15:14.150543Z","shell.execute_reply.started":"2023-02-20T15:14:56.465687Z","shell.execute_reply":"2023-02-20T15:15:14.149479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = df[df['label'].notna()].reset_index(drop=True)\ntest = df[df['label'].isna()].reset_index(drop=True)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:15:14.152116Z","iopub.execute_input":"2023-02-20T15:15:14.152503Z","iopub.status.idle":"2023-02-20T15:15:16.324990Z","shell.execute_reply.started":"2023-02-20T15:15:14.152468Z","shell.execute_reply":"2023-02-20T15:15:16.324006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape,test.shape","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:15:16.328833Z","iopub.execute_input":"2023-02-20T15:15:16.329139Z","iopub.status.idle":"2023-02-20T15:15:16.335410Z","shell.execute_reply.started":"2023-02-20T15:15:16.329113Z","shell.execute_reply":"2023-02-20T15:15:16.334345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain=train[\"label\"]","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:15:16.337029Z","iopub.execute_input":"2023-02-20T15:15:16.337405Z","iopub.status.idle":"2023-02-20T15:15:16.347762Z","shell.execute_reply.started":"2023-02-20T15:15:16.337364Z","shell.execute_reply":"2023-02-20T15:15:16.346692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ytrain=ytrain.values","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:15:16.349257Z","iopub.execute_input":"2023-02-20T15:15:16.349721Z","iopub.status.idle":"2023-02-20T15:15:16.359173Z","shell.execute_reply.started":"2023-02-20T15:15:16.349678Z","shell.execute_reply":"2023-02-20T15:15:16.358231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.drop(\"label\",axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:15:16.362349Z","iopub.execute_input":"2023-02-20T15:15:16.362887Z","iopub.status.idle":"2023-02-20T15:15:16.588625Z","shell.execute_reply.started":"2023-02-20T15:15:16.362853Z","shell.execute_reply":"2023-02-20T15:15:16.587651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nfrom sklearn.utils import class_weight\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import roc_auc_score \nfrom bayes_opt import BayesianOptimization\nfrom sklearn.model_selection import KFold\ndef xgb_evaluate(n_estimators,max_depth,subsample\n                ,colsample_bytree,learning_rate,scale_pos_weight,gamma):\n    model=XGBClassifier(n_estimators=int(n_estimators),\n                          max_depth=int(max_depth),\n                          subsample=subsample,\n                          colsample_bytree=colsample_bytree,\n                          learning_rate=learning_rate,n_jobs=-1,\n                          tree_method='gpu_hist',\n                          scale_pos_weight=int(scale_pos_weight),\n                          gamma=gamma,\n                          predictor='gpu_predictor')\n    rmse_scores=[]\n    kf = KFold(n_splits = 3)\n    for train_ix, test_ix in tqdm(kf.split(train.values)):\n        X_train, X_test = train.values[train_ix], train.values[test_ix]\n        Y_train, Y_test = ytrain[train_ix], ytrain[test_ix]\n        model.fit(X_train,Y_train)\n        pred = model.predict_proba(X_test)[:,1]\n        rmse_scores.append(roc_auc_score(Y_test,pred))\n    val=np.mean(rmse_scores)\n    return val\npbounds = {'max_depth': (5, 30),\n           'n_estimators': (100, 800),\n           \"subsample\":(0.001, 1),\n           \"colsample_bytree\":(0.001,1),\n           \"learning_rate\":(0.05,0.35),\n           \"scale_pos_weight\":(1,10),\n           \"gamma\":(0.1,5)}\nxgb_bo = BayesianOptimization(\n        f=xgb_evaluate,   # Objective function\n        pbounds=pbounds,  # Value space\n        verbose=2,  # \n        random_state=1,\n)\nxgb_bo.maximize(init_points=2,   # Steps of random search\n                   n_iter=18,    # Number of iterations to perform Bayesian optimization\n                   acq='ei')\nprint(xgb_bo.max)\nres_xgb = xgb_bo.max\nparams_xgb1 = res_xgb['params']\nfor key in params_xgb1:\n    if key in [\"max_depth\",\"scale_pos_weight\",\"n_estimators\"]:\n        params_xgb1[key]=int(params_xgb1[key])","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:15:16.592559Z","iopub.execute_input":"2023-02-20T15:15:16.593164Z","iopub.status.idle":"2023-02-20T15:15:16.600296Z","shell.execute_reply.started":"2023-02-20T15:15:16.593136Z","shell.execute_reply":"2023-02-20T15:15:16.599446Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns,test.columns","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:18:03.241078Z","iopub.execute_input":"2023-02-20T15:18:03.241633Z","iopub.status.idle":"2023-02-20T15:18:03.255507Z","shell.execute_reply.started":"2023-02-20T15:18:03.241586Z","shell.execute_reply":"2023-02-20T15:18:03.254468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nxgb_1=XGBClassifier(**params_xgb1,tree_method='gpu_hist',\n                    predictor='gpu_predictor',n_jobs=-1)\nxgb_1.fit(train.values,ytrain)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:15:30.506196Z","iopub.execute_input":"2023-02-20T15:15:30.506598Z","iopub.status.idle":"2023-02-20T15:17:10.619514Z","shell.execute_reply.started":"2023-02-20T15:15:30.506566Z","shell.execute_reply":"2023-02-20T15:17:10.618541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test=test.drop(\"label\",axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:18:27.726957Z","iopub.execute_input":"2023-02-20T15:18:27.727324Z","iopub.status.idle":"2023-02-20T15:18:27.875249Z","shell.execute_reply.started":"2023-02-20T15:18:27.727293Z","shell.execute_reply":"2023-02-20T15:18:27.874209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y=xgb_1.predict_proba(test.values)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:18:30.100139Z","iopub.execute_input":"2023-02-20T15:18:30.100535Z","iopub.status.idle":"2023-02-20T15:18:33.329620Z","shell.execute_reply.started":"2023-02-20T15:18:30.100505Z","shell.execute_reply":"2023-02-20T15:18:33.328630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub['label']=y[:,1]\nsub.to_csv(\"sample_submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:18:33.331513Z","iopub.execute_input":"2023-02-20T15:18:33.332102Z","iopub.status.idle":"2023-02-20T15:18:37.141541Z","shell.execute_reply.started":"2023-02-20T15:18:33.332062Z","shell.execute_reply":"2023-02-20T15:18:37.140584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub","metadata":{"execution":{"iopub.status.busy":"2023-02-20T15:18:37.143528Z","iopub.execute_input":"2023-02-20T15:18:37.143906Z","iopub.status.idle":"2023-02-20T15:18:37.159985Z","shell.execute_reply.started":"2023-02-20T15:18:37.143871Z","shell.execute_reply":"2023-02-20T15:18:37.158465Z"},"trusted":true},"execution_count":null,"outputs":[]}]}