{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":71710,"databundleVersionId":7843531,"sourceType":"competition"}],"dockerImageVersionId":30732,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-10T06:52:02.031275Z","iopub.execute_input":"2024-06-10T06:52:02.031685Z","iopub.status.idle":"2024-06-10T06:52:02.040461Z","shell.execute_reply.started":"2024-06-10T06:52:02.031655Z","shell.execute_reply":"2024-06-10T06:52:02.039344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import StratifiedKFold","metadata":{"execution":{"iopub.status.busy":"2024-06-15T07:37:26.071813Z","iopub.execute_input":"2024-06-15T07:37:26.072889Z","iopub.status.idle":"2024-06-15T07:37:26.083633Z","shell.execute_reply.started":"2024-06-15T07:37:26.072842Z","shell.execute_reply":"2024-06-15T07:37:26.082431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dt = pd.read_csv(r'/kaggle/input/deepshare-basic-competition/train_data.csv')#训练集\ntest_dt = pd.read_csv(r'/kaggle/input/deepshare-basic-competition/test_data.csv')#测试集\ntrain_dt['label'] = train_dt['label'].astype('int32')\nmerged_dt = pd.concat([train_dt,test_dt],axis = 0).reset_index(drop = True)\nmerged_dt.head()","metadata":{"execution":{"iopub.status.busy":"2024-06-14T05:06:50.660654Z","iopub.execute_input":"2024-06-14T05:06:50.661249Z","iopub.status.idle":"2024-06-14T05:07:37.810774Z","shell.execute_reply.started":"2024-06-14T05:06:50.661213Z","shell.execute_reply":"2024-06-14T05:07:37.809607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_memory_use(df,verbose = True):\n    dtypes = ['int8','int16','int32','int64','float16','float32','float64']\n    s_mem_use = df.memory_usage().sum() / 1024 ** 2 #MB\n    for col in df.columns:\n        col_type = df[col].dtypes\n        if str(col_type) in dtypes:\n            c_max = df[col].max()\n            c_min = df[col].min()\n            if str(col_type)[:3] == 'int':\n                if c_max < np.iinfo(np.int8).max and c_min > np.iinfo(np.int8).min:\n                    df[col] = df[col].astype('int8')\n                elif c_max < np.iinfo(np.int16).max and c_min > np.iinfo(np.int16).min:\n                    df[col] = df[col].astype('int16')\n                elif c_max < np.iinfo(np.int32).max and c_min > np.iinfo(np.int32).min:\n                    df[col] = df[col].astype('int32')\n                elif c_max < np.iinfo(np.int64).max and c_min > np.iinfo(np.int64).min:\n                    df[col] = df[col].astype('int64')\n            else:\n                c_prec = df[col].apply(lambda x : np.finfo(x).precision).max()\n                if c_max < np.finfo(np.float16).max and c_min > np.finfo(np.float16).min and c_prec == np.finfo(np.float16).precision:\n                    df[col] = df[col].astype('float16')\n                elif c_max < np.finfo(np.float32).max and c_min > np.finfo(np.float32).min and c_prec == np.finfo(np.float32).precision:\n                    df[col] = df[col].astype('float32')\n                elif c_max < np.finfo(np.float64).max and c_min > np.finfo(np.float64).min and c_prec == np.finfo(np.float64).precision:\n                    df[col] = df[col].astype('float64')\n    e_mem_use = df.memory_usage().sum() / 1024 ** 2 #MB\n    if verbose == True:\n        print('内存使用降低至{:5.2f}MB,降低了{:.2f}%'.format(e_mem_use,100 * (s_mem_use - e_mem_use)/s_mem_use))\n    return df\n    \n                ","metadata":{"execution":{"iopub.status.busy":"2024-06-14T05:07:52.810738Z","iopub.execute_input":"2024-06-14T05:07:52.811182Z","iopub.status.idle":"2024-06-14T05:07:52.830299Z","shell.execute_reply.started":"2024-06-14T05:07:52.811146Z","shell.execute_reply":"2024-06-14T05:07:52.828812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_dt.info(verbose = True)\nmerged_dt = reduce_memory_use(merged_dt)","metadata":{"execution":{"iopub.status.busy":"2024-06-13T07:47:50.116390Z","iopub.execute_input":"2024-06-13T07:47:50.116810Z","iopub.status.idle":"2024-06-13T07:51:25.202727Z","shell.execute_reply.started":"2024-06-13T07:47:50.116777Z","shell.execute_reply":"2024-06-13T07:51:25.201350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = [f'user_fea{i}'for i in range(6,9)]\ncat_cols = ['user_id','item_id','item_fea1']+[f'user_fea{i}'for i in range(1,6)]+['user_fea9']\nprint(num_cols)\nprint()\nprint(cat_cols)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T05:08:01.624185Z","iopub.execute_input":"2024-06-14T05:08:01.624630Z","iopub.status.idle":"2024-06-14T05:08:01.632339Z","shell.execute_reply.started":"2024-06-14T05:08:01.624596Z","shell.execute_reply":"2024-06-14T05:08:01.631049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#连续值特征编码\nfor col in num_cols: #Xscaled =X - Xmin/Xmax − Xmin\n    merged_dt[col].fillna(-1)\n    x_min = np.min(merged_dt[col])\n    x_max = np.max(merged_dt[col])\n    merged_dt[col] = merged_dt[col] - x_min/x_max - x_min","metadata":{"execution":{"iopub.status.busy":"2024-06-14T05:08:25.383561Z","iopub.execute_input":"2024-06-14T05:08:25.384031Z","iopub.status.idle":"2024-06-14T05:08:25.821720Z","shell.execute_reply.started":"2024-06-14T05:08:25.383998Z","shell.execute_reply":"2024-06-14T05:08:25.820460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#离散值特征编码\nfor col in cat_cols:\n    merged_dt[col].fillna('-1')\n    mapping = dict(zip(merged_dt[col].unique(),range(merged_dt[col].nunique())))\n    merged_dt[col] = merged_dt[col].map(mapping)\n    merged_dt[f'{col}_count'] = merged_dt[col].map(merged_dt[col].value_counts())\nmerged_dt = reduce_memory_use(merged_dt)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T05:08:30.054957Z","iopub.execute_input":"2024-06-14T05:08:30.055364Z","iopub.status.idle":"2024-06-14T05:13:02.304765Z","shell.execute_reply.started":"2024-06-14T05:08:30.055331Z","shell.execute_reply":"2024-06-14T05:13:02.303573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#666这个入开桂，穿越特征\nmerged_dt = merged_dt.sort_values(by = ['user_id','ts']).reset_index(drop = True)\nfor f in cat_cols:\n    tmp = merged_dt[[f,'ts']].groupby(f)\n    for gap in [1,2,3,5,9]:\n        merged_dt[f'{f}_pre_{gap}_click'] = tmp['ts'].shift(0) - tmp['ts'].shift(gap)\n        merged_dt[f'{f}_aft_{gap}_click'] = tmp['ts'].shift(-gap) - tmp['ts'].shift(0)\nmerged_dt = reduce_memory_use(merged_dt)\n    ","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#交叉特征（离散）\nitem_cate = ['item_fea1']\nuser_cate = [f'user_fea{i}'for i in range(1,6)]+['user_fea9']\ncross_feature = [['user_id','item_id']]\ncross_feature += [['user_id',item_fea] for item_fea in item_cate]\ncross_feature += [['item_id',user_fea] for user_fea in user_cate]\nfor k,v in cross_feature:\n    merged_dt[f'{k}_cross_{v}'] = merged_dt[k].astype('str') + '_' + merged_dt[v].astype('str')\n    merged_dt[f'{k}_cross_{v}_counts'] = merged_dt[f'{k}_cross_{v}'].map(merged_dt[f'{k}_cross_{v}'].value_counts())\n    merged_dt[f'{k}_cross_{v}/{k}'] = merged_dt[f'{k}_cross_{v}_counts']/merged_dt[f'{k}_count']\n    merged_dt[f'{k}_cross_{v}/{v}'] = merged_dt[f'{k}_cross_{v}_counts']/merged_dt[f'{v}_count']\n    del merged_dt[f'{k}_cross_{v}']\nmerged_dt = reduce_memory_use(merged_dt)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T05:15:57.623071Z","iopub.execute_input":"2024-06-14T05:15:57.623463Z","iopub.status.idle":"2024-06-14T05:31:15.788471Z","shell.execute_reply.started":"2024-06-14T05:15:57.623434Z","shell.execute_reply":"2024-06-14T05:31:15.787134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"merged_dt.info(verbose = True)","metadata":{"execution":{"iopub.status.busy":"2024-06-14T05:34:54.162154Z","iopub.execute_input":"2024-06-14T05:34:54.164086Z","iopub.status.idle":"2024-06-14T05:34:54.186233Z","shell.execute_reply.started":"2024-06-14T05:34:54.164039Z","shell.execute_reply":"2024-06-14T05:34:54.185047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#groupby特征\n#离散特恒-离散特征\nfor k,v in cross_feature:\n    tmp = merged_dt.groupby(k)[v].nunique().reset_index()\n    tmp.columns = [k,v+'_nunique']\n    merged_dt = merged_dt.merge(tmp,on = k,how = 'left')\n    \n#离散特征-连续特征\ncat_num_features = [['item_id',num_fea] for num_fea in num_cols]\nfor k,v in cat_num_features:\n    tmp = merged_dt.groupby(k)[v].describe().reset_index()\n    tmp.columns = [k] + ['group' + x for x in['count','min','std','mean','25%','50%','75%','max']]\n    merged_dt = merged_dt.merge(tmp[[k] + ['group' + x for x in['std','mean','25%','50%','75%']]],\n                                on = k,how = 'left'\n                               )\nmerged_dt = reduce_memory_use(merged_dt)\n                                    \n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#一定要用gc回收变量，防止爆内存\nimport gc\ntrain_s_dt = merged_dt[merged_dt['label'].notna()].reset_index(drop = True)\ntest_s_dt = merged_dt[merged_dt['label'].isna()].reset_index(drop = True)\ndel merged_dt\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2024-06-10T06:52:50.288236Z","iopub.execute_input":"2024-06-10T06:52:50.288590Z","iopub.status.idle":"2024-06-10T06:52:53.185851Z","shell.execute_reply.started":"2024-06-10T06:52:50.288562Z","shell.execute_reply":"2024-06-10T06:52:53.184721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#目标编码\nn_splits = 5\nstf = StratifiedKFold(n_splits = n_splits ,shuffle = True , random_state = 42)\nfor f in cat_cols:\n    train_s_dt[f'{f}_target_enc'] = 0\n    test_s_dt[f'{f}_target_enc'] = 0\n    for _fold,(trn_idx,valid_idx) in enumerate(stf.split(train_s_dt,train_s_dt['label'])):\n        trn_dt = train_s_dt[[f,'label']].iloc[trn_idx].reset_index(drop = True)\n        valid_dt = train_s_dt[[f]].iloc[valid_idx].reset_index(drop = True)\n        #合并同类项\n        enc_dt = trn_dt.groupby(f,as_index = False)['label'].agg({f'{f}_target_enc':'mean'})\n        valid_dt = valid_dt.merge(enc_dt,on = f,how = 'left')\n        test_x = test_s_dt[[f]].merge(enc_dt,on = f,how = 'left')\n        #全局填充\n        valid_dt[f'{f}_target_enc'] = valid_dt[f'{f}_target_enc'].fillna(train_s_dt['label'].mean())\n        test_x[f'{f}_target_enc'] = test_x[f'{f}_target_enc'].fillna(train_s_dt['label'].mean())\n        train_s_dt.loc[valid_idx,[f'{f}_target_enc']] = valid_dt[f'{f}_target_enc'].values\n        test_s_dt[f'{f}_target_enc'] += test_x[f'{f}_target_enc'].values/n_splits\ntrain_s_dt = reduce_memory_use(train_s_dt)\ntest_s_dt = reduce_memory_use(test_s_dt)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_s_dt.info(verbose = True)","metadata":{"execution":{"iopub.status.busy":"2024-06-10T06:52:53.187332Z","iopub.execute_input":"2024-06-10T06:52:53.187825Z","iopub.status.idle":"2024-06-10T06:52:53.201259Z","shell.execute_reply.started":"2024-06-10T06:52:53.187771Z","shell.execute_reply":"2024-06-10T06:52:53.199671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_feature = ['label','ID','lng','lat','ts','user_tag']\nfeatures = train_s_dt.columns\nused_features = [f for f in features if f not in drop_feature]\nprint(used_features)","metadata":{"execution":{"iopub.status.busy":"2024-06-10T06:52:53.202790Z","iopub.execute_input":"2024-06-10T06:52:53.203197Z","iopub.status.idle":"2024-06-10T06:52:53.214049Z","shell.execute_reply.started":"2024-06-10T06:52:53.203164Z","shell.execute_reply":"2024-06-10T06:52:53.212442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\nfrom lightgbm import log_evaluation,early_stopping\nimport multiprocessing","metadata":{"execution":{"iopub.status.busy":"2024-06-10T06:52:53.215569Z","iopub.execute_input":"2024-06-10T06:52:53.215956Z","iopub.status.idle":"2024-06-10T06:52:53.227174Z","shell.execute_reply.started":"2024-06-10T06:52:53.215925Z","shell.execute_reply":"2024-06-10T06:52:53.225584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\ny_pre = np.zeros(len(test_s_dt))\noof_pre = np.zeros(len(train_s_dt))\nn_fold = 5\nkf = KFold(n_splits = n_fold)\ncallbacks = [log_evaluation(period = 50),early_stopping(stopping_rounds = 50)]\n#祖传参数\nparams = {'num_leaves': 63, #结果对最终效果影响较大，越大值越好，太大会出现过拟合\n          'objective': 'binary', #定义的目标函数\n          'max_depth': 8,\n          'learning_rate': 0.1,\n          \"min_sum_hessian_in_leaf\": 6,\n          \"boosting\": \"gbdt\",\n          \"feature_fraction\": 1,  #提取的特征比率\n          \"bagging_freq\": 1,\n          \"bagging_fraction\": 1,\n          \"bagging_seed\": 11,\n#           \"lambda_l1\": 0.1,             #l1正则\n          # 'lambda_l2': 0.001,     #l2正则\n          \"verbosity\": -1,\n          \"nthread\": multiprocessing.cpu_count(),                #线程数量，-1表示全部线程，线程越多，运行的速度越快\n          'metric': {'binary_logloss', 'auc'},  ##评价函数选择\n          \"random_state\": 2077, #随机数种子，可以防止每次运行的结果不一致\n#            'device': 'gpu' ##如果安装的事gpu版本的lightgbm,可以加快运算\n          }\nfor fold_,(trn_idx,valid_idx) in enumerate(kf.split(train_s_dt)):\n    trn_dt = lgb.Dataset(train_s_dt[used_features].iloc[trn_idx],label = train_s_dt['label'].iloc[trn_idx])\n    valid_dt = lgb.Dataset(train_s_dt[used_features].iloc[valid_idx],label = train_s_dt['label'].iloc[valid_idx])\n    clf = lgb.train(\n                params,\n                trn_dt,\n                2000,\n                valid_sets = [trn_dt,valid_dt],\n                callbacks = callbacks\n    )\n    oof_pre[valid_idx] = clf.predict(train_s_dt[used_features].iloc[valid_idx],num_iteration = clf.best_iteration)\n    y_pre += clf.predict(test_s_dt[used_features],num_iteration = clf.best_iteration) / n_fold\n    ","metadata":{"execution":{"iopub.status.busy":"2024-06-10T06:55:14.816884Z","iopub.execute_input":"2024-06-10T06:55:14.818252Z","iopub.status.idle":"2024-06-10T08:40:17.841940Z","shell.execute_reply.started":"2024-06-10T06:55:14.818211Z","shell.execute_reply":"2024-06-10T08:40:17.840537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\nresults = roc_auc_score(train_s_dt['label'],oof_pre)\nprint(f'auc:{results}')\nprint(len(oof_pre))","metadata":{"execution":{"iopub.status.busy":"2024-06-10T09:13:44.573657Z","iopub.execute_input":"2024-06-10T09:13:44.574991Z","iopub.status.idle":"2024-06-10T09:13:44.581333Z","shell.execute_reply.started":"2024-06-10T09:13:44.574950Z","shell.execute_reply":"2024-06-10T09:13:44.580128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(y_pre))\nprint(len(test_s_dt))\nresult_df = pd.DataFrame()\nresult_df['ID'] = test_s_dt['ID']\nresult_df['label'] = y_pre\nresult_df.to_csv(r'/kaggle/working/sample_submission.csv',index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}