{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":30627,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# reshis Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-13T13:18:04.167594Z","iopub.execute_input":"2023-12-13T13:18:04.167885Z","iopub.status.idle":"2023-12-13T13:18:04.176140Z","shell.execute_reply.started":"2023-12-13T13:18:04.167863Z","shell.execute_reply":"2023-12-13T13:18:04.175137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Import requiered libraries","metadata":{}},{"cell_type":"code","source":"import random\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n# import category_encoders as ce\nimport time\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:18:04.177327Z","iopub.execute_input":"2023-12-13T13:18:04.177656Z","iopub.status.idle":"2023-12-13T13:18:04.184462Z","shell.execute_reply.started":"2023-12-13T13:18:04.177630Z","shell.execute_reply":"2023-12-13T13:18:04.183519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read the train set","metadata":{"execution":{"iopub.status.busy":"2023-12-11T03:02:52.059058Z","iopub.execute_input":"2023-12-11T03:02:52.059562Z","iopub.status.idle":"2023-12-11T03:02:52.065678Z","shell.execute_reply.started":"2023-12-11T03:02:52.059512Z","shell.execute_reply":"2023-12-11T03:02:52.064329Z"}}},{"cell_type":"code","source":"## 读入数据，因原始训练集有四千万条数据，数据量太多，于是进行抽样，从数据取出其中350万条数据\n## 预处理阶段可以只读一部分数据出来分析\n\nnum_records = 40428967\nsample_size = 3500000\nskip_values = sorted(random.sample(range(1,num_records), num_records - sample_size))\n\ndf_train = pd.read_csv(\"../input/avazu-ctr-prediction/train.gz\", compression='gzip',skiprows=skip_values,nrows=sample_size)\ndf_train['hour'] = pd.to_datetime(df_train['hour'], format='%y%m%d%H')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:18:04.185703Z","iopub.execute_input":"2023-12-13T13:18:04.186095Z","iopub.status.idle":"2023-12-13T13:20:42.939189Z","shell.execute_reply.started":"2023-12-13T13:18:04.186064Z","shell.execute_reply":"2023-12-13T13:20:42.938178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 无需重设索引了\ndf_train.index","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:42.940591Z","iopub.execute_input":"2023-12-13T13:20:42.940931Z","iopub.status.idle":"2023-12-13T13:20:42.948747Z","shell.execute_reply.started":"2023-12-13T13:20:42.940899Z","shell.execute_reply":"2023-12-13T13:20:42.947745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preprocessing","metadata":{"execution":{"iopub.status.busy":"2023-12-11T03:30:54.862080Z","iopub.execute_input":"2023-12-11T03:30:54.862469Z","iopub.status.idle":"2023-12-11T03:30:54.915470Z","shell.execute_reply.started":"2023-12-11T03:30:54.862439Z","shell.execute_reply":"2023-12-11T03:30:54.914248Z"}}},{"cell_type":"code","source":"# 本次预测目的是用前十天的点击情况去预测11天的情况，所以年月日无意义；星期数对应工作日和休息日，有意义；而小时对应每日的时段，根据个人生活习惯来说是有意义的。\ndf_train['dayofweek'] = df_train['hour'].dt.dayofweek\ndf_train['hour_time'] = df_train['hour'].dt.hour\n\n# 删除hour字段\ndf_train.drop('hour',axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:42.949964Z","iopub.execute_input":"2023-12-13T13:20:42.950260Z","iopub.status.idle":"2023-12-13T13:20:43.633850Z","shell.execute_reply.started":"2023-12-13T13:20:42.950234Z","shell.execute_reply":"2023-12-13T13:20:43.633020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:43.635081Z","iopub.execute_input":"2023-12-13T13:20:43.635406Z","iopub.status.idle":"2023-12-13T13:20:44.012605Z","shell.execute_reply.started":"2023-12-13T13:20:43.635379Z","shell.execute_reply":"2023-12-13T13:20:44.011397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exploratory data analysis","metadata":{}},{"cell_type":"code","source":"## 观察缺失值\ndf_train.isnull().sum()\n\n## 无缺失值","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:44.013879Z","iopub.execute_input":"2023-12-13T13:20:44.014188Z","iopub.status.idle":"2023-12-13T13:20:46.784116Z","shell.execute_reply.started":"2023-12-13T13:20:44.014161Z","shell.execute_reply":"2023-12-13T13:20:46.783190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 观察重复值\ndf_train[df_train.duplicated()]\n\n##无重复值","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:46.785289Z","iopub.execute_input":"2023-12-13T13:20:46.785574Z","iopub.status.idle":"2023-12-13T13:20:51.843779Z","shell.execute_reply.started":"2023-12-13T13:20:46.785551Z","shell.execute_reply":"2023-12-13T13:20:51.842745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 观察label的平衡性\nprint(df_train.click.value_counts(normalize = True))\nprint('\\n')\nplt.figure(figsize=(8,5))\nsns.countplot(x='click', data=df_train)\nplt.show()\n\n## 标签值极度不平衡","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:51.845059Z","iopub.execute_input":"2023-12-13T13:20:51.845399Z","iopub.status.idle":"2023-12-13T13:20:52.710422Z","shell.execute_reply.started":"2023-12-13T13:20:51.845371Z","shell.execute_reply":"2023-12-13T13:20:52.709512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ## 用箱图观察异常值\n# features = ['C14','C15','C16','C17','C18','C19','C20','C21'] \n\n# df_box = df_train[features]\n\n# # 创建一个 2x4 的子图布局\n# fig, ax = plt.subplots(2, 4, figsize=(12, 6))\n\n# # 将数据分成两行四列，分别画箱线图\n# for i in range(2):\n#     for j in range(4):\n#         index = i * 4 + j  # 计算每个子图的索引\n#         ax[i, j].boxplot(df_box.iloc[:, index])  # 画箱线图\n#         ax[i, j].set_title(f'{features[index]}')  # 设置子图标题\n\n# # 调整布局，避免重叠\n# plt.tight_layout()\n# plt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:52.711840Z","iopub.execute_input":"2023-12-13T13:20:52.712544Z","iopub.status.idle":"2023-12-13T13:20:52.716750Z","shell.execute_reply.started":"2023-12-13T13:20:52.712499Z","shell.execute_reply":"2023-12-13T13:20:52.715876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### C系列数据经过脱敏，很难从箱图判断是否有异常值","metadata":{}},{"cell_type":"code","source":"## 继续上一步\nlen_of_feature_count = []\nfor i in df_train.columns[2:].tolist():\n    print(i, ':', len(df_train[i].astype(str).value_counts()))\n    len_of_feature_count.append(len(df_train[i].astype(str).value_counts()))\n    \n### 从这些特征的value_count可看出，每个特征具有非常多重复值，分析得前面的箱图是无意义的，train set的数据类型应全部为object型","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:20:52.718142Z","iopub.execute_input":"2023-12-13T13:20:52.718399Z","iopub.status.idle":"2023-12-13T13:22:25.246849Z","shell.execute_reply.started":"2023-12-13T13:20:52.718376Z","shell.execute_reply":"2023-12-13T13:22:25.245730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 数值型特征相关性分析\ndf_numer = df_train.select_dtypes(include=np.number)\n\ncorr = df_numer.corr()\nf, ax = plt.subplots(figsize=(15,10))\nsns.heatmap(corr, ax=ax, annot=True,linewidths=3,cmap='YlGn')\nplt.title(\"Pearson correlation of Features\", y=1.05, size=15)\n\n\n# C14 和 C17 高度相关\n# device type 和 C1 高度相关\n# Removing C20 anomalised column, since it have got nearly 47% of values with -1 这个结论怎么来的？？我也不知道","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:22:25.248196Z","iopub.execute_input":"2023-12-13T13:22:25.248548Z","iopub.status.idle":"2023-12-13T13:22:29.762984Z","shell.execute_reply.started":"2023-12-13T13:22:25.248502Z","shell.execute_reply":"2023-12-13T13:22:29.761962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Feature enginerring\n\n* 1.Drop C17、C1、C20\n* 2.Balance data\n* 3.Encoder all object feature","metadata":{}},{"cell_type":"code","source":"## 删除C17、C1和C20\ndf_train.drop(['C17','C1','C20'],inplace=True,axis=1)\ndf_train.info()","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:22:29.764416Z","iopub.execute_input":"2023-12-13T13:22:29.765262Z","iopub.status.idle":"2023-12-13T13:22:30.365482Z","shell.execute_reply.started":"2023-12-13T13:22:29.765225Z","shell.execute_reply":"2023-12-13T13:22:30.364487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 分隔X和y\nX = df_train.iloc[:,2:]\ny = df_train.iloc[:,1]","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:22:30.366816Z","iopub.execute_input":"2023-12-13T13:22:30.367111Z","iopub.status.idle":"2023-12-13T13:22:30.924209Z","shell.execute_reply.started":"2023-12-13T13:22:30.367086Z","shell.execute_reply":"2023-12-13T13:22:30.923353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 平衡数据（下采样）\nfrom imblearn.under_sampling import RandomUnderSampler\n\nrandomsample = RandomUnderSampler()\nx_bal, y_bal = randomsample.fit_resample(X, y)\n\nfrom collections import Counter\nprint('Original dataset shape  {}'.format(Counter(y)))\nprint('Resampled dataset shape {}'.format(Counter(y_bal)))\nplt.figure()\nsns.countplot(data=pd.DataFrame({'click:y_bal'}),x=y_bal)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:22:30.926764Z","iopub.execute_input":"2023-12-13T13:22:30.927194Z","iopub.status.idle":"2023-12-13T13:22:33.765673Z","shell.execute_reply.started":"2023-12-13T13:22:30.927167Z","shell.execute_reply":"2023-12-13T13:22:33.764693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 编码之前需要先将测试集数据读出，进行处理，并与训练集合并，才能进行编码，否则会大大影响后续结果","metadata":{}},{"cell_type":"code","source":"## 记录x_bal的长度\nx_bal.reset_index()\nx_bal_len = len(x_bal)\nx_bal_len","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:22:33.767018Z","iopub.execute_input":"2023-12-13T13:22:33.767712Z","iopub.status.idle":"2023-12-13T13:22:34.526163Z","shell.execute_reply.started":"2023-12-13T13:22:33.767674Z","shell.execute_reply":"2023-12-13T13:22:34.525210Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 读入测试集\ndf_test = pd.read_csv(\"../input/avazu-ctr-prediction/test.gz\", compression='gzip',dtype={'id':'object'})\ndf_test['hour'] = pd.to_datetime(df_test['hour'], format='%y%m%d%H')\ndf_test['dayofweek'] = df_test['hour'].dt.dayofweek\ndf_test['hour_time'] = df_test['hour'].dt.hour\n\n# 删除hour字段\ndf_test.drop('hour',axis=1,inplace=True)\ndf_test.head()\n\n## 删除C17、C1和C20\ndf_test.drop(['C17','C1','C20'],inplace=True,axis=1)\n\n\ntest = df_test.iloc[:,1:]","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:22:34.527599Z","iopub.execute_input":"2023-12-13T13:22:34.527986Z","iopub.status.idle":"2023-12-13T13:22:56.646608Z","shell.execute_reply.started":"2023-12-13T13:22:34.527951Z","shell.execute_reply":"2023-12-13T13:22:56.645538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 对所有特征进行编码处理\nfrom sklearn.preprocessing import LabelEncoder\n\ntrain_test = pd.concat([x_bal,test],axis=0)\nfeatures_encoder = train_test.columns\n\nfor feature in features_encoder:\n    le = LabelEncoder()\n    le.fit(train_test[feature])\n    train_test[feature] = le.transform(train_test[feature])\n\ntrain_test.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:22:56.647899Z","iopub.execute_input":"2023-12-13T13:22:56.648201Z","iopub.status.idle":"2023-12-13T13:23:26.587984Z","shell.execute_reply.started":"2023-12-13T13:22:56.648175Z","shell.execute_reply":"2023-12-13T13:23:26.586988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 对数据进行标准化处理\nfrom sklearn.preprocessing import StandardScaler\ntrain_test = StandardScaler().fit_transform(train_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:23:26.589472Z","iopub.execute_input":"2023-12-13T13:23:26.589955Z","iopub.status.idle":"2023-12-13T13:23:29.276450Z","shell.execute_reply.started":"2023-12-13T13:23:26.589916Z","shell.execute_reply":"2023-12-13T13:23:29.275580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 重新将训练集测试分离出来\nx_bal = train_test[:x_bal_len,:]\ntest = train_test[x_bal_len:,:]\nprint(len(x_bal))\nprint(len(test))","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:23:29.277625Z","iopub.execute_input":"2023-12-13T13:23:29.277928Z","iopub.status.idle":"2023-12-13T13:23:29.488796Z","shell.execute_reply.started":"2023-12-13T13:23:29.277903Z","shell.execute_reply":"2023-12-13T13:23:29.487483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_bal","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:23:29.490362Z","iopub.execute_input":"2023-12-13T13:23:29.490992Z","iopub.status.idle":"2023-12-13T13:23:29.503533Z","shell.execute_reply.started":"2023-12-13T13:23:29.490953Z","shell.execute_reply":"2023-12-13T13:23:29.502485Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Modeling Selection","metadata":{}},{"cell_type":"code","source":"## 划分训练集和测试集\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(x_bal, y_bal, test_size=0.2,random_state=42)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:23:29.504448Z","iopub.execute_input":"2023-12-13T13:23:29.504752Z","iopub.status.idle":"2023-12-13T13:23:29.848825Z","shell.execute_reply.started":"2023-12-13T13:23:29.504728Z","shell.execute_reply":"2023-12-13T13:23:29.847722Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 定义评价函数\n\n# 混淆矩阵\ndef conf_mat(model,x,y):\n    from sklearn.metrics import confusion_matrix\n\n    # 计算混淆矩阵\n    y_pred = model.predict(x)\n    cm = confusion_matrix(y, y_pred)\n\n    # 创建一个热图来可视化混淆矩阵\n    plt.figure(figsize=(8, 6))\n    sns.heatmap(cm, annot=True, cmap='Blues', fmt='g', \n                xticklabels=['Predicted 0', 'Predicted 1'],\n                yticklabels=['Actual 0', 'Actual 1'])\n\n    plt.title('Confusion Matrix')\n    plt.xlabel('Predicted label')\n    plt.ylabel('True label')\n    plt.show()\n\n# 准确率和分类报告\ndef model_val(model,x,y):\n    from sklearn.metrics import accuracy_score\n    from sklearn.metrics import classification_report\n\n    acc = model.score(x,y)\n    cr = classification_report(y, model.predict(x))\n    \n    return acc,cr\n\n# 损失函数\ndef model_loss(model,x,y):\n    from sklearn.metrics import log_loss\n\n    y_pred_proba = model.predict_proba(x)[:,1]\n    loss = log_loss(y,y_pred_proba)\n    \n    return loss\n\n# ROC曲线\ndef draw_roc(model, x, y):\n    from sklearn import metrics\n    y_pred = model.predict_proba(x)[:,1]\n    fpr, tpr, thresholds = metrics.roc_curve(y, y_pred,\n                                              drop_intermediate = False )\n    auc_score = metrics.roc_auc_score(y, y_pred )\n    plt.figure(figsize=(6, 6))\n    plt.plot( fpr, tpr, label='ROC curve (area = %0.2f)' % auc_score )\n    plt.plot([0, 1], [0, 1], 'k--')\n    plt.xlim([0.0, 1.0])\n    plt.ylim([0.0, 1.05])\n    plt.xlabel('False Positive Rate or [1 - True Negative Rate]')\n    plt.ylabel('True Positive Rate')\n    plt.title('Receiver operating characteristic example')\n    plt.legend(loc=\"lower right\")\n    plt.show()\n\n    return fpr, tpr, thresholds","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:23:29.850349Z","iopub.execute_input":"2023-12-13T13:23:29.850667Z","iopub.status.idle":"2023-12-13T13:23:29.863372Z","shell.execute_reply.started":"2023-12-13T13:23:29.850640Z","shell.execute_reply":"2023-12-13T13:23:29.862429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 使用随机森林模型\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.ensemble import RandomForestClassifier\n\nstart_time = time.time()\n\nrfc_clf = RandomForestClassifier(n_estimators=100,max_depth=5,random_state=42,n_jobs=-1)\nrfc_clf.fit(X_train,y_train)\n\nend_time = time.time()\n\nprint(end_time-start_time)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:23:29.864893Z","iopub.execute_input":"2023-12-13T13:23:29.865209Z","iopub.status.idle":"2023-12-13T13:24:10.800620Z","shell.execute_reply.started":"2023-12-13T13:23:29.865182Z","shell.execute_reply":"2023-12-13T13:24:10.799607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 随机森林模型评分\nacc_train, cr_train = model_val(rfc_clf,X_train,y_train)\nacc_test, cr_test = model_val(rfc_clf,X_test,y_test)\nprint(f\"训练集准确率{acc_train}\")\nprint(f\"测试集准确率{acc_test}\")\nprint(f\"训练集分类报告：\\n{cr_train}\")\nprint(f\"测试集分类报告：\\n{cr_test}\")\n\n## 模型似乎过拟合了，需要更换模型","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:10.801939Z","iopub.execute_input":"2023-12-13T13:24:10.802240Z","iopub.status.idle":"2023-12-13T13:24:18.488860Z","shell.execute_reply.started":"2023-12-13T13:24:10.802213Z","shell.execute_reply":"2023-12-13T13:24:18.487851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 随机森林模型loss评分\nloss_train = model_loss(rfc_clf,X_train,y_train)\nloss_test = model_loss(rfc_clf,X_test,y_test)\nprint(f\"训练集loss\\n{loss_train}\")\nprint(f\"测试集loss\\n{loss_test}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:18.490189Z","iopub.execute_input":"2023-12-13T13:24:18.490575Z","iopub.status.idle":"2023-12-13T13:24:21.525410Z","shell.execute_reply.started":"2023-12-13T13:24:18.490539Z","shell.execute_reply":"2023-12-13T13:24:21.524487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 随机森林模型的混淆矩阵\nconf_mat(rfc_clf,X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:21.527201Z","iopub.execute_input":"2023-12-13T13:24:21.527520Z","iopub.status.idle":"2023-12-13T13:24:22.389508Z","shell.execute_reply.started":"2023-12-13T13:24:21.527492Z","shell.execute_reply":"2023-12-13T13:24:22.388595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 随机森林模型的roc曲线\ndraw_roc(rfc_clf, X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:22.390709Z","iopub.execute_input":"2023-12-13T13:24:22.390996Z","iopub.status.idle":"2023-12-13T13:24:23.266262Z","shell.execute_reply.started":"2023-12-13T13:24:22.390971Z","shell.execute_reply":"2023-12-13T13:24:23.265302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 随机森林模型的ROC曲线表现不错，但是由评分可看出模型对数据过拟合，更换XGBoost模型","metadata":{}},{"cell_type":"code","source":"## 使用xgb模型进行训练\nfrom xgboost import XGBClassifier\n\nstart_time = time.time()\n\n# 建模\nxgb_clf = XGBClassifier(tree_method = 'hist',n_estimators=500,max_depth=5,random_state=42,device = \"cuda\",eval_metric='logloss')\n\n# 训练\nxgb_clf.fit(X_train, y_train)\n\nend_time = time.time()\n\nprint(end_time-start_time)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:23.267355Z","iopub.execute_input":"2023-12-13T13:24:23.267644Z","iopub.status.idle":"2023-12-13T13:24:27.686233Z","shell.execute_reply.started":"2023-12-13T13:24:23.267618Z","shell.execute_reply":"2023-12-13T13:24:27.685340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## xgb模型评分\nacc_train, cr_train = model_val(xgb_clf,X_train,y_train)\nacc_test, cr_test = model_val(xgb_clf,X_test,y_test)\nprint(f\"训练集准确率{acc_train}\")\nprint(f\"测试集准确率{acc_test}\")\nprint(f\"训练集分类报告：\\n{cr_train}\")\nprint(f\"测试集分类报告：\\n{cr_test}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:27.687478Z","iopub.execute_input":"2023-12-13T13:24:27.687867Z","iopub.status.idle":"2023-12-13T13:24:30.685295Z","shell.execute_reply.started":"2023-12-13T13:24:27.687823Z","shell.execute_reply":"2023-12-13T13:24:30.684358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## XGBoost模型loss评分\nloss_train = model_loss(xgb_clf,X_train,y_train)\nloss_test = model_loss(xgb_clf,X_test,y_test)\nprint(f\"训练集loss\\n{loss_train}\")\nprint(f\"测试集loss\\n{loss_test}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:30.686589Z","iopub.execute_input":"2023-12-13T13:24:30.686869Z","iopub.status.idle":"2023-12-13T13:24:31.361594Z","shell.execute_reply.started":"2023-12-13T13:24:30.686844Z","shell.execute_reply":"2023-12-13T13:24:31.360688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## xgb模型roc曲线\ndraw_roc(xgb_clf, X_test, y_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:31.362840Z","iopub.execute_input":"2023-12-13T13:24:31.363123Z","iopub.status.idle":"2023-12-13T13:24:31.867396Z","shell.execute_reply.started":"2023-12-13T13:24:31.363098Z","shell.execute_reply":"2023-12-13T13:24:31.866305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 加多一个模型进行对比，使用模型LightGBM（与XGBoost模型同为GBDT类）","metadata":{}},{"cell_type":"code","source":"## 使用lgb模型进行训练\nfrom lightgbm import LGBMClassifier\n\nstart_time = time.time()\n\n# 建模\n## 不能定义深度，因为该模型是基于leaf—wise生长\nlgb_clf = LGBMClassifier(n_estimators=500,random_state=42)\n\n# 训练\nlgb_clf.fit(X_train, y_train)\n\nend_time = time.time()\n\nprint(end_time-start_time)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:31.868885Z","iopub.execute_input":"2023-12-13T13:24:31.869247Z","iopub.status.idle":"2023-12-13T13:24:54.102807Z","shell.execute_reply.started":"2023-12-13T13:24:31.869216Z","shell.execute_reply":"2023-12-13T13:24:54.102064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## lgb模型评分\nacc_train, cr_train = model_val(lgb_clf,X_train,y_train)\nacc_test, cr_test = model_val(lgb_clf,X_test,y_test)\nprint(f\"训练集准确率{acc_train}\")\nprint(f\"测试集准确率{acc_test}\")\nprint(f\"训练集分类报告：\\n{cr_train}\")\nprint(f\"训练集分类报告：\\n{cr_test}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:24:54.104238Z","iopub.execute_input":"2023-12-13T13:24:54.104863Z","iopub.status.idle":"2023-12-13T13:25:14.913738Z","shell.execute_reply.started":"2023-12-13T13:24:54.104831Z","shell.execute_reply":"2023-12-13T13:25:14.912442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## LightGBM模型loss评分\nloss_train = model_loss(lgb_clf,X_train,y_train)\nloss_test = model_loss(lgb_clf,X_test,y_test)\nprint(f\"训练集loss评分\\n{loss_train}\")\nprint(f\"测试集loss评分\\n{loss_test}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:14.915203Z","iopub.execute_input":"2023-12-13T13:25:14.915566Z","iopub.status.idle":"2023-12-13T13:25:24.117476Z","shell.execute_reply.started":"2023-12-13T13:25:14.915504Z","shell.execute_reply":"2023-12-13T13:25:24.116544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_roc(lgb_clf,X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:24.118845Z","iopub.execute_input":"2023-12-13T13:25:24.119218Z","iopub.status.idle":"2023-12-13T13:25:26.265732Z","shell.execute_reply.started":"2023-12-13T13:25:24.119183Z","shell.execute_reply":"2023-12-13T13:25:26.264817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 模型结果都差不多，随机森林模型存在过拟合风险，XGBoost在调用GPU进行训练的时候速度比LightGBM快(特征数量造成的)，而两者模型结果差不多，故此选用XGBoost模型。","metadata":{}},{"cell_type":"markdown","source":"### Hyperparameter tuning of XGBoost model\n#### key params:\n* 'n_estimators':[i for i in range(30,1000,50)]\n* 'max_depth':[3, 5, 6, 7, 9, 12, 15],\n* 'min_child_weight': [1, 3, 5, 7],\n* 'gamma':[0, 0.05, 0.07, 0.1, 0.3, 0.5, 0.7, 0.9, 1],\n* 'learning_rate':[0.01,0.015,0.025,0.05,0.1],\n* 'subsample':[0.6, 0.7, 0.8, 0.9, 1],\n* 'colsample_bytree':[0.6, 0.7, 0.8, 0.9, 1],\n* 'reg_lambda':[1e-5, 1e-2, 0.1,0.5, 1]\n* 'reg_alpha':[1e-5, 1e-2, 0.1, 0.5, 1],","metadata":{}},{"cell_type":"code","source":"## 第一次调参\n# from sklearn.model_selection import GridSearchCV\n# from xgboost import XGBClassifier\n\n# # 建模\n# xgb = XGBClassifier(tree_method='hist',device='cuda',objective='binary:logistic',booster='gbtree',random_state=42,eval_metric='logloss',\n#                    n_estimators=500,max_depth=8)\n\n\n# param1 = {\n#     'max_depth':[3, 5, 6, 7, 9, 12, 15],\n#     'min_child_weight': [1, 3, 5, 7]\n# }\n\n# gsearch1 = GridSearchCV(estimator = xgb, param_grid = param1,\n#                            scoring = 'neg_log_loss', cv = 3,verbose = 1)\n# gsearch1.fit(X_train,y_train)\n# print('gsearch1.best_params_', gsearch1.best_params_)\n# print('gsearch1.best_score_', gsearch1.best_score_)\n\n\n# # gsearch1.best_params_ {'max_depth': 3, 'min_child_weight': 1}\n# # gsearch1.best_score_ 0.9576342691486213","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:26.266933Z","iopub.execute_input":"2023-12-13T13:25:26.267206Z","iopub.status.idle":"2023-12-13T13:25:26.272063Z","shell.execute_reply.started":"2023-12-13T13:25:26.267183Z","shell.execute_reply":"2023-12-13T13:25:26.271116Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 第二次调参\n# from sklearn.model_selection import GridSearchCV\n# from xgboost import XGBClassifier\n\n# # 建模\n# xgb = XGBClassifier(tree_method='hist',device='cuda',objective='binary:logistic',booster='gbtree',random_state=42,eval_metric='logloss',\n#                    n_estimators=500,max_depth=3,\n#                    min_child_weight=1)\n\n\n# param2 = {\n#     'n_estimators':[i for i in range(30,1000,50)]\n# }\n\n# gsearch2 = GridSearchCV(estimator = xgb, param_grid = param2,\n#                            scoring = 'neg_log_loss', cv = 3,verbose = 1)\n# gsearch2.fit(X_train,y_train)\n# print('gsearch2.best_params_', gsearch2.best_params_)\n# print('gsearch2.best_score_', gsearch2.best_score_)\n\n# # gsearch2.best_params_ {'n_estimators': 730}\n# # gsearch2.best_score_ -0.2453473994366112","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:26.273439Z","iopub.execute_input":"2023-12-13T13:25:26.273831Z","iopub.status.idle":"2023-12-13T13:25:26.282617Z","shell.execute_reply.started":"2023-12-13T13:25:26.273794Z","shell.execute_reply":"2023-12-13T13:25:26.281666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 第三次调参\n# from sklearn.model_selection import GridSearchCV\n# from xgboost import XGBClassifier\n\n# # 建模\n# xgb = XGBClassifier(tree_method='hist',device='cuda',objective='binary:logistic',booster='gbtree',random_state=42,eval_metric='logloss',\n#                    n_estimators=830,max_depth=3,\n#                    min_child_weight=1)\n\n\n# param3 = {\n#     'gamma':[0, 0.05, 0.07, 0.1, 0.3, 0.5, 0.7, 0.9, 1],\n#     'learning_rate':[0.01,0.015,0.025,0.05,0.1],\n#     'reg_alpha':[1e-5, 1e-2, 0.1, 0.5, 1],\n# }\n\n# gsearch3 = GridSearchCV(estimator = xgb, param_grid = param3,\n#                            scoring = 'neg_log_loss', cv = 3,verbose = 1)\n# gsearch3.fit(X_train,y_train)\n# print('gsearch3.best_params_', gsearch3.best_params_)\n# print('gsearch3.best_score_', gsearch3.best_score_)\n\n# # gsearch3.best_params_ {'gamma': 0.3, 'learning_rate': 0.1, 'reg_alpha': 1}\n# # gsearch3.best_score_ 0.9574809173056372","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:26.283673Z","iopub.execute_input":"2023-12-13T13:25:26.283976Z","iopub.status.idle":"2023-12-13T13:25:26.298835Z","shell.execute_reply.started":"2023-12-13T13:25:26.283952Z","shell.execute_reply":"2023-12-13T13:25:26.297915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 第四次调参\n# from sklearn.model_selection import GridSearchCV\n# from xgboost import XGBClassifier\n\n# # 建模\n# xgb = XGBClassifier(tree_method='hist',device='cuda',objective='binary:logistic',booster='gbtree',random_state=42,eval_metric='logloss',\n#                    n_estimators=830,max_depth=3,\n#                    min_child_weight=1,\n#                    gamma=0.3,\n#                    learining_rate=0.1,\n#                    reg_alpha=1)\n\n\n# param4 = {\n#     'subsample':[0.6, 0.7, 0.8, 0.9, 1],\n#     'colsample_bytree':[0.6, 0.7, 0.8, 0.9, 1],\n#     'reg_lambda':[1e-5, 1e-2, 0.1,0.5, 1]\n# }\n\n# gsearch4 = GridSearchCV(estimator = xgb, param_grid = param4,\n#                            scoring = 'neg_log_loss', cv = 3,verbose = 1)\n# gsearch4.fit(X_train,y_train)\n# print('gsearch3.best_params_', gsearch4.best_params_)\n# print('gsearch3.best_score_', gsearch4.best_score_)\n\n# # gsearch3.best_params_ {'colsample_bytree': 0.6, 'reg_lambda': 1, 'subsample': 0.9}\n# # gsearch3.best_score_ 0.9577570544957386","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:26.299944Z","iopub.execute_input":"2023-12-13T13:25:26.300225Z","iopub.status.idle":"2023-12-13T13:25:26.310594Z","shell.execute_reply.started":"2023-12-13T13:25:26.300190Z","shell.execute_reply":"2023-12-13T13:25:26.309596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xgb_tune = XGBClassifier(tree_method='hist',device='cuda',objective='binary:logistic',booster='gbtree',random_state=42,eval_metric='logloss',\n                   n_estimators=730,max_depth=3,\n                   min_child_weight=1,\n                   gamma=0.3,\n                   learining_rate=0.1,\n                   reg_alpha=1,\n                   subsample=0.9,\n                   colsample_bytree=0.6,\n                   reg_lambda=1)\n\nxgb_tune.fit(X_train,y_train)\n\nacc_train, cr_train = model_val(xgb_tune,X_train,y_train)\nacc_test, cr_test = model_val(xgb_tune,X_test,y_test)\nprint(f\"训练集调参后准确率{acc_train}\")\nprint(f\"测试集调参后准确率{acc_test}\")\nprint(f\"训练集分类报告：\\n{cr_train}\")\nprint(f\"测试集分类报告：\\n{cr_test}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:26.312134Z","iopub.execute_input":"2023-12-13T13:25:26.312535Z","iopub.status.idle":"2023-12-13T13:25:35.011809Z","shell.execute_reply.started":"2023-12-13T13:25:26.312484Z","shell.execute_reply":"2023-12-13T13:25:35.010825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## XGBoost模型调参后loss评分\nloss_train = model_loss(xgb_tune,X_train,y_train)\nloss_test = model_loss(xgb_tune,X_test,y_test)\nprint(f\"训练集调参后loss评分\\n{loss_train}\")\nprint(f\"测试集调参后loss评分\\n{loss_test}\")","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:35.013199Z","iopub.execute_input":"2023-12-13T13:25:35.013612Z","iopub.status.idle":"2023-12-13T13:25:35.681323Z","shell.execute_reply.started":"2023-12-13T13:25:35.013575Z","shell.execute_reply":"2023-12-13T13:25:35.680170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conf_mat(xgb_tune,X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:35.682417Z","iopub.execute_input":"2023-12-13T13:25:35.682794Z","iopub.status.idle":"2023-12-13T13:25:36.158723Z","shell.execute_reply.started":"2023-12-13T13:25:35.682759Z","shell.execute_reply":"2023-12-13T13:25:36.157610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"draw_roc(xgb_tune,X_test,y_test)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:36.160722Z","iopub.execute_input":"2023-12-13T13:25:36.161367Z","iopub.status.idle":"2023-12-13T13:25:36.738061Z","shell.execute_reply.started":"2023-12-13T13:25:36.161333Z","shell.execute_reply":"2023-12-13T13:25:36.737013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Output the submission file","metadata":{}},{"cell_type":"code","source":"## 输出数据\ntest_pred = xgb_tune.predict_proba(test)\nsubmission = pd.DataFrame({'id':df_test.iloc[:,0],'click':test_pred[:,1]})\nsubmission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:36.739926Z","iopub.execute_input":"2023-12-13T13:25:36.740407Z","iopub.status.idle":"2023-12-13T13:25:54.816396Z","shell.execute_reply.started":"2023-12-13T13:25:36.740371Z","shell.execute_reply":"2023-12-13T13:25:54.815291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-13T13:25:54.817751Z","iopub.execute_input":"2023-12-13T13:25:54.818055Z","iopub.status.idle":"2023-12-13T13:25:54.827862Z","shell.execute_reply.started":"2023-12-13T13:25:54.818029Z","shell.execute_reply":"2023-12-13T13:25:54.826823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}