{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**This kernel uses Deepfm model from deepctr package**\n\n**Deepfm :** [Deepfm using deepctr](https://deepctr-doc.readthedocs.io/en/v0.6.3/deepctr.models.deepfm.html)\n\nGuo H, Tang R, Ye Y, et al. Deepfm: a factorization-machine based neural network for ctr prediction[J]. arXiv preprint arXiv:1703.04247, 2017.(https://arxiv.org/abs/1703.04247)","metadata":{}},{"cell_type":"code","source":"!pip install torchsummary\n# !pip install deepctr_torch","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:15.528045Z","iopub.execute_input":"2023-11-01T14:50:15.528396Z","iopub.status.idle":"2023-11-01T14:50:26.899028Z","shell.execute_reply.started":"2023-11-01T14:50:15.528346Z","shell.execute_reply":"2023-11-01T14:50:26.897592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 添加input目录\nimport sys\nsys.path.append('../input/deepctr-copy3')    \nprint(sys.path)\nfrom deepctr_torch.inputs import  SparseFeat, DenseFeat, get_feature_names\nfrom deepctr_torch.models import *","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:26.904294Z","iopub.execute_input":"2023-11-01T14:50:26.904634Z","iopub.status.idle":"2023-11-01T14:50:35.976263Z","shell.execute_reply.started":"2023-11-01T14:50:26.904579Z","shell.execute_reply":"2023-11-01T14:50:35.974532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import log_loss, roc_auc_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom tensorflow.keras.models import Model, load_model\n\nfrom tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, Callback\nfrom sklearn.preprocessing import OrdinalEncoder, LabelEncoder, OneHotEncoder\nfrom tensorflow.keras.utils import get_custom_objects\nfrom tensorflow.keras.optimizers import Adam,RMSprop\nfrom tensorflow.keras.layers import Activation\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras import utils\nimport tensorflow.keras as keras\nimport tensorflow as tf\nimport pandas as pd\nimport numpy as np\nimport warnings\nwarnings.simplefilter('ignore')\nimport torch\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:35.978176Z","iopub.execute_input":"2023-11-01T14:50:35.978615Z","iopub.status.idle":"2023-11-01T14:50:36.142036Z","shell.execute_reply.started":"2023-11-01T14:50:35.978532Z","shell.execute_reply":"2023-11-01T14:50:36.140827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/cat-in-the-dat-ii/train.csv')\n# test = pd.read_csv('../input/cat-in-the-dat-ii/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:36.143699Z","iopub.execute_input":"2023-11-01T14:50:36.144121Z","iopub.status.idle":"2023-11-01T14:50:40.457672Z","shell.execute_reply.started":"2023-11-01T14:50:36.14404Z","shell.execute_reply":"2023-11-01T14:50:40.456725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['target'].unique()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:40.46169Z","iopub.execute_input":"2023-11-01T14:50:40.462013Z","iopub.status.idle":"2023-11-01T14:50:40.480089Z","shell.execute_reply.started":"2023-11-01T14:50:40.461962Z","shell.execute_reply":"2023-11-01T14:50:40.479052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test['target'] = -1","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:40.482841Z","iopub.execute_input":"2023-11-01T14:50:40.483185Z","iopub.status.idle":"2023-11-01T14:50:40.487997Z","shell.execute_reply.started":"2023-11-01T14:50:40.483131Z","shell.execute_reply":"2023-11-01T14:50:40.48678Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = train","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:40.489591Z","iopub.execute_input":"2023-11-01T14:50:40.489945Z","iopub.status.idle":"2023-11-01T14:50:40.498437Z","shell.execute_reply.started":"2023-11-01T14:50:40.489885Z","shell.execute_reply":"2023-11-01T14:50:40.497669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data['null'] = data.isna().sum(axis=1)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:40.499964Z","iopub.execute_input":"2023-11-01T14:50:40.500548Z","iopub.status.idle":"2023-11-01T14:50:41.628176Z","shell.execute_reply.started":"2023-11-01T14:50:40.500484Z","shell.execute_reply":"2023-11-01T14:50:41.626826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sparse_features = [feat for feat in train.columns if feat not in ['id','target']]\n\ndata[sparse_features] = data[sparse_features].fillna('-1', )","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:41.629826Z","iopub.execute_input":"2023-11-01T14:50:41.63014Z","iopub.status.idle":"2023-11-01T14:50:46.428393Z","shell.execute_reply.started":"2023-11-01T14:50:41.630094Z","shell.execute_reply":"2023-11-01T14:50:46.427317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for feat in sparse_features:\n    lbe = LabelEncoder()\n    data[feat] = lbe.fit_transform(data[feat].fillna('-1').astype(str).values)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:50:46.429799Z","iopub.execute_input":"2023-11-01T14:50:46.43011Z","iopub.status.idle":"2023-11-01T14:51:00.158112Z","shell.execute_reply.started":"2023-11-01T14:50:46.43006Z","shell.execute_reply":"2023-11-01T14:51:00.156469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:00.160282Z","iopub.execute_input":"2023-11-01T14:51:00.160799Z","iopub.status.idle":"2023-11-01T14:51:00.210629Z","shell.execute_reply.started":"2023-11-01T14:51:00.160716Z","shell.execute_reply":"2023-11-01T14:51:00.207967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train = data[data.target != -1].reset_index(drop=True)\n# test  = data[data.target == -1].reset_index(drop=True)\n\nfrom sklearn.model_selection import train_test_split\nrandom_state = 10\n# 以默认的0.25作为分割比例进行分割（训练集:测试集=3:1）\ntrain,test = train_test_split(data,random_state=random_state)\n","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:00.212518Z","iopub.execute_input":"2023-11-01T14:51:00.213293Z","iopub.status.idle":"2023-11-01T14:51:00.517387Z","shell.execute_reply.started":"2023-11-01T14:51:00.212985Z","shell.execute_reply":"2023-11-01T14:51:00.516274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target = ['target']","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:00.519042Z","iopub.execute_input":"2023-11-01T14:51:00.519427Z","iopub.status.idle":"2023-11-01T14:51:00.524211Z","shell.execute_reply.started":"2023-11-01T14:51:00.519368Z","shell.execute_reply":"2023-11-01T14:51:00.523028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## 特征筛选\nfrom catboost import CatBoostClassifier\nmodel = CatBoostClassifier(\n    iterations=100,\n    learning_rate=0.4,\n    task_type='GPU',\n    loss_function='Logloss',\n    depth=8,\n)\n\nfit_model = model.fit(\n    train[sparse_features], train[target].values,\n    eval_set=(test[sparse_features], test[target].values),\n    cat_features = sparse_features,\n    verbose=10\n)\n\nfeature_im = fit_model.feature_importances_","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:00.525839Z","iopub.execute_input":"2023-11-01T14:51:00.526213Z","iopub.status.idle":"2023-11-01T14:51:08.421629Z","shell.execute_reply.started":"2023-11-01T14:51:00.526148Z","shell.execute_reply":"2023-11-01T14:51:08.420298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[sparse_features]","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.422698Z","iopub.status.idle":"2023-11-01T14:51:08.423378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef feature_selection(features, feature_importance, DISTRI=0.7, selection=False, plot=True):\n    '''do feature selection\n    '''\n    feimp = pd.DataFrame({'feature': features,\n                          'importance': feature_importance}).sort_values(by=['importance'], \n                                                           ascending=False)\n    if selection:\n        feimp = feimp.iloc[: int(feimp.shape[0] * DISTRI),]\n    else:\n        pass\n    \n    if plot:\n        plt.figure(figsize=(7, 10))\n        sns.barplot(data = feimp, x='importance', y='feature')\n        plt.title('Catboost Feature Importances')\n    else:\n        pass\n    return feimp\n    \n","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.424461Z","iopub.status.idle":"2023-11-01T14:51:08.424818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feimp = feature_selection(train[sparse_features].columns, feature_im, 0.7, selection=True, plot=False)\nfeimp","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.425699Z","iopub.status.idle":"2023-11-01T14:51:08.426077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train_1 = train[feimp['feature']]\n# test_1 = test[feimp['feature']]","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.427254Z","iopub.status.idle":"2023-11-01T14:51:08.427727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.429109Z","iopub.status.idle":"2023-11-01T14:51:08.429633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sparse_features = [feat for feat in feimp['feature'] if feat not in ['id','target']]","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.430708Z","iopub.status.idle":"2023-11-01T14:51:08.431292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fixlen_feature_columns = [SparseFeat(feat, data[feat].nunique()) for feat in sparse_features]\n\ndnn_feature_columns = fixlen_feature_columns\nlinear_feature_columns = fixlen_feature_columns\n\nfeature_names = get_feature_names(linear_feature_columns + dnn_feature_columns)\n\ntrain_model_input = {name:train[name] for name in feature_names}\ntest_model_input = {name:test[name] for name in feature_names}","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.432501Z","iopub.status.idle":"2023-11-01T14:51:08.43312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = ('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.434344Z","iopub.status.idle":"2023-11-01T14:51:08.434798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = xDeepFM(linear_feature_columns,dnn_feature_columns,task='binary',\n                device=device)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.4362Z","iopub.status.idle":"2023-11-01T14:51:08.436743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    optimizer=torch.optim.AdamW(model.parameters(),lr=3e-5, weight_decay=1e-5), # 3e-4 base_lr=1e-7, max_lr = 1e-4\n    loss='binary_crossentropy',\n    metrics=['binary_crossentropy','auc']\n)\nhist = model.fit(train_model_input, train[target].values, batch_size=1024,\n                    epochs=50,verbose=1,validation_split=0.25)","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.437893Z","iopub.status.idle":"2023-11-01T14:51:08.438429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.439515Z","iopub.status.idle":"2023-11-01T14:51:08.43996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# verbose = 0 为不在标准输出流输出日志信息，verbose = 1 为输出进度条记录，verbose = 2 为每个epoch输出一行记录\n #loss/binary_crossentropy/auc/val_binary_crossentropy/val_auc/\n# 创建一个绘图窗口\nplt.figure()\n \nacc = hist.history['auc']\nval_acc = hist.history['val_auc']\nloss = hist.history['binary_crossentropy']\nval_loss = hist.history['val_binary_crossentropy']\n \nepochs = range(len(acc))\n \nplt.plot(epochs, acc, 'r', label='Training acc') # 'bo'为画蓝色圆点，不连线\nplt.plot(epochs, val_acc, 'b', label='Validation acc') \nplt.title('Kcin+DenseDNN Accuracy')\nplt.legend() # 绘制图例，默认在右上角\nplt.savefig('cin_accuracy.png')","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.44109Z","iopub.status.idle":"2023-11-01T14:51:08.441703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure()\n \nplt.plot(epochs, loss, 'r', label='Training binary_crossentropy')\nplt.plot(epochs, val_loss, 'b', label='Validation binary_crossentropy')\nplt.title('Kcin+DenseDNN Binary_crossentropy')\nplt.legend()\nplt.savefig('cin_binary_crossentropy.png')","metadata":{"execution":{"iopub.status.busy":"2023-11-01T14:51:08.443063Z","iopub.status.idle":"2023-11-01T14:51:08.443586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}