{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"},{"sourceId":378227,"sourceType":"datasetVersion","datasetId":166071}],"dockerImageVersionId":29976,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nimport gzip\n\ntest_file = '../input/avazu-ctr-prediction/test.gz'\nsamplesubmision_file = '../input/avazu-ctr-prediction/sampleSubmission.gz'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-23T06:59:25.511202Z","iopub.execute_input":"2024-04-23T06:59:25.511624Z","iopub.status.idle":"2024-04-23T06:59:25.616115Z","shell.execute_reply.started":"2024-04-23T06:59:25.511589Z","shell.execute_reply":"2024-04-23T06:59:25.615288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"由於文件的Row數量太過龐大 （超過4千萬筆） ，Pandas DataFrame無法讀取這麼大的資料量，因此資料以每100萬筆為一批次分批讀取，並且從每個批次的資料中，隨機抽樣5％的資料，集成一個新的train set。","metadata":{}},{"cell_type":"code","source":"chunksize = 10 ** 6\nnum_of_chunk = 0\ntrain = pd.DataFrame()\n    \nfor chunk in pd.read_csv('../input/avazu-ctr-train/train.csv', chunksize=chunksize):\n    num_of_chunk += 1\n    train = pd.concat([train, chunk.sample(frac=.05, replace=False, random_state=123)], axis=0)\n    print('Processing Chunk No. ' + str(num_of_chunk))     \n    \ntrain.reset_index(inplace=True)\n\n# 備份train 資料長度，以便稍後df重新分割索引用途\ntrain_len = len(train)\ntrain_len","metadata":{"execution":{"iopub.status.busy":"2024-04-23T06:59:25.618343Z","iopub.execute_input":"2024-04-23T06:59:25.618669Z","iopub.status.idle":"2024-04-23T07:03:21.353771Z","shell.execute_reply.started":"2024-04-23T06:59:25.618637Z","shell.execute_reply":"2024-04-23T07:03:21.352855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"再將test資料集的資料讀出，並把train, test 合併成一個df，以便同時進行資料預處理。","metadata":{}},{"cell_type":"code","source":"df = pd.concat([train, pd.read_csv(test_file, compression='gzip')]).drop(['index', 'id'], axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T07:03:21.355258Z","iopub.execute_input":"2024-04-23T07:03:21.355583Z","iopub.status.idle":"2024-04-23T07:03:44.002690Z","shell.execute_reply.started":"2024-04-23T07:03:21.355550Z","shell.execute_reply":"2024-04-23T07:03:44.001440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"考量本次預測目的為用10天的點擊狀況訓練模型，用以預測地11天的狀況，因此年月日期無意義，但weekday理論上有意義。所以把原始hour特徵中的日期，轉化為weekday。\n\n又，每天的時段依據每個人不同的生活習慣，理論上有意義。未避免太多變數造成維度崩潰24，將小時的時間區分為12個時段，每兩小時一個時段。","metadata":{}},{"cell_type":"code","source":"# 建立一個將hour資料轉換為日期格式的function\ndef get_date(hour):\n    y = '20'+str(hour)[:2]\n    m = str(hour)[2:4]\n    d = str(hour)[4:6]\n    return y+'-'+m+'-'+d\n\n# 建立weekday欄位，將hour轉換後填入\ndf['weekday'] = pd.to_datetime(df.hour.apply(get_date)).dt.dayofweek.astype(str)\n\n# 建立一個將hour資料轉換為時段的function\ndef tran_hour(x):\n    x = x % 100\n    while x in [23,0]:\n        return '23-01'\n    while x in [1,2]:\n        return '01-03'\n    while x in [3,4]:\n        return '03-05'\n    while x in [5,6]:\n        return '05-07'\n    while x in [7,8]:\n        return '07-09'\n    while x in [9,10]:\n        return '09-11'\n    while x in [11,12]:\n        return '11-13'\n    while x in [13,14]:\n        return '13-15'\n    while x in [15,16]:\n        return '15-17'\n    while x in [17,18]:\n        return '17-19'\n    while x in [19,20]:\n        return '19-21'\n    while x in [21,22]:\n        return '21-23'\n\n# 將hour轉換為時段\ndf['hour'] = df.hour.apply(tran_hour)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T07:03:44.004064Z","iopub.execute_input":"2024-04-23T07:03:44.004402Z","iopub.status.idle":"2024-04-23T07:04:10.757370Z","shell.execute_reply.started":"2024-04-23T07:03:44.004368Z","shell.execute_reply":"2024-04-23T07:04:10.756359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 確認資料型別\ndf.info()","metadata":{"execution":{"iopub.status.busy":"2024-04-23T07:04:10.761760Z","iopub.execute_input":"2024-04-23T07:04:10.762088Z","iopub.status.idle":"2024-04-23T07:04:13.670402Z","shell.execute_reply.started":"2024-04-23T07:04:10.762058Z","shell.execute_reply":"2024-04-23T07:04:13.669534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"確認每個特徵的value count，發現int型別的特徵，最多value的特徵僅4,333個value count，再一個600萬筆資料的資料集中，顯然非連續行變數。由此可斷定，本資料集的所有特徵，皆為Object型態的變數。","metadata":{}},{"cell_type":"code","source":"len_of_feature_count = []\nfor i in df.columns[2:23].tolist():\n    print(i, ':', len(df[i].astype(str).value_counts()))\n    len_of_feature_count.append(len(df[i].astype(str).value_counts()))","metadata":{"execution":{"iopub.status.busy":"2024-04-23T07:04:13.680685Z","iopub.execute_input":"2024-04-23T07:04:13.680989Z","iopub.status.idle":"2024-04-23T07:08:48.373596Z","shell.execute_reply.started":"2024-04-23T07:04:13.680959Z","shell.execute_reply":"2024-04-23T07:08:48.372723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 建立一個list，將需要轉換行別的特徵名稱存入該list\nneed_tran_feature = df.columns[2:4].tolist() + df.columns[13:23].tolist()\n\n# 依序將變數轉換為object型別\nfor i in need_tran_feature:\n    df[i] = df[i].astype(str)","metadata":{"execution":{"iopub.status.busy":"2024-04-23T07:08:48.374743Z","iopub.execute_input":"2024-04-23T07:08:48.375065Z","iopub.status.idle":"2024-04-23T07:10:16.883652Z","shell.execute_reply.started":"2024-04-23T07:08:48.375033Z","shell.execute_reply":"2024-04-23T07:10:16.882617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"一些特徵的value count極高，甚至有上百萬個資料值，這種情況進行 one-hot編碼，無疑會造成維度崩潰。在此將每個特徵的value count以10為限10，一旦超過10個值，則將進行縮減作業。\n\n縮減的方式為，計算某變數所有值的點擊率，依點擊率區分為very_high, higher, mid, lower, very_low，等5個級距。","metadata":{}},{"cell_type":"code","source":"obj_features = []\n\nfor i in range(len(len_of_feature_count)):\n    if len_of_feature_count[i] > 10:\n        obj_features.append(df.columns[2:23].tolist()[i])\nobj_features","metadata":{"execution":{"iopub.status.busy":"2024-04-23T07:10:16.884889Z","iopub.execute_input":"2024-04-23T07:10:16.885178Z","iopub.status.idle":"2024-04-23T07:10:16.893725Z","shell.execute_reply.started":"2024-04-23T07:10:16.885151Z","shell.execute_reply":"2024-04-23T07:10:16.892833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_describe = df.describe()\ndf_describe","metadata":{"execution":{"iopub.status.busy":"2024-04-23T07:10:16.894753Z","iopub.execute_input":"2024-04-23T07:10:16.895052Z","iopub.status.idle":"2024-04-23T07:10:23.756518Z","shell.execute_reply.started":"2024-04-23T07:10:16.895024Z","shell.execute_reply":"2024-04-23T07:10:23.755522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def obj_clean(X):\n    # 定義一個縮減資料值的function，每次處理一個特徵向量\n\n    def get_click_rate(x):\n        # 定義一個取得點擊率的function\n        temp = train[train[X.columns[0]] == x]\n        res = round((temp.click.sum() / temp.click.count()),3)\n        return res\n\n    def get_type(V, str):\n        # 定義一個取得新資料值之級距判斷的function\n        very_high = df_describe.loc['mean','click'] + 0.04\n        higher = df_describe.loc['mean','click'] + 0.02\n        lower = df_describe.loc['mean','click'] - 0.02\n        very_low = df_describe.loc['mean','click'] - 0.04\n\n        vh_type = V[V[str] > very_high].index.tolist()\n        hr_type = V[(V[str] > higher) & (V[str] < very_high)].index.tolist()\n        vl_type = V[V[str] < very_low].index.tolist()\n        lr_type = V[(V[str] < lower) & (V[str] > very_low)].index.tolist()\n\n        return vh_type, hr_type, vl_type, lr_type\n\n    def clean_function(x):\n        # 定義一個依據級距轉換資料值的function\n        # 判斷之依據為：總平均點擊率的正負  4% 為very_high(low), 總平均點擊率的正負 2％為higher (lower)\n        while x in type_[0]:\n            return 'very_high'\n        while x in type_[1]:\n            return 'higher'\n        while x in type_[2]:\n            return 'very_low'\n        while x in type_[3]:\n            return 'lower'\n        return 'mid'\n        \n    print('Run: ', X.columns[0])\n    fq = X[X.columns[0]].value_counts()\n    # 建立一個暫存的資料值頻率列表\n    # 理論上，將全部的資料值都進行分類轉換，可得到最佳效果；實務上為了執行時間效能，將捨去頻率低於排名前1000 row以後的資料值。\n    if len(fq) > 1000:\n        fq = fq[:1000]\n\n    # 將頻率列表轉換為dataframe，並將index填入一個新的欄位。\n    fq = pd.DataFrame(fq)\n    fq['new_column'] = fq.index    \n\n    # 使用index叫用get_click_rate function，取得每個資料值的點擊率\n    fq['click_rate'] = fq.new_column.apply(get_click_rate)\n\n    # 叫用 get_type function取得分類級距，並儲存為一個list，以便提供給下一個clean_function使用\n    type_ = get_type(fq, 'click_rate')\n\n    # 叫用 clean_funtion funtion，回傳轉換後的特徵向量\n    return X[X.columns[0]].apply(clean_function)\n\n# 使用for 迴圈將需轉換的特徵輸入到 obj_clean function\nfor i in obj_features:    \n    df[[i]] = obj_clean(df[[i]])\n\ndf","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 確認所有特徵的資料值狀況\nfor i in df.columns:\n    sns.countplot(x = i, hue = \"click\", data = df)\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"根據上面所列出的圖表來看，顯然 ['device_id', 'C14', 'C17', 'C19', 'C20', 'C21'] 這些特徵，僅有一種value，對於預測模型來說沒有意義，因此將這些特徵移出資料集。","metadata":{}},{"cell_type":"code","source":"df.drop(['device_id', 'C14', 'C17', 'C19', 'C20', 'C21'], axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 對所有變數進行 one-hot 編碼\ndf = pd.get_dummies(df)\n\n# 依據處理過得df資料表，重新將train, test分割出來\ntrain = df[:train_len]\ntest = df[train_len:]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # 將處理過的train, test 資料集匯出，避免每次重新的冗長處理時間。\n\n# train.to_csv('new_train.csv', index=False)\n# test.to_csv('new_test.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # 讀取處理過的train, test 資料集，跳過冗長的重新執行處理時間。\n# train = pd.read_csv('new_train.csv')\n# test = pd.read_csv('new_test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"由於資料集非常的龐大，同時正向label的佔比僅佔全部資料17％左右，比例明顯失衡，需要具有強化加權功能的演算法。因此在此決定使用xgboost演算法，解決強化權重問題，同時運用GPU有效節省運算時間。\n\n為節省調參時間，實作預測模型之前，將先以建立100株決策樹，以Grid Search尋找最佳參數與重要特徵，最後再以xgboost演算法建立模型。為了縮短決策樹的建制時間，將從負向label的資料中抽樣，與所有正向label的資料整併成一份各佔50％的資料集，來平衡權重問題。同時也因為正反label比例平衡，將會採用ROC_AUC 分數來進行調參。","metadata":{}},{"cell_type":"code","source":"# 從train資料集中，標籤為0的資料中，隨機抽樣與標籤為1一樣多的數量，並將其結合成正反標籤佔筆各佔50％的資料集\npre_X = train[train['click'] == 0].sample(n=len(train[train['click'] == 1]), random_state=111)\npre_X = pd.concat([pre_X, train[train['click'] == 1]]).sample(frac=1)\npre_y = pre_X[['click']]\npre_X.drop(['click'], axis=1, inplace=True)\ntest.drop(['click'], axis=1, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\n# 將新的資料集分割為訓練集與驗證集\npre_X_train, pre_X_test, pre_y_train, pre_y_test = train_test_split(pre_X, pre_y, test_size=0.20, stratify=pre_y, random_state=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 執行Grid Search調參，建立100棵樹來取得最佳參數\nparams = {\"criterion\":[\"gini\", \"entropy\"], \"max_depth\":range(1,20)}\ngrid_search = GridSearchCV(DecisionTreeClassifier(), param_grid=params, scoring='roc_auc', cv=100, verbose=1, n_jobs=-1)\ngrid_search.fit(pre_X_train, pre_y_train)\ngrid_search.best_score_, grid_search.best_estimator_, grid_search.best_params_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 依據Grid Search的結果建立一個決策樹模型，並fit完整資料 (前置資料)\ntree = grid_search.best_estimator_\ntree.fit(pre_X,pre_y)\n\n# 輸出重要特徵，並依特徵之重要性排序\nfeature_importances = pd.DataFrame(tree.feature_importances_)\nfeature_importances.index = pre_X_train.columns\nfeature_importances = feature_importances.sort_values(0,ascending=False)\nfeature_importances","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 調整前置作業訓練集與驗證集，將特徵依特徵重要性縮減為重要性排名之1/3\npre_X_train = pre_X_train[feature_importances.index[:int(len(feature_importances)/3)]]\npre_X_test = pre_X_test[feature_importances.index[:int(len(feature_importances)/3)]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 使用33％的重要特徵重新進行Grid Search調參\nparams = {\"criterion\":[\"gini\", \"entropy\"], \"max_depth\":range(1,12)}\ngrid_search = GridSearchCV(DecisionTreeClassifier(), param_grid=params, scoring='roc_auc', cv=100, verbose=1, n_jobs=-1)\ngrid_search.fit(pre_X_train, pre_y_train)\ngrid_search.best_score_, grid_search.best_estimator_, grid_search.best_params_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 調整前置作業完整資料集，將特徵依特徵重要性縮減為重要性排名之1/3\npre_X = pre_X[feature_importances.index[:int(len(feature_importances)/3)]]\n\n# 依據Grid Search的結果建立一個決策樹模型，並fit完整資料 (前置資料)\ntree = grid_search.best_estimator_\ntree.fit(pre_X,pre_y)\n\n# 輸出重要特徵，並依特徵之重要性排序\nfeature_importances = pd.DataFrame(tree.feature_importances_)\nfeature_importances.index = pre_X_train.columns\nfeature_importances = feature_importances.sort_values(0,ascending=False)\nfeature_importances","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 最終預測模型之特徵，將採用特徵值 .005以上的變數\nfeature_len = len(feature_importances[feature_importances[feature_importances.columns[0]] > 0.005])\n\n# 調整最終完整Train Set 與 Test set之特徵\ny = train[['click']]\nX = train[feature_importances[:feature_len].index]\ntest = test[feature_importances[:feature_len].index]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from xgboost import XGBClassifier\nfrom sklearn.metrics import roc_auc_score, confusion_matrix, roc_curve\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\n# 使用xgboost 建模，並指定先前調參得到的節點深度限制\nmodel = XGBClassifier(tree_method='gpu_hist', n_jobs=-1, n_estimators=500, max_depth=11)\nmodel.fit(X, y.values.ravel())\ny_pred = model.predict(X)\nprint(\"Roc_auc_score: \", roc_auc_score(y, y_pred) * 100, \"%\")\n\n# 繪出混淆矩陣，查看預測結果\nconfmat = confusion_matrix(y_true=y, y_pred=y_pred, labels=[0, 1])\n\nfig, ax = plt.subplots(figsize=(2.5, 2.5))\nax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)\nfor i in range(confmat.shape[0]):\n    for j in range(confmat.shape[1]):\n        ax.text(x=j, y=i, s=confmat[i, j], va='center', ha='center')\n\nplt.xlabel('Predicted label')\nplt.ylabel('True label')\n\nplt.tight_layout()\nplt.show()\n\n# 绘制 ROC 曲线\nfpr, tpr, thresholds = roc_curve(y, model.predict_proba(X)[:, 1])\nplt.figure()\nplt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc_score(y, model.predict(X)))\nplt.plot([0, 1], [0, 1], 'k--')\nplt.xlim([0.0, 1.0])\nplt.ylim([0.0, 1.05])\nplt.xlabel('False Positive Rate')\nplt.ylabel('True Positive Rate')\nplt.title('Receiver Operating Characteristic Curve')\nplt.legend(loc=\"lower right\")\nplt.show()\n\n# 匯出submission並進行提交\nsubmission = pd.read_csv(samplesubmision_file, compression='gzip', index_col='id')\nsubmission[submission.columns[0]] = model.predict_proba(test)[:, 1]\nsubmission.to_csv('submission.csv')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"本次提交分數為 .40560；Leader Board Top 1 分數為 0.37913","metadata":{}}]}