{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport pyarrow as pa\nimport pyarrow.parquet as pq\nfrom tqdm import tqdm\n\ndtypes = {\n    'id': 'uint64', 'click': 'uint8', 'hour': 'uint32', 'C1': 'uint16',\n    'banner_pos': 'uint8', 'site_id': 'string', 'site_domain': 'string',\n    'site_category': 'string', 'app_id': 'string', 'app_domain': 'string',\n    'app_category': 'string', 'device_id': 'string', 'device_ip': 'string',\n    'device_model': 'string', 'device_type': 'uint8', 'device_conn_type': 'uint8',\n    **{f'C{i}': 'uint16' for i in range(14, 22)}\n}\n\ntrain_gz = \"/kaggle/input/competitions/avazu-ctr-prediction/train.gz\"\nparquet_out = \"/kaggle/working/train.parquet\"\n\nif os.path.exists(parquet_out):\n    os.remove(parquet_out)\n\nprint(\"正在进行格式转换(GZ -> Parquet)\")\nwriter = None\n\nwith pd.read_csv(train_gz, compression='gzip', chunksize=2000000, dtype=dtypes) as reader:\n    for chunk in tqdm(reader):\n        chunk['hour_of_day'] = (chunk['hour'] % 100).astype('uint8')\n        chunk['day_of_week'] = (((chunk['hour'] // 100) % 100) % 7).astype('uint8')\n        \n        table = pa.Table.from_pandas(chunk, preserve_index=False)\n        \n        if writer is None:\n            writer = pq.ParquetWriter(parquet_out, table.schema, compression='snappy')\n        \n        writer.write_table(table)\n        \nif writer is not None:\n    writer.close()\nprint(\"Parquet转换完成！\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-06-11T03:29:46.949021Z","iopub.execute_input":"2026-06-11T03:29:46.949470Z","iopub.status.idle":"2026-06-11T03:32:54.902017Z","shell.execute_reply.started":"2026-06-11T03:29:46.949427Z","shell.execute_reply":"2026-06-11T03:32:54.901213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pyarrow.parquet as pq\nfrom collections import Counter\nimport pickle\nimport gc\nfrom tqdm import tqdm\n\nparquet_in = \"/kaggle/working/train.parquet\"\npfile = pq.ParquetFile(parquet_in)\nnum_groups = pfile.num_row_groups\n\ncount_cols = ['device_ip', 'device_id', 'site_id', 'app_id']\nglobal_counters = {col: Counter() for col in count_cols}\n\nprint(\"开始扫描数据，构建字典...\")\nfor i in tqdm(range(num_groups)):\n    table_chunk = pfile.read_row_group(i, columns=count_cols)\n    \n    for col in count_cols:\n        arr = table_chunk.column(col).to_numpy()\n        global_counters[col].update(arr)\n        \n    del table_chunk\n    gc.collect()\n\nwith open(\"global_counters.pkl\", \"wb\") as f:\n    pickle.dump(global_counters, f)\n\nprint(\"全局频次字典提取完毕！\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-11T03:32:54.903522Z","iopub.execute_input":"2026-06-11T03:32:54.903849Z","iopub.status.idle":"2026-06-11T03:33:49.199941Z","shell.execute_reply.started":"2026-06-11T03:32:54.903823Z","shell.execute_reply":"2026-06-11T03:33:49.199228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pyarrow.parquet as pq\nimport pandas as pd\nimport os\nfrom tqdm import tqdm\n\nparquet_in = \"/kaggle/working/train.parquet\"\ncsv_out = \"/kaggle/working/xgb_train_stream.csv\"\n\nif os.path.exists(csv_out):\n    os.remove(csv_out)\n\npfile = pq.ParquetFile(parquet_in)\nnum_groups = pfile.num_row_groups\n\nprint(\"正在生成XGBoost外部内存流文件\")\nfor i in tqdm(range(num_groups)):\n    df_chunk = pfile.read_row_group(i).to_pandas()\n    \n    features_to_keep = [c for c in df_chunk.columns if c not in ['id', 'hour']]\n    df_chunk = df_chunk[features_to_keep]\n    \n    if i == 0:\n        df_chunk.to_csv(csv_out, index=False, header=True)\n    else:\n        df_chunk.to_csv(csv_out, index=False, header=False, mode='a')\n\nprint(\"流文件生成完毕！开始XGBoost训练。\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-11T03:33:49.200835Z","iopub.execute_input":"2026-06-11T03:33:49.201192Z","iopub.status.idle":"2026-06-11T03:39:44.947102Z","shell.execute_reply.started":"2026-06-11T03:33:49.201168Z","shell.execute_reply":"2026-06-11T03:39:44.946133Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pyarrow as pa\nimport pyarrow.parquet as pq\nimport pandas as pd\nimport numpy as np\nimport xgboost as xgb\nimport pickle\nimport gc\n\nwith open(\"global_counters.pkl\", \"rb\") as f:\n    global_counters = pickle.load(f)\n\nparquet_in = \"/kaggle/working/train.parquet\"\npfile = pq.ParquetFile(parquet_in)\nnum_groups = pfile.num_row_groups\nprint(f\"[方案一：高阶交叉哈希模式] 共有 {num_groups} 个 Row Groups。\")\n\nstep = 4 \ntrees_per_chunk = 35\n\nparams = {\n    'objective': 'binary:logistic',\n    'eval_metric': 'logloss',\n    'learning_rate': 0.08,       \n    'max_depth': 6,\n    'min_child_weight': 40,      \n    'subsample': 0.8,\n    'colsample_bytree': 0.7,\n    'reg_alpha': 2.0,            \n    'reg_lambda': 5.0,\n    'tree_method': 'hist',\n    'device': 'cuda',\n}\n\nhash_cols = ['site_id', 'site_domain', 'site_category', 'app_id', 'app_domain', 'app_category', 'device_model']\nHASH_SPACE = 25000 \n\ncross_pairs = [\n    ('site_id', 'app_id'),\n    ('device_model', 'banner_pos')\n]\n\nfeatures = [\n    'C1', 'banner_pos', 'device_type', 'device_conn_type', \n    'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21',\n    'hour_of_day', 'day_of_week',\n    'device_ip_global_log_count', 'device_id_global_log_count', \n    'site_id_global_log_count', 'app_id_global_log_count'\n] + [f'{c}_hash' for c in hash_cols] + [f'{p[0]}_{p[1]}_cross_hash' for p in cross_pairs]\n\nfeature_types = []\nfor col in features:\n    if col.endswith('_hash') or 'cross' in col:\n        feature_types.append('c') \n    else:\n        feature_types.append('q') \n\nbst = None\n\nprint(\"[ENGINE START] 正在流式生成高阶交叉哈希并训练...\")\n\nfor start_idx in range(0, num_groups, step):\n    end_idx = min(start_idx + step, num_groups)\n    print(f\"正在加载 Row Group {start_idx} 到 {end_idx-1}...\")\n    \n    table_chunk = pa.concat_tables([pfile.read_row_group(j) for j in range(start_idx, end_idx)])\n    df_chunk = table_chunk.to_pandas()\n    del table_chunk\n    gc.collect()\n    \n    df_chunk['hour_of_day'] = (df_chunk['hour'] % 100).astype('uint8')\n    df_chunk['day_of_week'] = (((df_chunk['hour'] // 100) % 100) % 7).astype('uint8')\n    \n    for col in ['device_ip', 'device_id', 'site_id', 'app_id']:\n        raw_count = df_chunk[col].map(global_counters[col]).fillna(1).astype('float32')\n        df_chunk[f'{col}_global_log_count'] = np.log1p(raw_count)\n        \n    for col in hash_cols:\n        hashed_vals = pd.util.hash_array(df_chunk[col].astype(str).values)\n        df_chunk[f'{col}_hash'] = (hashed_vals % HASH_SPACE).astype('int32')\n        \n    for col1, col2 in cross_pairs:\n        cross_str = df_chunk[col1].astype(str) + \"_\" + df_chunk[col2].astype(str)\n        hashed_cross = pd.util.hash_array(cross_str.values)\n        df_chunk[f'{col1}_{col2}_cross_hash'] = (hashed_cross % HASH_SPACE).astype('int32')\n        \n    dtrain_chunk = xgb.DMatrix(\n        df_chunk[features], \n        label=df_chunk['click'], \n        feature_types=feature_types,\n        enable_categorical=True \n    )\n    \n    bst = xgb.train(params, dtrain_chunk, num_boost_round=trees_per_chunk, xgb_model=bst)\n    \n    del df_chunk, dtrain_chunk\n    gc.collect()\n\nbst.save_model(\"xgb_avazu_cross_final.json\")\nprint(\"\\n[SUCCESS] 拥有高阶交叉匹配视野的全量大模型已成功诞生！\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-11T03:39:44.949030Z","iopub.execute_input":"2026-06-11T03:39:44.949348Z","iopub.status.idle":"2026-06-11T03:42:15.678703Z","shell.execute_reply.started":"2026-06-11T03:39:44.949317Z","shell.execute_reply":"2026-06-11T03:42:15.677454Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport xgboost as xgb\nimport pickle\nimport gc\nimport os\n\ntest_gz = \"/kaggle/input/competitions/avazu-ctr-prediction/test.gz\"\nsubmission_out = \"/kaggle/working/submission.csv\"\n\nbst = xgb.Booster()\nbst.load_model(\"xgb_avazu_cross_final.json\")\n\nwith open(\"global_counters.pkl\", \"rb\") as f:\n    global_counters = pickle.load(f)\n\nhash_cols = ['site_id', 'site_domain', 'site_category', 'app_id', 'app_domain', 'app_category', 'device_model']\nHASH_SPACE = 25000 \n\ncross_pairs = [\n    ('site_id', 'app_id'),\n    ('device_model', 'banner_pos')\n]\n\nfeatures = [\n    'C1', 'banner_pos', 'device_type', 'device_conn_type', \n    'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21',\n    'hour_of_day', 'day_of_week',\n    'device_ip_global_log_count', 'device_id_global_log_count', \n    'site_id_global_log_count', 'app_id_global_log_count'\n] + [f'{c}_hash' for c in hash_cols] + [f'{p[0]}_{p[1]}_cross_hash' for p in cross_pairs]\n\nfeature_types = []\nfor col in features:\n    if col.endswith('_hash') or 'cross' in col:\n        feature_types.append('c')\n    else:\n        feature_types.append('q')\n\ntest_dtypes = {\n    'id': 'string', 'hour': 'uint32', 'C1': 'uint16', 'banner_pos': 'uint8', \n    'site_id': 'string', 'site_domain': 'string', 'site_category': 'string', \n    'app_id': 'string', 'app_domain': 'string', 'app_category': 'string', \n    'device_id': 'string', 'device_ip': 'string', 'device_model': 'string', \n    'device_type': 'uint8', 'device_conn_type': 'uint8',\n    **{f'C{i}': 'uint16' for i in range(14, 22)}\n}\n\nif os.path.exists(submission_out):\n    os.remove(submission_out)\n\nprint(\"开始对测试集提取高阶交叉哈希并预测...\")\nchunk_size = 2000000\nis_first_chunk = True\n\nwith pd.read_csv(test_gz, compression='gzip', chunksize=chunk_size, dtype=test_dtypes) as reader:\n    for chunk_idx, chunk in enumerate(reader):\n        print(f\"正在预测测试集第 {chunk_idx + 1} 块数据...\")\n        \n        chunk['hour_of_day'] = (chunk['hour'] % 100).astype('uint8')\n        chunk['day_of_week'] = (((chunk['hour'] // 100) % 100) % 7).astype('uint8')\n        \n        for col in ['device_ip', 'device_id', 'site_id', 'app_id']:\n            raw_count = chunk[col].map(global_counters[col]).fillna(1).astype('float32')\n            chunk[f'{col}_global_log_count'] = np.log1p(raw_count)\n            \n        for col in hash_cols:\n            hashed_vals = pd.util.hash_array(chunk[col].astype(str).values)\n            chunk[f'{col}_hash'] = (hashed_vals % HASH_SPACE).astype('int32')\n            \n        for col1, col2 in cross_pairs:\n            cross_str = chunk[col1].astype(str) + \"_\" + chunk[col2].astype(str)\n            hashed_cross = pd.util.hash_array(cross_str.values)\n            chunk[f'{col1}_{col2}_cross_hash'] = (hashed_cross % HASH_SPACE).astype('int32')\n        \n        dtest = xgb.DMatrix(chunk[features], feature_types=feature_types, enable_categorical=True)\n        preds = bst.predict(dtest)\n        \n        part_df = pd.DataFrame({'id': chunk['id'], 'click': preds})\n        \n        if is_first_chunk:\n            part_df.to_csv(submission_out, index=False, header=True)\n            is_first_chunk = False\n        else:\n            part_df.to_csv(submission_out, index=False, header=False, mode='a')\n            \n        del chunk, dtest, part_df\n        gc.collect()\n\nprint(\"[SUCCESS] 带有强力交叉特征的最终预测文件已成功交付！\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-11T03:42:15.679524Z","iopub.status.idle":"2026-06-11T03:42:15.679807Z","shell.execute_reply.started":"2026-06-11T03:42:15.679688Z","shell.execute_reply":"2026-06-11T03:42:15.679703Z"}},"outputs":[],"execution_count":null}]}