{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":31011,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-04-21T04:30:12.039292Z","iopub.execute_input":"2025-04-21T04:30:12.039522Z","iopub.status.idle":"2025-04-21T04:30:13.774306Z","shell.execute_reply.started":"2025-04-21T04:30:12.039497Z","shell.execute_reply":"2025-04-21T04:30:13.773667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport polars as pl\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import roc_auc_score, log_loss\nfrom torch.utils.data import Dataset, DataLoader\nimport gc","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T04:30:31.428644Z","iopub.execute_input":"2025-04-21T04:30:31.428891Z","iopub.status.idle":"2025-04-21T04:30:31.433338Z","shell.execute_reply.started":"2025-04-21T04:30:31.428873Z","shell.execute_reply":"2025-04-21T04:30:31.432609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_schema = {\n    'id': pl.UInt64,\n    'click': pl.Int8,\n    'hour': pl.String,\n    'C1': pl.Int16,\n    'banner_pos': pl.Int8,\n    'site_id': pl.Categorical,\n    'site_domain': pl.Categorical,\n    'site_category': pl.Categorical,\n    'app_id': pl.Categorical,\n    'app_domain': pl.Categorical,\n    'app_category': pl.Categorical,\n    'device_id': pl.Categorical,\n    'device_ip': pl.Categorical,\n    'device_model': pl.Categorical,\n    'device_type': pl.Int8,\n    'device_conn_type': pl.Int8,\n    'C14': pl.Int16,\n    'C15': pl.Int16,\n    'C16': pl.Int16,\n    'C17': pl.Int16,\n    'C18': pl.Int16,\n    'C19': pl.Int16,\n    'C20': pl.Int32,\n    'C21': pl.Int16\n}\n\ntest_schema = {\n    'id': pl.UInt64,\n    'hour': pl.String,\n    'C1': pl.Int16,\n    'banner_pos': pl.Int8,\n    'site_id': pl.Categorical,\n    'site_domain': pl.Categorical,\n    'site_category': pl.Categorical,\n    'app_id': pl.Categorical,\n    'app_domain': pl.Categorical,\n    'app_category': pl.Categorical,\n    'device_id': pl.Categorical,\n    'device_ip': pl.Categorical,\n    'device_model': pl.Categorical,\n    'device_type': pl.Int8,\n    'device_conn_type': pl.Int8,\n    'C14': pl.Int16,\n    'C15': pl.Int16,\n    'C16': pl.Int16,\n    'C17': pl.Int16,\n    'C18': pl.Int16,\n    'C19': pl.Int16,\n    'C20': pl.Int32,\n    'C21': pl.Int16\n}\n\ntrain_df = pl.read_csv('/kaggle/input/avazu-ctr-prediction/train.gz', schema=train_schema)\ntest_df = pl.read_csv('/kaggle/input/avazu-ctr-prediction/test.gz', schema=test_schema)\n\ntrain_df_sampled = train_df.sample(n=int(len(train_df) * 0.001))\ntest_df_sampled = test_df.sample(n=int(len(test_df) * 0.001))\n\ndef extract_hour_features(df):\n    df = df.with_columns([\n        (pl.col('hour').str.slice(6, 2).cast(pl.Int8)).alias('hour_of_day'),\n        (pl.col('hour').str.slice(4, 2).cast(pl.Int8)).alias('day_of_month')\n    ])\n    return df.drop('hour')\n\ntrain_df_sampled = extract_hour_features(train_df_sampled)\ntest_df_sampled = extract_hour_features(test_df_sampled)\n\nimportant_columns = [\n    'click', 'C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n    'app_id', 'app_domain', 'app_category', 'device_id', 'device_model',\n    'device_type', 'device_conn_type', 'C14', 'C15', 'C16', 'hour_of_day', 'day_of_month'\n]\n\ntrain_df_trimmed = train_df_sampled.select(important_columns)\ntest_df_trimmed = test_df_sampled.select([col for col in important_columns if col != 'click'])\n\ncategorical_columns = ['site_id', 'site_domain', 'site_category', 'app_id', 'app_domain',\n                       'app_category', 'device_id', 'device_model']\n\ndef label_encode_columns(df, columns):\n    for col in columns:\n        le = LabelEncoder()\n        df = df.with_columns([(pl.Series(col, le.fit_transform(df[col].to_numpy()))).alias(col)])\n    return df\n\ntrain_df_encoded = label_encode_columns(train_df_trimmed, categorical_columns)\ntest_df_encoded = label_encode_columns(test_df_trimmed, categorical_columns)\n\nX = train_df_encoded.drop(['click']).to_pandas().values\ny = train_df_encoded['click'].to_pandas().values\n\nX_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)\nX_test = test_df_encoded.to_pandas().values","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:20:40.041779Z","iopub.execute_input":"2025-04-21T05:20:40.042055Z","iopub.status.idle":"2025-04-21T05:21:30.107903Z","shell.execute_reply.started":"2025-04-21T05:20:40.042033Z","shell.execute_reply":"2025-04-21T05:21:30.107344Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"len(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:21:30.109051Z","iopub.execute_input":"2025-04-21T05:21:30.109249Z","iopub.status.idle":"2025-04-21T05:21:30.114036Z","shell.execute_reply.started":"2025-04-21T05:21:30.109234Z","shell.execute_reply":"2025-04-21T05:21:30.113339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df_encoded.corr()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:21:30.114764Z","iopub.execute_input":"2025-04-21T05:21:30.114991Z","iopub.status.idle":"2025-04-21T05:21:30.141117Z","shell.execute_reply.started":"2025-04-21T05:21:30.114968Z","shell.execute_reply":"2025-04-21T05:21:30.140415Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\ndef train_epoch(model, loader):\n    model.train()\n    total_loss = 0\n    criterion = nn.BCELoss()\n    optimizer = optim.Adam(model.parameters(), lr=0.001)\n\n    for cat_inputs, user_inputs, y_batch in loader:\n        cat_inputs = cat_inputs.to(device)\n        user_inputs = user_inputs.to(device)\n        y_batch = y_batch.to(device)\n\n        optimizer.zero_grad()\n        outputs = model(cat_inputs, user_inputs).squeeze()\n\n        loss = criterion(outputs, y_batch)\n        loss.backward()\n        optimizer.step()\n\n        total_loss += loss.item()\n\n    return total_loss / len(loader)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:21:30.142625Z","iopub.execute_input":"2025-04-21T05:21:30.142824Z","iopub.status.idle":"2025-04-21T05:21:30.147569Z","shell.execute_reply.started":"2025-04-21T05:21:30.142808Z","shell.execute_reply":"2025-04-21T05:21:30.146853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate(model, loader):\n    model.eval()\n    preds =[]\n    targets = []\n\n    with torch.no_grad():\n        for cat_inputs, user_inputs, y_batch in loader:\n            cat_inputs = cat_inputs.to(device)\n            user_inputs = user_inputs.to(device)\n            y_batch = y_batch.to(device)\n\n            outputs = model(cat_inputs, user_inputs).squeeze()\n            preds.extend(outputs.cpu().numpy())\n            targets.extend(y_batch.cpu().numpy())\n\n    auc = roc_auc_score(targets, preds)\n    loss = log_loss(targets, preds)\n\n    return auc, loss","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:21:30.148262Z","iopub.execute_input":"2025-04-21T05:21:30.148503Z","iopub.status.idle":"2025-04-21T05:21:30.160678Z","shell.execute_reply.started":"2025-04-21T05:21:30.148479Z","shell.execute_reply":"2025-04-21T05:21:30.160095Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nclass CTRDataset(Dataset):\n    def __init__(self, X, y=None, num_cat_features=8):\n        \"\"\"\n        Args:\n        - X: np.array of features (categorical + continuous)\n        - y: labels (optional)\n        - num_cat_features: number of categorical features at the start of X\n        \"\"\"\n        self.X = X\n        self.y = y\n        self.num_cat_features = num_cat_features\n\n    def __len__(self):\n        return len(self.X)\n\n    def __getitem__(self, idx):\n        row = self.X[idx]\n\n        # Ensure categorical features are valid long tensors\n        cat_feats = torch.tensor(row[:self.num_cat_features], dtype=torch.long)\n        cat_feats = torch.clamp(cat_feats, min=0, max=999)  \n\n        user_feats = torch.tensor(row[self.num_cat_features:], dtype=torch.float)\n\n        if self.y is not None:\n            label = torch.tensor(self.y[idx], dtype=torch.float)\n            return cat_feats, user_feats, label\n        else:\n            return cat_feats, user_feats\n\nclass MultiHeadAttention(nn.Module):\n    def __init__(self, num_heads, dk, dv):\n        super(MultiHeadAttention, self).__init__()\n        assert dk % num_heads == 0, \"dk must be divisible by num_heads\"\n        \n        self.dk = dk\n        self.dv = dv\n        self.num_heads = num_heads\n        self.head_dim = self.dk // self.num_heads\n        \n        self.query_weight = nn.Linear(dk, dk)\n        self.key_weight = nn.Linear(dk, dk)\n        self.value_weight = nn.Linear(dv, dv)\n        self.output_layer = nn.Linear(dk, dk)\n        self.dropout = nn.Dropout(p=0.1)\n\n    def scaled_dot_product_attention(self, query, key, value):\n        prod = torch.matmul(query, key.transpose(-2, -1)) / (self.head_dim ** 0.5)\n        attention_score = F.softmax(prod, dim=-1)\n        attn = self.dropout(attention_score)\n        final = torch.matmul(attn, value)\n        return final\n\n    def split_heads(self, x, batch_size):\n        x = x.view(batch_size, -1, self.num_heads, self.head_dim)\n        return x.permute(0, 2, 1, 3)\n\n    def forward(self, query, key, value):\n        batch_size = query.size(0)\n        query = self.split_heads(self.query_weight(query), batch_size)\n        key = self.split_heads(self.key_weight(key), batch_size)\n        value = self.split_heads(self.value_weight(value), batch_size)\n        attention = self.scaled_dot_product_attention(query, key, value)\n        attention = attention.permute(0, 2, 1, 3).contiguous().view(batch_size, -1, self.dk)\n        return self.output_layer(attention)\n\nclass InteractionLayer(nn.Module):\n    def __init__(self, dk, dv, num_heads):\n        super(InteractionLayer, self).__init__()\n        self.multiheadattn = MultiHeadAttention(num_heads, dk, dv)\n        self.Residual = nn.Linear(dk, dk)\n        self.relu = nn.ReLU()\n\n    def forward(self, q, k, v):\n        emhat = self.multiheadattn(q, k, v)\n        residual = self.Residual(emhat)\n        return self.relu(residual)\n\nclass InterestAcquisitionLayer(nn.Module):\n    def __init__(self, input_dim, hidden_dims, output_dim):\n        super(InterestAcquisitionLayer, self).__init__()\n        layers = []\n        last_dim = input_dim\n        for dim in hidden_dims:\n            layers.append(nn.Linear(last_dim, dim))\n            layers.append(nn.ReLU())\n            last_dim = dim\n        layers.append(nn.Linear(last_dim, output_dim))\n        layers.append(nn.ReLU())\n        self.mlp = nn.Sequential(*layers)\n\n    def forward(self, user_features):\n        return self.mlp(user_features)\n\nclass IARM(nn.Module):\n    def __init__(self, num_fields, embedding_dim, user_input_dim, user_interest_hidden_dims, num_heads, interaction_layers=2):\n        super(IARM, self).__init__()\n\n        # Define embeddings AFTER calling super()\n        self.embeddings = nn.ModuleList([\n            nn.Embedding(1000, embedding_dim) for _ in range(num_fields)\n        ])\n\n        self.user_interest = InterestAcquisitionLayer(user_input_dim, user_interest_hidden_dims, embedding_dim)\n\n        self.interaction_layers = nn.ModuleList([\n            InteractionLayer(embedding_dim, embedding_dim, num_heads)\n            for _ in range(interaction_layers)\n        ])\n\n        self.fc = nn.Linear(embedding_dim * (num_fields + 1), 1)\n\n    def forward(self, cat_inputs, user_inputs):\n        batch_size = user_inputs.size(0)\n\n        print(\"Categorical inputs (sample):\", cat_inputs[:5])\n        print(\"Max category index:\", cat_inputs.max().item())\n        print(\"Min category index:\", cat_inputs.min().item())\n\n        feature_embeds = [embed(cat_inputs[:, i]) for i, embed in enumerate(self.embeddings)]\n        feature_embeds = torch.stack(feature_embeds, dim=1)  \n\n        user_interest_embed = self.user_interest(user_inputs)\n\n        combined_embeds = torch.cat([feature_embeds, user_interest_embed.unsqueeze(1)], dim=1)\n\n        for layer in self.interaction_layers:\n            combined_embeds = layer(combined_embeds, combined_embeds, combined_embeds)\n\n        flattened = combined_embeds.view(batch_size, -1)\n\n        logits = self.fc(flattened)\n        return torch.sigmoid(logits)\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\nnum_fields = 8\nuser_input_dim = 9\nembedding_dim = 16\nnum_heads = 4\ninteraction_layers = 2\n\nmodel = IARM(\n    num_fields=num_fields,\n    embedding_dim=embedding_dim,\n    user_input_dim=user_input_dim,\n    user_interest_hidden_dims=[32, 16],\n    num_heads=num_heads,\n    interaction_layers=interaction_layers\n).to(device)\n\nprint(\"Model initialized successfully!\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:21:30.161372Z","iopub.execute_input":"2025-04-21T05:21:30.161565Z","iopub.status.idle":"2025-04-21T05:21:30.188743Z","shell.execute_reply.started":"2025-04-21T05:21:30.16155Z","shell.execute_reply":"2025-04-21T05:21:30.188106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"num_cat_features = 8\ntrain_dataset = CTRDataset(X_train, y_train, num_cat_features=num_cat_features)\nval_dataset = CTRDataset(X_val, y_val, num_cat_features=num_cat_features)\n\ntrain_loader = DataLoader(train_dataset, batch_size=512, shuffle=True)\nval_loader = DataLoader(val_dataset, batch_size=512, shuffle=False)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:21:30.189463Z","iopub.execute_input":"2025-04-21T05:21:30.190129Z","iopub.status.idle":"2025-04-21T05:21:30.195473Z","shell.execute_reply.started":"2025-04-21T05:21:30.190055Z","shell.execute_reply":"2025-04-21T05:21:30.194896Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.shape","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T05:21:30.196245Z","iopub.execute_input":"2025-04-21T05:21:30.196744Z","iopub.status.idle":"2025-04-21T05:21:30.209264Z","shell.execute_reply.started":"2025-04-21T05:21:30.196721Z","shell.execute_reply":"2025-04-21T05:21:30.208639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"epochs = 10\n# for epoch in range(epochs):\n#     train_loss = train_epoch(model, train_loader)\n#     val_auc, val_loss = evaluate(model, val_loader)\n\ntrain_losses = []\n\nfor epoch in range(epochs):\n    train_loss = train_epoch(model, train_loader)\n    val_auc, val_loss = evaluate(model, val_loader)\n    \n    train_losses.append(train_loss)\n    print(f\"Epoch {epoch}: Train Loss = {train_loss:.4f}, Val AUC = {val_auc:.4f}, Val Loss = {val_loss:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T06:02:16.821138Z","iopub.execute_input":"2025-04-21T06:02:16.821701Z","iopub.status.idle":"2025-04-21T06:02:31.857322Z","shell.execute_reply.started":"2025-04-21T06:02:16.821675Z","shell.execute_reply":"2025-04-21T06:02:31.856544Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nepochs_range = list(range(epochs))  # [0, 1, 2, ..., epochs-1]\n\nplt.figure(figsize=(6, 4))\nplt.plot(epochs_range, train_losses, marker='o', label='Avazu Training Loss', color='orange')\nplt.title('Avazu Training Loss Over Epochs')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.grid(True)\nplt.tight_layout()\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-04-21T06:05:00.11535Z","iopub.execute_input":"2025-04-21T06:05:00.11563Z","iopub.status.idle":"2025-04-21T06:05:00.266766Z","shell.execute_reply.started":"2025-04-21T06:05:00.115609Z","shell.execute_reply":"2025-04-21T06:05:00.26606Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}