{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":105343,"databundleVersionId":12700041,"sourceType":"competition"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Multimodal PyTorch Baseline for “Applications of Deep Learning (WUSTL, Summer 2025)”**\n\n1. **Data loading & sampling**  \n   - Reads `train.csv` and `test.csv` from your Kaggle input folder.  \n   - Applies a quick “smoke-test” sampling via `SAMPLE_PERCENT` (e.g. `0.1` for 10% of rows) so you can iterate rapidly before scaling up to the full dataset.  \n\n2. **Tabular preprocessing**  \n   - Numeric columns are standardized with `StandardScaler`.  \n   - Categorical columns are label-encoded on the combined train+test set to handle unseen categories.  \n\n3. **Image handling**  \n   - Loads each JPEG by casting the float `id` → `int` → `\"{id}.jpg\"` and applying standard ImageNet transforms (resize, flip, normalize).  \n\n4. **Model architecture**  \n   - **CNN branch:** Pretrained ResNet-18 (ImageNet weights), dropping the final FC and using it as a feature extractor.  \n   - **Tabular branch:** A small MLP (128→64→ReLU).  \n   - **Head:** Concatenates both feature streams, then runs through 256→1 for regression (preservation score).  \n\n5. **Training loop**  \n   - Uses `MSELoss` + `Adam(lr=1e-4)`.  \n   - Iterates for `num_epochs` (default 5) over the sampled subset.  \n\n6. **Submission cell**  \n   - Switches the dataset to inference mode, runs the model on `df_test`, and writes out a `submission.csv`.  \n\n---\n\n### Performance tips\n\n- **Sample size:** Change `SAMPLE_PERCENT` to control how much of your training data you use during prototyping (e.g. `0.01` or `1.0` for full data).  \n- **Epochs:** Increase `num_epochs` once your pipeline is stable to fully converge on the real dataset.  \n- **Early stopping:** Wrap your training loop with a simple early-stopping criterion (e.g. monitor validation loss and halt when it hasn’t improved for N epochs) to avoid overfitting and save GPU time.  \n","metadata":{}},{"cell_type":"code","source":"# Imports and Configuration\n\nimport os                                                             # OS interaction\nimport math\nimport numpy as np                                                    # arrays, functions\nimport pandas as pd                                                   # dataframes, functions\nfrom PIL import Image                                                 # processing images\n\nfrom sklearn.model_selection import train_test_split, KFold           # Splitting data for training, testing, k-fold cross validation\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler        # Encoding and scaling data for normalization\nfrom sklearn.ensemble import RandomForestClassifier                   # Imputation model for categorical data -- Needed afer XGB??\nfrom sklearn.ensemble import StackingRegressor\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import mean_squared_error\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\n\nimport torch\nimport torch.nn as nn                                                 # neural network via pytorch\nfrom torch.utils.data import Dataset, DataLoader, Subset              # creating/loading datasets for nn\nimport torchvision.transforms as transforms                           # transform for image augmentation\nimport torchvision.models as models                                   # use of model functions\n\n# Global settings\nnp.random.seed(42)                                                    # Set random seeds for both torch and numpy for reproducability\ntorch.manual_seed(42)\n\nSAMPLE_PERCENT = 0.1                                                  \nNUM_EPOCHS = 100\nBATCH_SIZE = 32\n\n# Set paths for data\ndata_dir = os.path.dirname('../input/applications-of-deep-learning-wustl-summer-2025/train.csv')\ntrain_csv = os.path.join(data_dir, 'train.csv')\ntest_csv = os.path.join(data_dir, 'test.csv')\n\n# Load data to csvs\ndf_train_full = pd.read_csv(train_csv)\ndf_test = pd.read_csv(test_csv)\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nprint(f\"Using device: {device}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:08.771386Z","iopub.execute_input":"2025-07-19T23:57:08.771651Z","iopub.status.idle":"2025-07-19T23:57:18.250226Z","shell.execute_reply.started":"2025-07-19T23:57:08.771628Z","shell.execute_reply":"2025-07-19T23:57:18.249414Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def encode_categoricals(df_train, df_test, cat_cols):\n    encoders = {}\n    for col in cat_cols:\n        le = LabelEncoder()\n        df_train[col] = df_train[col].astype(str)\n        le.fit(df_train[col])\n        df_train[col] = le.transform(df_train[col])\n\n        if col in df_test.columns:\n            df_test[col] = df_test[col].astype(str).apply(lambda x: x if x in le.classes_ else 'unknown')\n            le.classes_ = np.append(le.classes_, 'unknown')\n            df_test[col] = le.transform(df_test[col])\n\n        encoders[col] = le\n    return encoders","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:18.251385Z","iopub.execute_input":"2025-07-19T23:57:18.251610Z","iopub.status.idle":"2025-07-19T23:57:18.256905Z","shell.execute_reply.started":"2025-07-19T23:57:18.251591Z","shell.execute_reply":"2025-07-19T23:57:18.256168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Imputation Function for categorical, numeric, feature columns\n\ndef impute_missing_values(df, categorical_cols, numeric_cols, feature_cols):\n    \"\"\" For each column with nulls, train a model to predict null values based on other features in the dataset.\n        Using Random Forest Classifier for categorical values and median for numeric\"\"\"\n    \n    df = df.copy()\n    df_encoded = pd.get_dummies(df[feature_cols], drop_first=True, dummy_na=True)    # Use one hot encoding on feature columns\n    print(\"Starting imputation...\")                                                  # log process start\n    \n    # Categorical - Classification\n    for col in categorical_cols:\n        if df[col].isnull().sum() == 0:                                              # skip columns w/o nulls\n            continue\n        mask = df[col].notnull()                                                     # create mask filter for non-nulls\n        X_train = df_encoded.loc[mask].dropna()                                      # use encoded cols (X) with non-null target col to train\n        y_train = df.loc[mask, col][X_train.index]                                   # select non-null target (y) for classification\n        model = RandomForestClassifier(n_estimators=100, random_state=42)            # classification model for categorical values\n        model.fit(X_train, y_train)\n\n        X_pred = df_encoded.loc[df[col].isnull()].dropna()                           # make prediction on x values\n        df.loc[X_pred.index, col] = model.predict(X_pred)                            # update dataframe with results\n\n    # Numeric - Median\n    for col in numeric_cols:\n        df[col] = df[col].fillna(df[col].median())\n\n    print(\"Imputation completed.\")                                                   # log process stop\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:19.239006Z","iopub.execute_input":"2025-07-19T23:57:19.239254Z","iopub.status.idle":"2025-07-19T23:57:19.245519Z","shell.execute_reply.started":"2025-07-19T23:57:19.239236Z","shell.execute_reply":"2025-07-19T23:57:19.244756Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Preprocessing\n\ndef preprocess_tabular(df_train, df_test, target_col, id_col):\n    # ignores target and id columns, assigns the rest to tabular_cols variable\n    ignore_cols = [id_col, target_col]\n    tabular_cols = [c for c in df_train.columns if c not in ignore_cols]\n\n    # Assigns numeric columns and categorical columns to respective variables per datatype for training data\n    numeric_cols = df_train[tabular_cols].select_dtypes(include='number').columns.tolist()\n    cat_cols = df_train[tabular_cols].select_dtypes(include=['object', 'category']).columns.tolist()\n\n    # Refines numeric/categorical cols to be those in testing data\n    numeric_cols = [c for c in numeric_cols if c in df_test.columns]\n    cat_cols = [c for c in cat_cols if c in df_test.columns]\n\n    # Scales data in numeric columns to standardize # Not necessary to scale for XGBoost or CatBoost but retained for models (NN/linear) that might benefit\n    scaler = StandardScaler()\n    if numeric_cols:\n        df_train[numeric_cols] = scaler.fit_transform(df_train[numeric_cols])\n        df_test[numeric_cols] = scaler.transform(df_test[numeric_cols])\n\n    # Call encoding function\n    encoders = encode_categoricals(df_train, df_test, cat_cols)\n\n    # Assigns full list of updated features to full list of features\n    tabular_feats = numeric_cols + cat_cols\n    return tabular_feats, encoders, len(tabular_feats)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:21.894849Z","iopub.execute_input":"2025-07-19T23:57:21.895612Z","iopub.status.idle":"2025-07-19T23:57:21.901895Z","shell.execute_reply.started":"2025-07-19T23:57:21.895586Z","shell.execute_reply":"2025-07-19T23:57:21.901063Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Dataset Class\n\nclass MultiModalDataset(Dataset):\n    def __init__(self, df, data_dir, id_col, tabular_feats, target_col=None, transform=None, is_train=True):\n        self.df = df.reset_index(drop=True)\n        self.data_dir = data_dir\n        self.id_col = id_col\n        self.tabular_feats = tabular_feats\n        self.target_col = target_col\n        self.transform = transform\n        self.is_train = is_train\n\n        self.tab_data = torch.tensor(df[tabular_feats].fillna(0).values, dtype=torch.float32)                  # Extract features, fillna with 0, \n        if is_train:\n            self.targets = torch.tensor(df[target_col].values, dtype=torch.float32)                            # Will this always be true since we're setting default?\n\n    def __len__(self):\n        \"\"\" Mark the size of the dataset for later use (i.e., batching)\"\"\"\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        \"\"\"Iterate through dataset rows. \n            For imgs, build path to img, use predefined img to transform (if available).\n            For tabular, get data as a tensor (see self.tab_data).\n            Selective return results for training vs. testing.\"\"\"\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.data_dir, f\"{int(row[self.id_col])}.jpg\")\n        img = Image.open(img_path).convert('RGB')\n        if self.transform:\n            img = self.transform(img)\n\n        tab = self.tab_data[idx]\n        return (img, tab, self.targets[idx]) if self.is_train else (img, tab, row[self.id_col])\n\n# Get image only dataset\n\nclass ImageOnlyDataset(Dataset):\n    def __init__(self, df, data_dir, id_col, transform):\n        self.df = df.reset_index(drop=True)\n        self.data_dir = data_dir\n        self.id_col = id_col\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.df)\n\n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        img_path = os.path.join(self.data_dir, f\"{int(row[self.id_col])}.jpg\")\n        img = Image.open(img_path).convert('RGB')\n        img = self.transform(img)\n        return img, row[self.id_col]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:24.036887Z","iopub.execute_input":"2025-07-19T23:57:24.037162Z","iopub.status.idle":"2025-07-19T23:57:24.045761Z","shell.execute_reply.started":"2025-07-19T23:57:24.037142Z","shell.execute_reply":"2025-07-19T23:57:24.045085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Class for adding Gaussian Noise (more img transformation)\n\nclass AddGaussianNoise(object):\n    def __init__(self, mean=0., std=0.1):\n        self.mean = mean\n        self.std = std\n\n    def __call__(self, tensor):\n        return tensor + torch.randn_like(tensor) * self.std + self.mean\n\n    def __repr__(self):\n        return f\"{self.__class__.__name__}(mean={self.mean}, std={self.std})\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:26.361357Z","iopub.execute_input":"2025-07-19T23:57:26.361954Z","iopub.status.idle":"2025-07-19T23:57:26.366176Z","shell.execute_reply.started":"2025-07-19T23:57:26.361926Z","shell.execute_reply":"2025-07-19T23:57:26.365598Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Image Transformations\n\n    # \"\"\"Transformations on the image data to improve generalization,\n    #     simulate real-world variability to improve predictions \"\"\"\n\ntransform = transforms.Compose([\n    transforms.RandomResizedCrop(size=(224, 224), scale=(0.8, 1.0)),                  # combine RandomCrop + Resize (zoom/framing variability)\n    transforms.RandomRotation(degrees=20),                                            # for possibility of real-world change in orientation\n    transforms.RandomPerspective(distortion_scale=0.5, p=0.5),                        # add some perspective warp\n    transforms.RandomAdjustSharpness(sharpness_factor=2, p=0.3),                      # play with sharpness in image\n    transforms.RandomAutocontrast(p=0.3),                                             # play with contrast\n    transforms.GaussianBlur(kernel_size=(5, 9), sigma=(0.1, 5.0)),                    # random blur applied\n    transforms.RandomHorizontalFlip(p=0.5),                                           # common, covers generalization per horiz symmetry\n    transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.1, hue=0.05),   # for lighting/color imbalance\n    transforms.RandomAffine(degrees=0, translate=(0.05, 0.05)),                       # for real-world, slight scale/rotation changes\n    transforms.ToTensor(),                                                            # Must come after all PIL-based transforms except noise\n    AddGaussianNoise(mean=0.0, std=0.03),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],std=[0.229, 0.224, 0.225]),\n])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:27.583962Z","iopub.execute_input":"2025-07-19T23:57:27.584644Z","iopub.status.idle":"2025-07-19T23:57:27.590351Z","shell.execute_reply.started":"2025-07-19T23:57:27.584620Z","shell.execute_reply":"2025-07-19T23:57:27.589639Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Learning Models\n\nclass MultiModalModel(nn.Module):\n    \"\"\"Merges models and passes through final set of layers for prediction\"\"\"\n    def __init__(self, num_tab_feats, backbone='resnet18', dropout_prob=0.3):\n        super().__init__()\n        cnn = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)              # use resnet18 for image feature extraction\n        \n        for param in cnn.parameters():                                              # set parameters to customize resnet\n            param.requires_grad = False                                             # start with freezing layers then gradually unfreeze them\n\n        for param in cnn.layer4.parameters():\n            param.requires_grad = True                                              # calculate gradients for parameters so they're updated during backpropgagation\n        \n        feat_dim = cnn.fc.in_features                                               # process image data, needs done to use with self.head later\n        cnn.fc = nn.Identity()                                                      \n        self.cnn = cnn\n\n        self.tab_net = nn.Sequential(                                               # process tabular data with feed forward network\n            nn.Linear(num_tab_feats, 256),\n            nn.BatchNorm1d(256),\n            nn.ReLU(),\n            nn.Dropout(dropout_prob),\n            nn.Linear(256, 128),\n            nn.BatchNorm1d(128),\n            nn.ReLU(),\n            nn.Dropout(dropout_prob),\n        )\n\n        self.head = nn.Sequential(                                                 # concatenates img (feat_dim) and tab features (128) into a vector\n            nn.Linear(feat_dim + 128, 256),\n            nn.ReLU(),\n            nn.Dropout(dropout_prob),\n            nn.Linear(256, 1)\n        )\n\n    def forward(self, img, tab):\n        img_feat = self.cnn(img)                                                  # process image data\n        tab_feat = self.tab_net(tab)                                              # process tabular data\n        x = torch.cat([img_feat, tab_feat], dim=1)                                # combine image/tab data\n        return self.head(x)                                                       # return final prediction\n\nclass ImageFeatureExtractor(nn.Module):\n    def __init__(self, model):\n        super().__init__()\n        self.cnn = model.cnn  # Only keep image model\n    def forward(self, img):\n        return self.cnn(img)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:29.610952Z","iopub.execute_input":"2025-07-19T23:57:29.611238Z","iopub.status.idle":"2025-07-19T23:57:29.618588Z","shell.execute_reply.started":"2025-07-19T23:57:29.611215Z","shell.execute_reply":"2025-07-19T23:57:29.617860Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Image feature extraction\n\ndef extract_image_features(model, df, data_dir, transform, batch_size=32):\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    model = ImageFeatureExtractor(model).to(device)\n    model.eval()\n\n    dataset = ImageOnlyDataset(df, data_dir, id_col='id', transform=transform)\n    loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)\n\n    all_features, all_ids = [], []\n\n    with torch.no_grad():\n        for imgs, ids in loader:\n            imgs = imgs.to(device)\n            feats = model(imgs)\n            all_features.append(feats.cpu().numpy())\n            all_ids.extend(ids.tolist())\n\n    features_array = np.vstack(all_features)\n    return pd.DataFrame(features_array, index=all_ids).sort_index()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:31.470219Z","iopub.execute_input":"2025-07-19T23:57:31.470792Z","iopub.status.idle":"2025-07-19T23:57:31.475763Z","shell.execute_reply.started":"2025-07-19T23:57:31.470753Z","shell.execute_reply":"2025-07-19T23:57:31.475145Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training Model\n\ndef train_model(model, train_loader, val_loader, device, num_epochs):\n    \n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-4)  # updated after unfreezing to pick up new trainable parameters\n    criterion = nn.MSELoss()                                                                      # loss function\n    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(                                       # scheduler for image data\n        optimizer, mode='min', factor=0.5, patience=1, verbose=True\n    )\n    model.to(device)\n    \n    best_rmse = float('inf')                                                                  # early stopping setup\n    patience, patience_counter = 30, 0                                                         # early stopping params\n\n    for epoch in range(num_epochs):\n        \"\"\" For each epoch, train the model, gather running loss, predictions, targets\"\"\"\n        model.train()\n        running_loss, all_preds, all_targets = 0, [], []\n\n        for imgs, tabs, ys in train_loader:\n            # Move data to device\n            imgs, tabs, ys = imgs.to(device), tabs.to(device), ys.unsqueeze(1).to(device)    #for img, tab, target move to CPU/GPU, reshape target for model\n\n            # Ensure data is on GPU\n            assert imgs.device.type == 'cuda', \"Images not on GPU\"\n            assert tabs.device.type == 'cuda', \"Tabular data not on GPU\"\n            assert ys.device.type == 'cuda', \"Targets not on GPU\"            \n            \n            # Forward pass \n            preds = model(imgs, tabs)                                                        #pass img, tabular data through model for predictions\n            \n            # Ensure outputs are on GPU\n            assert preds.device.type == 'cuda', \"Model outputs not on GPU\"            \n            \n            loss = criterion(preds, ys)                                                      #calculate loss for predictions/targets\n\n            # Backpropagation\n            optimizer.zero_grad()\n            loss.backward()\n            optimizer.step()\n\n            running_loss += loss.item() * imgs.size(0)                                       #accumulate total loss per epoch by batch\n            all_preds.append(preds.detach().cpu().numpy())\n            all_targets.append(ys.detach().cpu().numpy())                                    #prep for calculating RMSE\n\n\n        \n        train_rmse = mean_squared_error(np.vstack(all_targets), np.vstack(all_preds), squared=False)\n        print(f\"Epoch {epoch+1} | Loss: {running_loss/len(train_loader.dataset):.4f} | Train RMSE: {train_rmse:.4f}\")\n\n        # Validation\n        if val_loader is not None:\n            model.eval()\n            val_preds, val_targets = [], []\n            \n            with torch.no_grad():\n                for imgs, tabs, ys in val_loader:\n                    imgs, tabs, ys = imgs.to(device), tabs.to(device), ys.unsqueeze(1).to(device)\n                    \n                    # GPU checks\n                    assert imgs.device.type == 'cuda'\n                    assert tabs.device.type == 'cuda'\n                    assert ys.device.type == 'cuda'\n                    \n                    preds = model(imgs, tabs)\n    \n                    # GPU check\n                    assert preds.device.type == 'cuda'\n        \n                    val_preds.append(preds.cpu().numpy())\n                    val_targets.append(ys.cpu().numpy())\n    \n            val_rmse = mean_squared_error(np.vstack(val_targets), np.vstack(val_preds), squared=False) # End of epochs\n            print(f\"Epoch {epoch+1} | Val RMSE: {val_rmse:.4f}\")\n    \n            scheduler.step(val_rmse)                                                                   # Use scheduler, prevent overfitting on images\n            \n            if val_rmse < best_rmse:\n                best_rmse = val_rmse\n                patience_counter = 0\n                torch.save(model.state_dict(), 'best_model.pth')\n            else:\n                patience_counter += 1\n                if patience_counter >= patience:\n                    print(\"Early stopping triggered.\")\n                    break\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:33.181829Z","iopub.execute_input":"2025-07-19T23:57:33.182348Z","iopub.status.idle":"2025-07-19T23:57:33.193750Z","shell.execute_reply.started":"2025-07-19T23:57:33.182327Z","shell.execute_reply":"2025-07-19T23:57:33.193030Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n# Submission Generation\n\ndef generate_submission(model, test_loader, device, id_col, target_col, filename='submission.csv'):\n    model.eval()\n    ids, preds = [], []\n    with torch.no_grad():\n        for imgs, tabs, img_ids in test_loader:\n            imgs, tabs = imgs.to(device), tabs.to(device)\n            outputs = model(imgs, tabs).squeeze(1).cpu().numpy()\n            ids.extend(img_ids.tolist())\n            preds.extend(outputs.tolist())\n\n    df_submission = pd.DataFrame({id_col: ids, target_col: preds})\n    df_submission.to_csv(filename, index=False)\n    print(f\"Saved submission file: {filename}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:36.443445Z","iopub.execute_input":"2025-07-19T23:57:36.444169Z","iopub.status.idle":"2025-07-19T23:57:36.448989Z","shell.execute_reply.started":"2025-07-19T23:57:36.444141Z","shell.execute_reply":"2025-07-19T23:57:36.448223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Usage outline\n\n# Define columns for imputation\ncategorical_cols = ['gem_type', 'primary_color', 'origin_culture', 'inscription_script',\n                    'pattern_type', 'environment_type', 'historical_significance', 'craftsmanship_level']\nnumeric_cols = df_train_full.select_dtypes(include=['int64', 'float64']).columns.tolist()\nfeature_cols = ['item_type', 'primary_material', 'secondary_material', 'surface_texture',\n                'rarity_level', 'oxidation_level', 'restoration_status', 'authentication_status',\n                'shine_factor', 'description']\n\n# 1. Impute missing values (run separately for debugging)\n\n# Check if we already have the imputed file\nimputed_path = 'train_imputed.csv'\nif os.path.exists(imputed_path):\n    print(\"Loading pre-imputed training data...\")\n    df_train_full = pd.read_csv(imputed_path)\nelse:\n    print(\"Imputing missing values...\")\n    df_train_full = impute_missing_values(df_train_full, categorical_cols, numeric_cols, feature_cols)\n    df_train_full.to_csv(imputed_path, index=False)\n    print(\"Saved imputed training data.\")\n\n\n# 2. Sample subset for fast iteration\n# df_train = df_train_full.sample(frac=SAMPLE_PERCENT, random_state=42).reset_index(drop=True)\n\ndf_train = df_train_full.copy().reset_index(drop=True)\n\n# 3. Preprocess tabular data (scaling, encoding)\ntabular_feats, encoders, num_tab_feats = preprocess_tabular(df_train, df_test, target_col='preservation_score', id_col='id')\n\n# 4. Add in XGBoost Model\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\nkf = KFold(n_splits=5, shuffle=True, random_state=42)\n\nval_scores, blend_preds_all, targets_all = [], [], []\n\nfor fold, (train_idx, val_idx) in enumerate(kf.split(df_train)):\n    print(f\"=== Fold {fold + 1} ===\")\n    df_train_fold = df_train.iloc[train_idx].reset_index(drop=True)\n    df_val_fold   = df_train.iloc[val_idx].reset_index(drop=True)\n\n    # XGBoost on tabular\n    X_train_tab = df_train_fold[tabular_feats]\n    y_train_tab = df_train_fold['preservation_score']\n    X_val_tab   = df_val_fold[tabular_feats]\n    y_val       = df_val_fold['preservation_score']\n\n    xgb_model = XGBRegressor(objective='reg:squarederror', n_estimators=100, max_depth=3, random_state=42)\n    xgb_model.fit(X_train_tab, y_train_tab)\n    xgb_preds = xgb_model.predict(X_val_tab)\n\n    # CNN\n    train_dataset = MultiModalDataset(df_train_fold, data_dir, 'id', tabular_feats, 'preservation_score', transform, is_train=True)\n    val_dataset   = MultiModalDataset(df_val_fold, data_dir, 'id', tabular_feats, 'preservation_score', transform, is_train=True)\n    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)\n    val_loader   = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\n    model = MultiModalModel(num_tab_feats).to(device)\n    train_model(model, train_loader, val_loader, device, NUM_EPOCHS)\n\n    model.eval()\n    cnn_preds = []\n    with torch.no_grad():\n        for imgs, tabs, _ in val_loader:\n            imgs, tabs = imgs.to(device), tabs.to(device)\n            pred = model(imgs, tabs).squeeze(1).cpu().numpy()\n            cnn_preds.extend(pred.tolist())\n    \n    cnn_preds = np.array(cnn_preds)\n    xgb_preds = np.array(xgb_preds)\n    y_val_np  = y_val.to_numpy()\n\n    blended = (cnn_preds + xgb_preds) / 2\n    rmse = mean_squared_error(y_val_np, blended, squared=False)\n    val_scores.append(rmse)\n    blend_preds_all.extend(blended)\n    targets_all.extend(y_val_np)\n\n    print(f\"Fold {fold+1} RMSE: {rmse:.4f}\")\n\nprint(f\"Avg K-Fold Blended RMSE: {np.mean(val_scores):.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-19T23:57:39.388194Z","iopub.execute_input":"2025-07-19T23:57:39.388478Z","iopub.status.idle":"2025-07-20T00:00:01.165252Z","shell.execute_reply.started":"2025-07-19T23:57:39.388455Z","shell.execute_reply":"2025-07-20T00:00:01.164230Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Submission Generation\n\nThe cell below runs inference over the full test set and produces a Kaggle-ready `submission.csv`. It:\n\n1. Switches the model to evaluation mode  \n2. Builds a DataLoader over `df_test` (using the same preprocessing and transforms)  \n3. Collects each `id` and its predicted preservation score  \n4. Constructs a DataFrame and saves it as `submission.csv`  \n","metadata":{}},{"cell_type":"code","source":"# === Fit stacking model on full tabular training data ===\ncat_model = CatBoostRegressor(verbose=0, random_state=42)\nxgb_model = XGBRegressor(objective='reg:squarederror', n_estimators=100, max_depth=3, random_state=42)\n\nstack_model = StackingRegressor(\n    estimators=[('xgb', xgb_model), ('cat', cat_model)],\n    final_estimator=LinearRegression()\n)\nstack_model.fit(df_train[tabular_feats], df_train['preservation_score'])\n\n# === Fit CNN model on full data ===\nmodel = MultiModalModel(num_tab_feats).to(device)\nfull_dataset = MultiModalDataset(df_train, data_dir, 'id', tabular_feats, 'preservation_score', transform, is_train=True)\nfull_loader = DataLoader(full_dataset, batch_size=BATCH_SIZE, shuffle=True)\ntrain_model(model, full_loader, None, device, NUM_EPOCHS)\n\n# === Predict CNN on test set ===\ncnn_test_preds, test_ids = [], []\ntest_dataset = MultiModalDataset(df_test, data_dir, 'id', tabular_feats, transform=transform, is_train=False)\ntest_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False)\n\nmodel.eval()\nwith torch.no_grad():\n    for imgs, tabs, ids in test_loader:\n        imgs, tabs = imgs.to(device), tabs.to(device)\n        preds = model(imgs, tabs).squeeze(1).cpu().numpy()\n        cnn_test_preds.extend(preds.tolist())\n        test_ids.extend(ids.tolist())\n\n# === Predict stacking model on test set ===\nstack_test_preds = stack_model.predict(df_test[tabular_feats])\n\n# === Blend predictions ===\nblended_test_preds = (np.array(cnn_test_preds) + stack_test_preds) / 2\n\n# === Save submission ===\nsubmission = pd.DataFrame({'id': test_ids, 'preservation_score': blended_test_preds})\nsubmission['id'] = submission['id'].astype(int)\nsubmission.to_csv('submission.csv', index=False)\nprint(\"Submission saved.\")","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-19T23:53:41.900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # === Submission Generation ===\n# # Prepare test dataset and loader\n# # Note: df_test must be preprocessed similarly to df_train (scaled, encoded)\n# test_dataset = MultiModalDataset(df_test, data_dir, id_col, transform=transform, is_train=False)\n# test_loader  = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n\n# # Inference on test set\n# model.eval()\n# all_ids   = []\n# all_preds = []\n# with torch.no_grad():\n#     for imgs, tabs, img_ids in test_loader:\n#         imgs, tabs = imgs.to(device), tabs.to(device)\n#         outputs = model(imgs, tabs).squeeze(1).cpu().numpy()\n#         all_ids.extend(img_ids.tolist())\n#         all_preds.extend(outputs.tolist())\n\n# # Build submission DataFrame\n# df_submission = pd.DataFrame({\n#     id_col: all_ids,\n#     target_col: all_preds\n# })\n# # Save to CSV in Kaggle required format\n# submission_file = 'submission.csv'\n# df_submission.to_csv(submission_file, index=False)\n# print(f\"Saved submission file: {submission_file}\")\n","metadata":{"trusted":true,"execution":{"execution_failed":"2025-07-19T23:53:41.900Z"}},"outputs":[],"execution_count":null}]}