{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":131037,"databundleVersionId":15734623}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-03-02T04:04:40.440101Z","iopub.execute_input":"2026-03-02T04:04:40.440469Z","iopub.status.idle":"2026-03-02T04:04:41.778765Z","shell.execute_reply.started":"2026-03-02T04:04:40.440427Z","shell.execute_reply":"2026-03-02T04:04:41.777646Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This Python script provides a basic PyTorch regression solution for the Kaggle assignment. It begins by loading the training and test datasets, then processes categorical features using label encoding to convert them into numerical form. The script selects relevant numeric features, excludes the 'id' column, and designates 'performance_score' as the target variable. It splits the data into training and validation sets and constructs a fully connected neural network using PyTorch. The model is trained with a small number of epochs and a moderate architecture to balance performance and efficiency. Finally, it evaluates the model using R-squared (R²) and RMSE, printing the results to indicate how well the model explains the variance in the target variable.","metadata":{}},{"cell_type":"code","source":"import os\nfrom pathlib import Path\n\ndef find_dataset_dir(required=(\"train.csv\", \"test.csv\")) -> Path:\n    \"\"\"Locate the dataset directory in Kaggle or locally, with helpful diagnostics.\"\"\"\n    # 0) Env var overrides\n    env_dir = os.getenv(\"DATASET_DIR\")\n    if env_dir:\n        p = Path(env_dir)\n        if p.is_dir() and all((p/f).exists() for f in required):\n            print(f\"🔧 Using DATASET_DIR override: {p}\")\n            return p\n        else:\n            print(f\"⚠️ DATASET_DIR set but invalid: {p}\")\n\n    env_name = os.getenv(\"DATASET_NAME\")\n    kaggle_input = Path(\"/kaggle/input\")\n    if env_name and kaggle_input.exists():\n        cand = kaggle_input / env_name\n        if cand.is_dir() and all((cand/f).exists() for f in required):\n            print(f\"🔧 Using DATASET_NAME under /kaggle/input: {cand}\")\n            return cand\n        else:\n            print(f\"⚠️ DATASET_NAME set but not found or missing files: {cand}\")\n\n    candidates = []\n\n    # 1) Exact files at /kaggle/input (rare, but check)\n    if kaggle_input.exists():\n        if all((kaggle_input/f).exists() for f in required):\n            candidates.append(kaggle_input)\n\n        # 2) Any subdir of /kaggle/input containing required files (non-recursive, then recursive)\n        for d in kaggle_input.iterdir():\n            if d.is_dir() and all((d/f).exists() for f in required):\n                candidates.append(d)\n\n        # 3) Recursive search as a last resort (handles nested or renamed datasets)\n        if not candidates:\n            trains = list(kaggle_input.rglob(required[0]))\n            tests  = set(p.parent for p in kaggle_input.rglob(required[1]))\n            for t in trains:\n                if t.parent in tests:\n                    candidates.append(t.parent)\n\n    # 4) Local working directory fallback\n    cwd = Path(\".\").resolve()\n    if all((cwd/f).exists() for f in required):\n        candidates.append(cwd)\n\n    # Deduplicate while preserving order\n    seen = set()\n    unique = []\n    for c in candidates:\n        if str(c) not in seen:\n            seen.add(str(c))\n            unique.append(c)\n\n    if not unique:\n        # Helpful diagnostics\n        print(\"❌ Could not find a folder containing both train.csv and test.csv.\")\n        if kaggle_input.exists():\n            print(\"📁 Top-level entries in /kaggle/input:\")\n            for d in kaggle_input.iterdir():\n                print(\"  -\", d.name)\n        print(\"🔎 Searched:\", kaggle_input if kaggle_input.exists() else \"(no /kaggle/input)\")\n        print(\"🔎 Also checked working dir:\", cwd)\n        raise FileNotFoundError(\"Could not find dataset directory containing train.csv and test.csv\")\n\n    # Pick the most recently modified candidate\n    chosen = max(unique, key=lambda p: p.stat().st_mtime if p.exists() else 0)\n    # Print a short diagnostic list\n    print(\"✅ Candidates that contain the required files:\")\n    for c in unique:\n        marker = \" (chosen)\" if c == chosen else \"\"\n        print(f\"  - {c}{marker}\")\n    return chosen\n\n# Usage:\ndataset_dir = find_dataset_dir()\nprint(f\"📂 Using dataset directory: {dataset_dir}\")\n\ntrain_path = dataset_dir / \"train.csv\"\ntest_path  = dataset_dir / \"test.csv\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-02T04:05:45.461788Z","iopub.execute_input":"2026-03-02T04:05:45.462151Z","iopub.status.idle":"2026-03-02T04:05:45.488976Z","shell.execute_reply.started":"2026-03-02T04:05:45.462119Z","shell.execute_reply":"2026-03-02T04:05:45.487548Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"This notebook loads train and test from the paths discovered earlier, infers the first column as the ID and the right-most column as the regression target, drops rows with invalid targets, and splits the remaining data into train and validation sets. Numeric features are coerced to numbers, infinities are replaced with NaN, then imputed with the median and scaled with StandardScaler (fit only on the training fold to avoid leakage). Categorical features are label-encoded with explicit handling for unseen values via an “unknown” token, using the same encoders for validation and test. The processed blocks are stacked into design matrices, checked for any lingering NaN or inf values, and converted to PyTorch tensors. A small fully connected regressor (128-64-1 with ReLU) is trained with Adam, MSE loss, and light weight decay, and progress is monitored on the validation split. Finally, the model is evaluated using R² and RMSE on the validation set.\n\n**Ways to improve accuracy**\n- Use cross-validation: replace a single holdout with K-fold or StratifiedKFold (on binned target) to reduce variance, reporting CV mean and std.  \n- Hyperparameter search: tune network width/depth, dropout, weight decay, learning rate, batch size, and epochs with early stopping on a validation fold.  \n- Learning-rate schedule: cosine decay or ReduceLROnPlateau; optionally warmup for a few epochs.  \n- Regularization: add dropout and/or batch normalization; consider L1/L2 on weights; try RobustScaler for heavy-tailed numerics.  \n- Target transform: if the target is skewed, train on log1p(y) and invert with expm1 at inference.  \n- Feature engineering: interactions, ratios, counts, date parts, text embeddings for any textual fields, and binning heavily skewed numerics.  \n- Categorical handling: try target encoding or CatBoost encoding with nested CV to avoid leakage; for very high-cardinality columns, consider hashing or learned embeddings in the NN.  \n- Outlier strategy: winsorize or clip extreme numeric values; optionally train a Huber or Quantile loss variant for robustness.  \n- Model alternatives and ensembling: compare tree models (XGBoost/LightGBM/CatBoost with early stopping) and average or stack with the NN.  \n- Data leakage checks: ensure all preprocessors are fit only on training folds; audit any derived features that might peek at validation.  \n- Calibration and error analysis: slice metrics by important subgroups, inspect permutation or SHAP importances, and iterate on weak areas.\n","metadata":{}},{"cell_type":"code","source":"# --- Robust tabular -> PyTorch regressor (auto target = right-most col) ---\n\nimport pandas as pd\nimport numpy as np\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nfrom typing import Optional\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.metrics import r2_score\n\n# ---------------------------------------------------------------------------\n# Assumes train_path and test_path are already defined by your find_dataset_dir()\n# ---------------------------------------------------------------------------\n\n# --- Load ---\ndf = pd.read_csv(train_path)\ndf_test = pd.read_csv(test_path)\n\n# --- Identify columns ---\nid_col = df.columns[0]\ntarget_col = df.columns[-1]\nprint(f\"🪪 ID column: {id_col}\")\nprint(f\"🎯 Target column: {target_col}\")\n\n# --- Target cleaning: drop rows with NaN/inf in target ---\ny_raw = df[target_col].replace([np.inf, -np.inf], np.nan)\nkeep = y_raw.notna()\ndropped = (~keep).sum()\nif dropped:\n    print(f\"⚠️ Dropping {dropped} training rows due to invalid target values\")\ndf = df.loc[keep].reset_index(drop=True)\n\n# --- Split features/target ---\nX_df = df.drop(columns=[id_col, target_col])\ny = df[target_col].astype(float)\n\n# --- Column types ---\ncat_cols = X_df.select_dtypes(include=[\"object\", \"category\"]).columns.tolist()\nnum_cols = X_df.select_dtypes(include=[np.number, \"bool\"]).columns.tolist()\n\n# --- Coerce numerics and scrub infs (so the imputer can work) ---\nfor c in num_cols:\n    X_df[c] = pd.to_numeric(X_df[c], errors=\"coerce\")\n    X_df[c] = X_df[c].replace([np.inf, -np.inf], np.nan)\n\n# --- Train/val split BEFORE fitting preprocessors (avoid leakage) ---\nX_train_df, X_val_df, y_train, y_val = train_test_split(\n    X_df, y, test_size=0.2, random_state=42\n)\n\n# --- Categorical encoding with 'unknown' handling for val/test ---\nlabel_encoders = {}\n\ndef _fit_transform_cat(train_s: pd.Series, val_s: pd.Series, test_s: Optional[pd.Series]):\n    le = LabelEncoder()\n    train_f = train_s.astype(str).fillna(\"missing\")\n    le.fit(train_f.unique())\n\n    # ensure 'unknown' exists (for unseen categories)\n    if \"unknown\" not in le.classes_:\n        le.classes_ = np.append(le.classes_, \"unknown\")\n\n    def _transform(s: pd.Series):\n        s2 = s.astype(str).fillna(\"missing\")\n        mask = ~np.isin(s2, le.classes_)\n        if mask.any():\n            s2.loc[mask] = \"unknown\"\n        return le.transform(s2)\n\n    train_t = _transform(train_s)\n    val_t   = _transform(val_s)\n    test_t  = _transform(test_s) if test_s is not None else None\n    return le, train_t, val_t, test_t\n\nX_train_cat, X_val_cat, X_test_cat = [], [], []\nfor c in cat_cols:\n    test_series = df_test[c] if c in df_test.columns else None\n    le, tr, va, te = _fit_transform_cat(X_train_df[c], X_val_df[c], test_series)\n    label_encoders[c] = le\n    X_train_cat.append(tr.reshape(-1, 1))\n    X_val_cat.append(va.reshape(-1, 1))\n    if te is not None:\n        X_test_cat.append(te.reshape(-1, 1))\n\n# --- Numeric: Impute (median) then scale (fit on train only) ---\nnum_imputer = SimpleImputer(strategy=\"median\")\nX_train_num = num_imputer.fit_transform(X_train_df[num_cols]) if num_cols else np.empty((len(X_train_df), 0))\nX_val_num   = num_imputer.transform(X_val_df[num_cols])       if num_cols else np.empty((len(X_val_df), 0))\nX_test_num  = num_imputer.transform(df_test[num_cols])        if num_cols else np.empty((len(df_test), 0))\n\nnum_scaler = StandardScaler()\nX_train_num = num_scaler.fit_transform(X_train_num) if num_cols else X_train_num\nX_val_num   = num_scaler.transform(X_val_num)       if num_cols else X_val_num\nX_test_num  = num_scaler.transform(X_test_num)      if num_cols else X_test_num\n\n# --- Stack feature blocks ---\ndef _hstack(parts):\n    parts = [p for p in parts if p is not None and getattr(p, \"size\", 0) > 0]\n    return np.hstack(parts) if parts else np.empty((len(X_train_df), 0))\n\nX_train = _hstack([X_train_num] + X_train_cat)\nX_val   = _hstack([X_val_num]   + X_val_cat)\n\n# Build test features (optional if you need submit preds later)\ndef _hstack_test(parts):\n    parts = [p for p in parts if p is not None and getattr(p, \"size\", 0) > 0]\n    return np.hstack(parts) if parts else np.empty((len(df_test), 0))\n\nX_test = _hstack_test([X_test_num] + X_test_cat)\n\n# --- Final NaN/inf guards ---\ndef _validate(arr, name):\n    bad = np.isnan(arr).sum()\n    if bad:\n        raise ValueError(f\"{name} still contains {bad} NaNs after preprocessing.\")\n    if np.isinf(arr).any():\n        raise ValueError(f\"{name} contains inf values after preprocessing.\")\n\n_validate(X_train, \"X_train\")\n_validate(X_val,   \"X_val\")\nif np.isnan(y_train).any() or np.isinf(y_train).any():\n    raise ValueError(\"y_train contains NaN/inf.\")\nif np.isnan(y_val).any() or np.isinf(y_val).any():\n    raise ValueError(\"y_val contains NaN/inf.\")\n\n# --- Torch tensors ---\nX_train_tensor = torch.tensor(X_train, dtype=torch.float32)\nX_val_tensor   = torch.tensor(X_val,   dtype=torch.float32)\ny_train_tensor = torch.tensor(y_train.values, dtype=torch.float32).view(-1, 1)\ny_val_tensor   = torch.tensor(y_val.values,   dtype=torch.float32).view(-1, 1)\n\n# --- Model ---\nclass RegressionNN(nn.Module):\n    def __init__(self, input_dim: int):\n        super().__init__()\n        self.fc = nn.Sequential(\n            nn.Linear(input_dim, 128),\n            nn.ReLU(),\n            nn.Linear(128, 64),\n            nn.ReLU(),\n            nn.Linear(64, 1)\n        )\n    def forward(self, x):\n        return self.fc(x)\n\ninput_dim = X_train.shape[1]\nprint(f\"✅ Input dimensionality after preprocessing: {input_dim}\")\nmodel = RegressionNN(input_dim)\n\ncriterion = nn.MSELoss()\noptimizer = optim.Adam(model.parameters(), lr=0.001, weight_decay=1e-4)\n\n# --- Training ---\nepochs = 50\nfor epoch in range(epochs):\n    model.train()\n    optimizer.zero_grad()\n    y_pred = model(X_train_tensor)\n    loss = criterion(y_pred, y_train_tensor)\n    if not torch.isfinite(loss):\n        raise RuntimeError(\"Loss became non-finite; check preprocessing or reduce LR.\")\n    loss.backward()\n    optimizer.step()\n\n    if (epoch + 1) % 10 == 0:\n        model.eval()\n        with torch.no_grad():\n            val_pred = model(X_val_tensor)\n            val_loss = criterion(val_pred, y_val_tensor).item()\n        print(f\"Epoch [{epoch+1}/{epochs}]  train_loss={loss.item():.6f}  val_loss={val_loss:.6f}\")\n\n# --- Evaluation ---\nmodel.eval()\nwith torch.no_grad():\n    y_pred_val = model(X_val_tensor).cpu().numpy().ravel()\n\n# Compute metrics\nr2 = r2_score(y_val, y_pred_val)\nrmse = np.sqrt(np.mean((y_pred_val - y_val.values) ** 2))\n\nprint(f\"R-squared: {r2:.4f}\")\nprint(f\"RMSE: {rmse:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-02T04:07:10.465991Z","iopub.execute_input":"2026-03-02T04:07:10.466969Z","iopub.status.idle":"2026-03-02T04:07:20.897783Z","shell.execute_reply.started":"2026-03-02T04:07:10.466923Z","shell.execute_reply":"2026-03-02T04:07:20.896688Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature Importance\n\nThis cell computes permutation-based feature importance for the trained PyTorch regression model. It begins by switching the model to evaluation mode and defining a helper function that measures how much the model’s predictive performance (R²) drops when the values of each feature are randomly shuffled. For each column in the validation feature matrix, a copy of the data is made, that single feature is permuted, and the model’s predictions are compared against the baseline. The resulting difference in R² quantifies how strongly that feature influences predictions—the larger the decrease, the more important the feature. The code then pairs these importance values with the corresponding feature names, sorts them in descending order, and visualizes the results as a horizontal bar chart. This helps interpret which input features the neural network relies on most when making predictions.\n","metadata":{}},{"cell_type":"code","source":"# --- Permutation Importance (for PyTorch model) ---\nfrom sklearn.inspection import permutation_importance\nimport matplotlib.pyplot as plt\n\n# Ensure model is in evaluation mode\nmodel.eval()\n\ndef compute_permutation_importance(model, X_tensor, y_tensor, metric=r2_score):\n    \"\"\"Compute permutation importance for each feature.\"\"\"\n    # Convert tensors to NumPy for metrics\n    y_true = y_tensor.detach().cpu().numpy().flatten()\n    baseline_pred = model(X_tensor).detach().cpu().numpy().flatten()\n    baseline_score = metric(y_true, baseline_pred)\n\n    importances = []\n    for i in range(X_tensor.shape[1]):\n        X_perm = X_tensor.clone()\n        # Shuffle the i-th feature\n        X_perm[:, i] = X_perm[:, i][torch.randperm(X_tensor.shape[0])]\n        perm_pred = model(X_perm).detach().cpu().numpy().flatten()\n        perm_score = metric(y_true, perm_pred)\n        importances.append(baseline_score - perm_score)\n\n    return np.array(importances)\n\n# Use validation data for importance (more honest than training)\nimportances = compute_permutation_importance(model, X_val_tensor, y_val_tensor)\n\n# Retrieve dynamic feature names (without hardcoding column names)\nfeature_names = X_df.columns.tolist()\n\n# Create DataFrame for feature importance\nfeature_importance_df = pd.DataFrame({\n    'Feature': feature_names,\n    'Importance': importances\n}).sort_values(by='Importance', ascending=False)\n\n# Display feature importance\nplt.figure(figsize=(10, 6))\nplt.barh(feature_importance_df['Feature'], feature_importance_df['Importance'])\nplt.xlabel(\"Importance\")\nplt.ylabel(\"Feature\")\nplt.title(\"Feature Importance (Permutation, PyTorch Model)\")\nplt.gca().invert_yaxis()\nplt.tight_layout()\nplt.show()\n\n# Optional: print top 10\nprint(\"\\nTop 10 most important features:\")\nprint(feature_importance_df.head(10))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-02T04:08:16.827679Z","iopub.execute_input":"2026-03-02T04:08:16.828243Z","iopub.status.idle":"2026-03-02T04:08:17.851533Z","shell.execute_reply.started":"2026-03-02T04:08:16.828207Z","shell.execute_reply":"2026-03-02T04:08:17.849746Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Kaggle Submit\n\nThis cell generates the final Kaggle submission file using the trained PyTorch model and the same preprocessing pipeline applied to the training data. It first retrieves the test set IDs without altering the original `df_test` and relies on the precomputed `X_test`, which was processed through the same label encoders, imputers, and scaler fitted on the training data to maintain consistency. The feature matrix is converted to a PyTorch tensor, and the model—set to evaluation mode—produces predictions for each test record. These predictions are flattened and paired with their corresponding IDs to create a submission DataFrame, where the predicted values are stored under the dynamically detected target column name. Finally, the results are saved as `submission.csv`, the standard output format for Kaggle competitions.\n","metadata":{}},{"cell_type":"code","source":"# --- Kaggle submission (uses the same preprocessing + dynamic target name) ---\n\n# Grab test IDs without mutating df_test\ntest_ids = df_test[id_col].copy()\n\n# X_test was already built earlier from:\n# - categorical encoders with 'unknown' handling\n# - numeric median imputer + StandardScaler (fit on train only)\n# If you need to rebuild, ensure you use the SAME fitted imputers/encoders/scaler.\n\n# Convert to tensor and predict\nX_test_final_tensor = torch.tensor(X_test, dtype=torch.float32)\n\nmodel.eval()\nwith torch.no_grad():\n    submission_preds = model(X_test_final_tensor).cpu().numpy().flatten()\n\n# Create submission with the dynamically detected target column name\nsubmission = pd.DataFrame({\n    id_col: test_ids,\n    target_col: submission_preds\n})\n\n# Save\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"✅ Kaggle submission file 'submission.csv' created successfully.\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-03-02T04:09:23.160917Z","iopub.execute_input":"2026-03-02T04:09:23.161394Z","iopub.status.idle":"2026-03-02T04:09:23.181651Z","shell.execute_reply.started":"2026-03-02T04:09:23.161362Z","shell.execute_reply":"2026-03-02T04:09:23.180550Z"}},"outputs":[],"execution_count":null}]}