{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pathlib\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn import decomposition, model_selection, preprocessing, pipeline\nimport torch\nfrom torch import nn, utils\nimport torchmetrics\n\nfrom sklearn.preprocessing import FunctionTransformer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:23.222502Z","iopub.execute_input":"2025-12-10T20:15:23.222889Z","iopub.status.idle":"2025-12-10T20:15:43.173950Z","shell.execute_reply.started":"2025-12-10T20:15:23.222865Z","shell.execute_reply":"2025-12-10T20:15:43.173019Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 0. Load the data (5 pts)\n\nComplete the code in the cells below to load the MNIST dataset in to a Pandas `DataFrame` using the `pd.read_csv` function. Be sure to use the `np.uint8` data-type to represent the pixel values as non-negative integers.","metadata":{}},{"cell_type":"code","source":"INPUT_DIR = pathlib.Path(\"/kaggle/input\")\nOUTPUT_DIR = pathlib.Path(\"/kaggle/working\")\nTEMP_DIR = pathlib.Path(\"/kaggle/temp\")\nDATA_DIR = INPUT_DIR / \"digit-recognizer\"\n\nRANDOM_STATE = np.random.RandomState(42)\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:43.175472Z","iopub.execute_input":"2025-12-10T20:15:43.176038Z","iopub.status.idle":"2025-12-10T20:15:43.185173Z","shell.execute_reply.started":"2025-12-10T20:15:43.176006Z","shell.execute_reply":"2025-12-10T20:15:43.183654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv?","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:43.186455Z","iopub.execute_input":"2025-12-10T20:15:43.187099Z","iopub.status.idle":"2025-12-10T20:15:43.256024Z","shell.execute_reply.started":"2025-12-10T20:15:43.187062Z","shell.execute_reply":"2025-12-10T20:15:43.254696Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mnist_df = pd.read_csv(\n    DATA_DIR / \"train.csv\",\n    dtype=np.uint8,\n)\n\ntest_features_df = pd.read_csv(\n    DATA_DIR / \"test.csv\",\n    dtype=np.uint8,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:43.256932Z","iopub.execute_input":"2025-12-10T20:15:43.257183Z","iopub.status.idle":"2025-12-10T20:15:47.084202Z","shell.execute_reply.started":"2025-12-10T20:15:43.257163Z","shell.execute_reply":"2025-12-10T20:15:47.083093Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Quick check\nprint(mnist_df.shape, test_features_df.shape)\nmnist_df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.086781Z","iopub.execute_input":"2025-12-10T20:15:47.087059Z","iopub.status.idle":"2025-12-10T20:15:47.120252Z","shell.execute_reply.started":"2025-12-10T20:15:47.087037Z","shell.execute_reply":"2025-12-10T20:15:47.119499Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Simulate a partially labeled dataset (5 pts)\n\nComplete the code in the cell below to create a simulated, partially labeled version of the MNIST dataset.\n\nYou must use the `model_selection.train_test_split` function from Scikit-Learn to split the provided training data into \"labeled\" and \"unlabeled\" pieces using stratified sampling. The unlabeled piece should contain 80% of the original training data. ","metadata":{}},{"cell_type":"code","source":"model_selection.train_test_split?","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.121013Z","iopub.execute_input":"2025-12-10T20:15:47.121266Z","iopub.status.idle":"2025-12-10T20:15:47.141154Z","shell.execute_reply.started":"2025-12-10T20:15:47.121242Z","shell.execute_reply":"2025-12-10T20:15:47.140097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_mnist_df, unlabeled_mnist_df = (\n    model_selection.train_test_split(\n        mnist_df,\n        stratify=mnist_df[\"label\"],\n        test_size=0.8,\n        random_state=42\n    )\n)\n\n# we drop the label to simulate an unlabeled subset of data\nunlabeled_features_df = unlabeled_mnist_df.drop(\"label\", axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.142066Z","iopub.execute_input":"2025-12-10T20:15:47.142329Z","iopub.status.idle":"2025-12-10T20:15:47.343839Z","shell.execute_reply.started":"2025-12-10T20:15:47.142310Z","shell.execute_reply":"2025-12-10T20:15:47.342911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 8400 rows and 785 columns!\nlabeled_mnist_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.344801Z","iopub.execute_input":"2025-12-10T20:15:47.345076Z","iopub.status.idle":"2025-12-10T20:15:47.382207Z","shell.execute_reply.started":"2025-12-10T20:15:47.345054Z","shell.execute_reply":"2025-12-10T20:15:47.381333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 33600 rows and 784 cols!\nunlabeled_features_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.383035Z","iopub.execute_input":"2025-12-10T20:15:47.383378Z","iopub.status.idle":"2025-12-10T20:15:47.421244Z","shell.execute_reply.started":"2025-12-10T20:15:47.383357Z","shell.execute_reply":"2025-12-10T20:15:47.420450Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Create a train/val split for the labeled data (5 pts)\n\nComplete the code in the cell below to split your labeled training data into training and validation sets using stratified sampling. The validation set should contain 10% of the labeled training data. ","metadata":{}},{"cell_type":"code","source":"train_df, val_df = (\n    model_selection.train_test_split(\n        labeled_mnist_df,\n        stratify=labeled_mnist_df[\"label\"],\n        test_size=0.1,\n        random_state=42\n    )\n)\n\ntrain_features_df = train_df.drop(\"label\", axis=1)\ntrain_labels = train_df.loc[:, \"label\"]\n\nval_features_df = val_df.drop(\"label\", axis=1)\nval_labels = val_df.loc[:, \"label\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.422081Z","iopub.execute_input":"2025-12-10T20:15:47.422320Z","iopub.status.idle":"2025-12-10T20:15:47.451782Z","shell.execute_reply.started":"2025-12-10T20:15:47.422299Z","shell.execute_reply":"2025-12-10T20:15:47.450558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 7560 rows and 784 cols!\ntrain_features_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.452571Z","iopub.execute_input":"2025-12-10T20:15:47.452887Z","iopub.status.idle":"2025-12-10T20:15:47.486031Z","shell.execute_reply.started":"2025-12-10T20:15:47.452864Z","shell.execute_reply":"2025-12-10T20:15:47.485136Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 840 rows and 784 cols!\nval_features_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.487027Z","iopub.execute_input":"2025-12-10T20:15:47.487355Z","iopub.status.idle":"2025-12-10T20:15:47.521853Z","shell.execute_reply.started":"2025-12-10T20:15:47.487328Z","shell.execute_reply":"2025-12-10T20:15:47.520945Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Prepare the datasets (20 pts)","metadata":{}},{"cell_type":"markdown","source":"### Converting DataFrames to Tensors (10 pts)\n\nComplete the code in the cell below in order to define a Scikit Learn preprocessing pipeline for your features that has the following steps.\n\n1. Uses a `StandardScaler` to standardize the features.\n2. Uses a `FunctionTransformer` to convert a Numpy Array to a Torch Tensor with a `torch.float32` dtype.\n\nNext, define a Scikit Learn preprocessing pipeline for your target that has the following steps.\n\n1. Uses a `FunctionTransformer` to convert a Pandas Series to a Numpy Array.\n2. Uses a `FunctionTransformer` to convert a NumPy Array to a Torch Tensor with a `torch.int64` dtype.\n\n\n\n","metadata":{}},{"cell_type":"code","source":"features_preprocessing_pipeline = pipeline.make_pipeline(\n    preprocessing.StandardScaler(),  # standardize features\n    FunctionTransformer(lambda X: torch.tensor(X, dtype=torch.float32))  # convert to torch tensor\n\n)\n\ntarget_preprocessing_pipeline = pipeline.make_pipeline(\n    FunctionTransformer(lambda y: y.to_numpy()),  # convert Pandas Series to NumPy array\n    FunctionTransformer(lambda y: torch.tensor(y, dtype=torch.int64))  # convert to torch tensor\n)\n\ntrain_features_tensor = (\n    features_preprocessing_pipeline.fit_transform(\n        train_features_df\n    )\n)\n\ntrain_target_tensor = (\n    target_preprocessing_pipeline.fit_transform(\n        train_labels\n    )\n)\n\nval_features_tensor = (\n    features_preprocessing_pipeline.transform(\n        val_features_df\n    )\n)\n\nval_target_tensor = (\n    target_preprocessing_pipeline.transform(\n        val_labels\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.522803Z","iopub.execute_input":"2025-12-10T20:15:47.523208Z","iopub.status.idle":"2025-12-10T20:15:47.718304Z","shell.execute_reply.started":"2025-12-10T20:15:47.523179Z","shell.execute_reply":"2025-12-10T20:15:47.717525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 7560 by 784!\nprint(train_features_tensor.shape)\n\n# should be float32!\nprint(train_features_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.722342Z","iopub.execute_input":"2025-12-10T20:15:47.722658Z","iopub.status.idle":"2025-12-10T20:15:47.727971Z","shell.execute_reply.started":"2025-12-10T20:15:47.722623Z","shell.execute_reply":"2025-12-10T20:15:47.727052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 7560!\nprint(train_target_tensor.shape)\n\n# should be int64!\nprint(train_target_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.728776Z","iopub.execute_input":"2025-12-10T20:15:47.729039Z","iopub.status.idle":"2025-12-10T20:15:47.746014Z","shell.execute_reply.started":"2025-12-10T20:15:47.729020Z","shell.execute_reply":"2025-12-10T20:15:47.745032Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 840 by 784!\nprint(val_features_tensor.shape)\n\n# should be float32!\nprint(val_features_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.746972Z","iopub.execute_input":"2025-12-10T20:15:47.747239Z","iopub.status.idle":"2025-12-10T20:15:47.764577Z","shell.execute_reply.started":"2025-12-10T20:15:47.747220Z","shell.execute_reply":"2025-12-10T20:15:47.763518Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 840!\nprint(val_target_tensor.shape)\n\n# should be int64!\nprint(val_target_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.765653Z","iopub.execute_input":"2025-12-10T20:15:47.766189Z","iopub.status.idle":"2025-12-10T20:15:47.782092Z","shell.execute_reply.started":"2025-12-10T20:15:47.766159Z","shell.execute_reply":"2025-12-10T20:15:47.781258Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create TensorDatasets (5 pts)\n\nComplete the code in the cell below to create `utils.data.TensorDatasets` from the training and validation tensors.","metadata":{}},{"cell_type":"code","source":"train_dataset = utils.data.TensorDataset(\n    train_features_tensor,  # features tensor\n    train_target_tensor     # target tensor\n)\n\nval_dataset = utils.data.TensorDataset(\n    val_features_tensor,    # features tensor\n    val_target_tensor       # target tensor\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.783039Z","iopub.execute_input":"2025-12-10T20:15:47.783277Z","iopub.status.idle":"2025-12-10T20:15:47.803240Z","shell.execute_reply.started":"2025-12-10T20:15:47.783260Z","shell.execute_reply":"2025-12-10T20:15:47.802229Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create DataLoaders (5 pts)\n\nComplete the code in the cell below to create `utils.data.DataLoaders` for the training and validation datasets.\n\n","metadata":{}},{"cell_type":"code","source":"# Define batch size and number workers\nBATCH_SIZE = 64\nNUM_WORKERS = 2","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.804310Z","iopub.execute_input":"2025-12-10T20:15:47.804609Z","iopub.status.idle":"2025-12-10T20:15:47.820162Z","shell.execute_reply.started":"2025-12-10T20:15:47.804560Z","shell.execute_reply":"2025-12-10T20:15:47.819287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_dataloader = utils.data.DataLoader(\n    train_dataset,      # TensorDataset\n    batch_size=BATCH_SIZE,\n    persistent_workers=True,\n    num_workers=NUM_WORKERS,\n    shuffle=True        # shuffle training data\n\n)\n\n\nval_dataloader = utils.data.DataLoader(\n    val_dataset,        # TensorDataset\n    batch_size=BATCH_SIZE,\n    persistent_workers=True,\n    num_workers=NUM_WORKERS,\n    shuffle=False       # no need to shuffle validation data\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.821058Z","iopub.execute_input":"2025-12-10T20:15:47.821309Z","iopub.status.idle":"2025-12-10T20:15:47.838650Z","shell.execute_reply.started":"2025-12-10T20:15:47.821289Z","shell.execute_reply":"2025-12-10T20:15:47.837813Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Define a model, optimizer, and LR scheduler (20 pts)\n\nUsing only the functions provided below define a self-normalizing MLP with three hidden layers. Configure the `optim.AdamW` optimizer, and configure the `optim.lr_scheduler.ReduceLROnPlateau` learning rate scheduler.","metadata":{}},{"cell_type":"code","source":"import torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.utils as utils","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.839699Z","iopub.execute_input":"2025-12-10T20:15:47.839963Z","iopub.status.idle":"2025-12-10T20:15:47.858471Z","shell.execute_reply.started":"2025-12-10T20:15:47.839944Z","shell.execute_reply":"2025-12-10T20:15:47.857555Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def initialize_linear_layer(\n    in_features,\n    out_features,\n    init_strategy_=nn.init.kaiming_uniform_,\n    init_strategy_kwargs=None,\n    ):\n    linear_layer = nn.Linear(in_features, out_features)\n\n    if init_strategy_ is not None:\n        if init_strategy_kwargs is None:\n            init_strategy_kwargs = {}\n        init_strategy_(linear_layer.weight, **init_strategy_kwargs)\n        linear_layer.bias.data.fill_(0.0)\n\n    return linear_layer\n\n\ndef make_mlp_classifier(\n    input_size,\n    hidden_sizes=None,\n    output_size=2,\n    activation_fn=None,\n    init_strategy_=nn.init.kaiming_uniform_,\n    init_strategy_kwargs=None,\n    batch_normalization=False\n    ):\n    \n    modules = []\n    hidden_sizes = [] if hidden_sizes is None else hidden_sizes\n    for hidden_size in hidden_sizes:\n        hidden_layer = initialize_linear_layer(\n            input_size,\n            hidden_size,\n            init_strategy_,\n            init_strategy_kwargs,\n        )\n        modules.append(hidden_layer)\n\n        # batch normalization goes after the linear layer...\n        if batch_normalization:\n            modules.append(nn.BatchNorm1d(hidden_size))\n\n        # ...but before the activation_fn!\n        if activation_fn is not None:\n            modules.append(activation_fn)\n        input_size=hidden_size\n\n    output_layer = initialize_linear_layer(\n            input_size,\n            output_size,\n            init_strategy_,\n            init_strategy_kwargs,\n    )\n    modules.append(output_layer)\n    model_fn = nn.Sequential(*modules)\n    return nn.CrossEntropyLoss(), model_fn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.859425Z","iopub.execute_input":"2025-12-10T20:15:47.859725Z","iopub.status.idle":"2025-12-10T20:15:47.877970Z","shell.execute_reply.started":"2025-12-10T20:15:47.859704Z","shell.execute_reply":"2025-12-10T20:15:47.876917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_size = train_features_tensor.shape[1]  # 784 for MNIST,\noutput_size = 10                              # 10 classes for digits 0-9\nhidden_size = int((2 / 3) * (input_size + output_size))\nhidden_sizes = [hidden_size] * 3  # [530, 530, 530]\n\ncriterion, model_fn = make_mlp_classifier(\n    input_size,\n    hidden_sizes,\n    output_size,\n    activation_fn= nn.SELU(),\n    init_strategy_ = nn.init.kaiming_normal_,\n    init_strategy_kwargs = {\"mode\": \"fan_in\", \"nonlinearity\": \"linear\"},\n    batch_normalization=False       # SELU does not need batchnorm\n)\nmodel_fn = model_fn.to(DEVICE)\n\noptimizer = optim.AdamW(\n    model_fn.parameters(),\n    lr=1e-3,          # standard starting LR\n    weight_decay=1e-4 # optional regularization\n)\n\nlr_scheduler = (\n    optim.lr_scheduler\n         .ReduceLROnPlateau(\n                 optimizer,\n                 mode='min',            # monitor validation loss\n                 factor=0.5,            # reduce LR by half on plateau\n                 patience=5,            # wait 5 epochs before reducing\n                 verbose=True\n\n         )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.878924Z","iopub.execute_input":"2025-12-10T20:15:47.879183Z","iopub.status.idle":"2025-12-10T20:15:47.923709Z","shell.execute_reply.started":"2025-12-10T20:15:47.879165Z","shell.execute_reply":"2025-12-10T20:15:47.922867Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Train your model (20 pts)\n\nUsing the training loop provided below, train your MLP for 100 epochs and plot the training and validation curves as well as the learning rate.","metadata":{}},{"cell_type":"code","source":"def compute_average_loss_per_batch(dataloader, criterion, model_fn):\n    total_loss = torch.zeros(1, 1)\n    num_batches = len(dataloader)\n    for features, targets in dataloader:\n        predictions = model_fn(features)\n        batch_loss = criterion(predictions, targets)\n        total_loss += batch_loss\n    average_loss_per_batch = total_loss / num_batches\n    return average_loss_per_batch\n\n\ndef clip_gradients_(\n    clip_grad_strategy,\n    model_fn,\n    clip_value=None,\n    error_if_nonfinite=False,\n    max_norm=None,\n    norm_type=2.0):\n    if clip_grad_strategy == \"value\" and clip_value is not None:\n        nn.utils.clip_grad_value_(\n            model_fn.parameters(),\n            clip_value\n        )\n    elif clip_grad_strategy == \"norm\" and max_norm is not None:\n        nn.utils.clip_grad_norm_(\n            model_fn.parameters(),\n            max_norm,\n            norm_type,\n            error_if_nonfinite\n        )\n    else:\n        raise NotImplementedError()\n\n\ndef evaluate(model_fn, dataloader, metric, device=\"cpu\"):\n    model_fn.eval()\n    metric.reset()\n    with torch.inference_mode():\n        for X_batch, y_batch in dataloader:\n            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n            y_pred = model_fn(X_batch)\n            metric.update(y_pred, y_batch)\n    return metric.compute()\n\n\ndef fit(\n    criterion,\n    model_fn,\n    optimizer,\n    train_dataloader,\n    val_dataloader,\n    device=\"cpu\",\n    clip_grad_strategy=None,\n    clip_value=None,\n    error_if_nonfinite=False,\n    lr_scheduler=None,\n    log_epochs=1,\n    max_epochs=1,\n    max_norm=None,\n    norm_type=2.0\n    ):\n\n    history = {\n        \"epoch\": [],\n        \"average_train_loss\": [],\n        \"average_val_loss\": [],\n        \"lr\": [],\n    }\n    num_train_batches = len(train_dataloader)\n    for epoch in range(max_epochs):\n        total_train_loss = torch.zeros(1, 1)\n        model_fn = model_fn.train()\n        for features, targets in train_dataloader:\n\n            # forward pass\n            features, targets = features.to(device), targets.to(device)\n            predictions = model_fn(features)\n            batch_loss = criterion(predictions, targets)\n            total_train_loss += batch_loss\n\n            # backward pass\n            optimizer.zero_grad()\n            batch_loss.backward()\n            if clip_grad_strategy is not None:\n                clip_gradients_(\n                    clip_grad_strategy,\n                    model_fn,\n                    clip_value,\n                    error_if_nonfinite,\n                    max_norm,\n                    norm_type\n                )\n            optimizer.step()\n\n        history[\"epoch\"].append(epoch)\n\n        average_train_loss_per_batch = total_train_loss / num_train_batches\n        history[\"average_train_loss\"].append(average_train_loss_per_batch.item())\n\n        model_fn = model_fn.eval()\n        with torch.inference_mode():\n            average_val_loss_per_batch = compute_average_loss_per_batch(\n                val_dataloader,\n                criterion,\n                model_fn\n            )\n        history[\"average_val_loss\"].append(average_val_loss_per_batch.item())\n\n        # update the learning rate after every training epoch\n        if lr_scheduler is not None:\n            lr_scheduler.step(average_val_loss_per_batch.item())\n            history[\"lr\"].append(lr_scheduler._last_lr[-1])\n    \n        if (epoch + 1) % log_epochs == 0:\n            print(\n                f\"Epoch {epoch},\",\n                f\"Average train Loss {average_train_loss_per_batch.item():.4f},\",\n                f\"Average val Loss {average_val_loss_per_batch.item():.4f}\"\n            )\n\n    history_df = (\n        pd.DataFrame.from_dict(history)\n                    .set_index(\"epoch\")\n    )\n\n    return history_df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.924674Z","iopub.execute_input":"2025-12-10T20:15:47.924939Z","iopub.status.idle":"2025-12-10T20:15:47.938717Z","shell.execute_reply.started":"2025-12-10T20:15:47.924917Z","shell.execute_reply":"2025-12-10T20:15:47.937654Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_df = fit(\n    criterion=criterion,\n    model_fn=model_fn,\n    optimizer=optimizer,\n    train_dataloader=train_dataloader,\n    val_dataloader=val_dataloader,\n    device=DEVICE,\n    clip_grad_strategy=\"norm\",  # optional, helps stabilize training\n    max_norm=1.0,               # max gradient norm for clipping\n    lr_scheduler=lr_scheduler,\n    log_epochs=5,                # print progress every 5 epochs\n    max_epochs=100               # train for 100 epochs\n\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:15:47.939713Z","iopub.execute_input":"2025-12-10T20:15:47.940034Z","iopub.status.idle":"2025-12-10T20:18:30.930721Z","shell.execute_reply.started":"2025-12-10T20:15:47.940010Z","shell.execute_reply":"2025-12-10T20:18:30.929405Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Prepare the test dataset (10 pts)\n\nComplete the code in the cell below to prepare the test features and create the test dataloader.","metadata":{}},{"cell_type":"code","source":"test_dataset = features_preprocessing_pipeline.transform(\n    test_features_df\n)\n\n\ntest_dataloader = utils.data.DataLoader(\n    test_dataset,\n    batch_size=BATCH_SIZE,  # same batch size as training\n    shuffle=False           # no need to shuffle test data\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:30.933278Z","iopub.execute_input":"2025-12-10T20:18:30.933667Z","iopub.status.idle":"2025-12-10T20:18:31.145147Z","shell.execute_reply.started":"2025-12-10T20:18:30.933566Z","shell.execute_reply":"2025-12-10T20:18:31.144196Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Generate predictions (10 pts)\n\nUsing the function defined below, complete the code to generate predictions for the test dataset using your trained model.","metadata":{}},{"cell_type":"code","source":"def predict(model_fn, dataloader, device=\"cpu\"):\n    model_fn = model_fn.eval()\n    \n    batch_predictions = []\n    with torch.inference_mode():\n        for X in dataloader:\n            X = X.to(device)\n            logits = model_fn(X)\n\n            # convert logits to probabilities\n            probs = logits.softmax(dim=1)\n            preds = torch.argmax(probs, dim=1)\n            batch_predictions.append(preds.cpu())\n\n    predictions = torch.cat(batch_predictions, dim=0)\n    \n    return predictions.numpy()\n        ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:31.146155Z","iopub.execute_input":"2025-12-10T20:18:31.146396Z","iopub.status.idle":"2025-12-10T20:18:31.152748Z","shell.execute_reply.started":"2025-12-10T20:18:31.146379Z","shell.execute_reply":"2025-12-10T20:18:31.151873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions = predict(\n    model_fn=model_fn,       # trained model\n    dataloader=test_dataloader,\n    device=DEVICE            # CPU or GPU\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:31.153733Z","iopub.execute_input":"2025-12-10T20:18:31.153970Z","iopub.status.idle":"2025-12-10T20:18:32.127864Z","shell.execute_reply.started":"2025-12-10T20:18:31.153952Z","shell.execute_reply":"2025-12-10T20:18:32.126902Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Submit predictions to the competition (5 pts)","metadata":{}},{"cell_type":"code","source":"_ = (\n    pd.read_csv(\n        DATA_DIR / \"sample_submission.csv\",\n        index_col=\"ImageId\",\n    ).assign(\n        Label=test_predictions\n    ).to_csv(\n        OUTPUT_DIR / \"submission.csv\",\n        index=True,\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:32.128962Z","iopub.execute_input":"2025-12-10T20:18:32.129239Z","iopub.status.idle":"2025-12-10T20:18:32.183374Z","shell.execute_reply.started":"2025-12-10T20:18:32.129219Z","shell.execute_reply":"2025-12-10T20:18:32.182106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\n\ncat submission.csv | head -n 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:32.184573Z","iopub.execute_input":"2025-12-10T20:18:32.185311Z","iopub.status.idle":"2025-12-10T20:18:32.199458Z","shell.execute_reply.started":"2025-12-10T20:18:32.185270Z","shell.execute_reply":"2025-12-10T20:18:32.198643Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Unsupervised pretraining (25 pts, Optional)\n\nLeverage the unlabeled dataset created above to perform unsupervised pretraining and assess whether using unsupervised pretraining improves the performance of your model.\n\nPlot the training and validation loss curves for your original model, and plot the training and validation loss curves for your new model that uses unsupervised pretraining. Which model do you prefer and why?\n","metadata":{}},{"cell_type":"code","source":"encoder = pipeline.make_pipeline(\n    preprocessing.StandardScaler(),\n    decomposition.PCA(\n        n_components=0.95,\n        whiten=True,\n    ),\n)\n\n_ = encoder.fit(unlabeled_features_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:32.200354Z","iopub.execute_input":"2025-12-10T20:18:32.200901Z","iopub.status.idle":"2025-12-10T20:18:36.828959Z","shell.execute_reply.started":"2025-12-10T20:18:32.200871Z","shell.execute_reply":"2025-12-10T20:18:36.827930Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoded_train_features_df = encoder.transform(train_features_df)\nencoded_val_features_df = encoder.transform(val_features_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:36.829992Z","iopub.execute_input":"2025-12-10T20:18:36.830963Z","iopub.status.idle":"2025-12-10T20:18:37.008753Z","shell.execute_reply.started":"2025-12-10T20:18:36.830932Z","shell.execute_reply":"2025-12-10T20:18:37.007809Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoded_train_features_tensor = (\n    torch.tensor(encoded_train_features_df, dtype=torch.float32)\n)\n\nencoded_val_features_tensor = (\n    torch.tensor(encoded_val_features_df, dtype=torch.float32)\n)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:37.009837Z","iopub.execute_input":"2025-12-10T20:18:37.010297Z","iopub.status.idle":"2025-12-10T20:18:37.016227Z","shell.execute_reply.started":"2025-12-10T20:18:37.010270Z","shell.execute_reply":"2025-12-10T20:18:37.015463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_encoded_train_dataset = utils.data.TensorDataset(\n    encoded_train_features_tensor, train_target_tensor\n)\n\n_encoded_val_dataset = utils.data.TensorDataset(\n    encoded_val_features_tensor, val_target_tensor\n)\n\nencoded_train_dataloader = utils.data.DataLoader(\n    _encoded_train_dataset,\n    batch_size=BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    persistent_workers=True,\n    shuffle=True,\n)\n\nencoded_val_dataloader = utils.data.DataLoader(\n    _encoded_val_dataset,\n    batch_size=BATCH_SIZE,\n    num_workers=NUM_WORKERS,\n    shuffle=False,\n)","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:37.017235Z","iopub.execute_input":"2025-12-10T20:18:37.017696Z","iopub.status.idle":"2025-12-10T20:18:37.034490Z","shell.execute_reply.started":"2025-12-10T20:18:37.017666Z","shell.execute_reply":"2025-12-10T20:18:37.033529Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"criterion, model_fn = make_mlp_classifier(\n    input_size= encoded_train_features_tensor.shape[1],\n    hidden_sizes=hidden_sizes,\n    output_size=output_size,\n    activation_fn=nn.SELU(),\n    init_strategy_=nn.init.kaiming_normal_,\n    init_strategy_kwargs={\"mode\": \"fan_in\", \"nonlinearity\": \"linear\"},\n)\nmodel_fn = model_fn.to(DEVICE)\n\noptimizer = optim.AdamW(\n    model_fn.parameters(), lr=1e-3, weight_decay=5e-2\n)\n\n\nlr_scheduler = (\n    optim.lr_scheduler\n         .ReduceLROnPlateau(\n                 optimizer,\n                 mode='min',            # monitor validation loss\n                 factor=0.5,            # reduce LR by half on plateau\n                 patience=5,            # wait 5 epochs before reducing\n                 verbose=True\n\n         )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:37.035403Z","iopub.execute_input":"2025-12-10T20:18:37.035674Z","iopub.status.idle":"2025-12-10T20:18:37.067426Z","shell.execute_reply.started":"2025-12-10T20:18:37.035656Z","shell.execute_reply":"2025-12-10T20:18:37.066488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unsupervised_pretraining_history_df = fit(\n    criterion=criterion,\n    model_fn=model_fn,\n    optimizer=optimizer,\n    train_dataloader=encoded_train_dataloader,\n    val_dataloader=encoded_val_dataloader,\n    device=DEVICE,\n    clip_grad_strategy=\"norm\",\n    max_norm=1.0,\n    lr_scheduler=lr_scheduler,\n    log_epochs=5,\n    max_epochs=100\n\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:18:37.068287Z","iopub.execute_input":"2025-12-10T20:18:37.068525Z","iopub.status.idle":"2025-12-10T20:20:59.200934Z","shell.execute_reply.started":"2025-12-10T20:18:37.068507Z","shell.execute_reply":"2025-12-10T20:20:59.199721Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\n\n# --- Step 7: Plot training & validation loss curves ---\nplt.figure(figsize=(10,5))\nplt.plot(unsupervised_pretraining_history_df.index, unsupervised_pretraining_history_df[\"average_train_loss\"], label=\"Train Loss (pretrained)\")\nplt.plot(unsupervised_pretraining_history_df.index, unsupervised_pretraining_history_df[\"average_val_loss\"], label=\"Val Loss (pretrained)\")\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.title(\"Training & Validation Loss with Unsupervised Pretraining\")\nplt.legend()\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:20:59.202890Z","iopub.execute_input":"2025-12-10T20:20:59.203220Z","iopub.status.idle":"2025-12-10T20:20:59.498423Z","shell.execute_reply.started":"2025-12-10T20:20:59.203178Z","shell.execute_reply":"2025-12-10T20:20:59.497703Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\n\n# Original model loss curves\nplt.plot(history_df.index, history_df[\"average_train_loss\"], label=\"Train Loss (original)\", color=\"blue\")\nplt.plot(history_df.index, history_df[\"average_val_loss\"], label=\"Val Loss (original)\", color=\"cyan\")\n\n# Unsupervised-pretrained model loss curves\nplt.plot(unsupervised_pretraining_history_df.index, unsupervised_pretraining_history_df[\"average_train_loss\"], label=\"Train Loss (pretrained)\", color=\"red\")\nplt.plot(unsupervised_pretraining_history_df.index, unsupervised_pretraining_history_df[\"average_val_loss\"], label=\"Val Loss (pretrained)\", color=\"orange\")\n\nplt.xlabel(\"Epoch\")\nplt.ylabel(\"Loss\")\nplt.title(\"Training & Validation Loss: Original vs Unsupervised-Pretrained MLP\")\nplt.legend()\nplt.grid(True)\nplt.show()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-10T20:20:59.499404Z","iopub.execute_input":"2025-12-10T20:20:59.499799Z","iopub.status.idle":"2025-12-10T20:20:59.759009Z","shell.execute_reply.started":"2025-12-10T20:20:59.499771Z","shell.execute_reply":"2025-12-10T20:20:59.757999Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"As we see in the graph, the unsupervised pretraining didn't improve the training and validation, the curve is below the original one.\n\nI prefer the original model as the unsupervised training did not achieve better validation error but actually it is worse. Maybe another approach for the unsupervised learning could help achieve better accuracy.","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}}]}