{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3004,"databundleVersionId":861823,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pathlib\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn import decomposition, model_selection, preprocessing, pipeline\nimport torch\nfrom torch import nn, utils\nimport torchmetrics","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:20.604475Z","iopub.execute_input":"2025-12-07T13:46:20.605079Z","iopub.status.idle":"2025-12-07T13:46:34.782177Z","shell.execute_reply.started":"2025-12-07T13:46:20.605053Z","shell.execute_reply":"2025-12-07T13:46:34.781578Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Notes on the Assignment \nIn Task 5 (Train your model), the provided functions had a small issue when running on GPU. During training, I got a runtime error about tensors being on different devices (`cuda:0` and `cpu`). The problem was that the loss accumulators were initialized on the CPU, while the batch losses were computed on the GPU. To fix this, I made sure to initialize the accumulator `torch.zeros(1, 1, device=device)`.","metadata":{}},{"cell_type":"markdown","source":"## 0. Load the data (5 pts)\n\nComplete the code in the cells below to load the MNIST dataset in to a Pandas `DataFrame` using the `pd.read_csv` function. Be sure to use the `np.uint8` data-type to represent the pixel values as non-negative integers.","metadata":{}},{"cell_type":"code","source":"INPUT_DIR = pathlib.Path(\"/kaggle/input\")\nOUTPUT_DIR = pathlib.Path(\"/kaggle/working\")\nTEMP_DIR = pathlib.Path(\"/kaggle/temp\")\nDATA_DIR = INPUT_DIR / \"digit-recognizer\"\n\nRANDOM_STATE = np.random.RandomState(42)\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:34.783197Z","iopub.execute_input":"2025-12-07T13:46:34.783590Z","iopub.status.idle":"2025-12-07T13:46:34.863224Z","shell.execute_reply.started":"2025-12-07T13:46:34.783570Z","shell.execute_reply":"2025-12-07T13:46:34.862533Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pd.read_csv?","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:45:33.661714Z","iopub.execute_input":"2025-12-07T13:45:33.661925Z","iopub.status.idle":"2025-12-07T13:45:33.890703Z","execution_failed":"2025-12-07T13:45:58.781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"mnist_df = pd.read_csv(\n    DATA_DIR/\"train.csv\",\n    dtype=np.uint8,\n)\n\ntest_features_df = pd.read_csv(\n    DATA_DIR/\"test.csv\",\n    dtype=np.uint8,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:34.864214Z","iopub.execute_input":"2025-12-07T13:46:34.864912Z","iopub.status.idle":"2025-12-07T13:46:38.125119Z","shell.execute_reply.started":"2025-12-07T13:46:34.864886Z","shell.execute_reply":"2025-12-07T13:46:38.124334Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 1. Simulate a partially labeled dataset (5 pts)\n\nComplete the code in the cell below to create a simulated, partially labeled version of the MNIST dataset.\n\nYou must use the `model_selection.train_test_split` function from Scikit-Learn to split the provided training data into \"labeled\" and \"unlabeled\" pieces using stratified sampling. The unlabeled piece should contain 80% of the original training data. ","metadata":{}},{"cell_type":"code","source":"model_selection.train_test_split?","metadata":{"trusted":true,"execution":{"execution_failed":"2025-12-07T13:45:58.782Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"labeled_mnist_df, unlabeled_mnist_df = (\n    model_selection.train_test_split(\n        mnist_df,\n        stratify=mnist_df[\"label\"],\n        test_size=0.8,\n        random_state=RANDOM_STATE\n    )\n)\n\n# we drop the label to simulate an unlabeled subset of data\nunlabeled_features_df = unlabeled_mnist_df.drop(\"label\", axis=1)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:49.911305Z","iopub.execute_input":"2025-12-07T13:46:49.912024Z","iopub.status.idle":"2025-12-07T13:46:50.072686Z","shell.execute_reply.started":"2025-12-07T13:46:49.911994Z","shell.execute_reply":"2025-12-07T13:46:50.071885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 8400 rows and 785 columns!\nlabeled_mnist_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:53.311124Z","iopub.execute_input":"2025-12-07T13:46:53.311927Z","iopub.status.idle":"2025-12-07T13:46:53.340513Z","shell.execute_reply.started":"2025-12-07T13:46:53.311898Z","shell.execute_reply":"2025-12-07T13:46:53.339885Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 33600 rows and 784 cols!\nunlabeled_features_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:53.551446Z","iopub.execute_input":"2025-12-07T13:46:53.551941Z","iopub.status.idle":"2025-12-07T13:46:53.574436Z","shell.execute_reply.started":"2025-12-07T13:46:53.551919Z","shell.execute_reply":"2025-12-07T13:46:53.573596Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 2. Create a train/val split for the labeled data (5 pts)\n\nComplete the code in the cell below to split your labeled training data into training and validation sets using stratified sampling. The validation set should contain 10% of the labeled training data. ","metadata":{}},{"cell_type":"code","source":"train_df, val_df = (\n    model_selection.train_test_split(\n        labeled_mnist_df,\n        stratify=labeled_mnist_df[\"label\"],\n        test_size=0.1,\n        random_state=RANDOM_STATE,\n    )\n)\n\ntrain_features_df = train_df.drop(\"label\", axis=1)\ntrain_labels = train_df.loc[:, \"label\"]\n\nval_features_df = val_df.drop(\"label\", axis=1)\nval_labels = val_df.loc[:, \"label\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:56.840197Z","iopub.execute_input":"2025-12-07T13:46:56.840922Z","iopub.status.idle":"2025-12-07T13:46:56.865955Z","shell.execute_reply.started":"2025-12-07T13:46:56.840898Z","shell.execute_reply":"2025-12-07T13:46:56.865371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 7560 rows and 784 cols!\ntrain_features_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:58.598378Z","iopub.execute_input":"2025-12-07T13:46:58.599016Z","iopub.status.idle":"2025-12-07T13:46:58.623244Z","shell.execute_reply.started":"2025-12-07T13:46:58.598989Z","shell.execute_reply":"2025-12-07T13:46:58.622636Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should have 840 rows and 784 cols!\nval_features_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:46:58.823120Z","iopub.execute_input":"2025-12-07T13:46:58.823652Z","iopub.status.idle":"2025-12-07T13:46:58.847683Z","shell.execute_reply.started":"2025-12-07T13:46:58.823633Z","shell.execute_reply":"2025-12-07T13:46:58.846903Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 3. Prepare the datasets (20 pts)","metadata":{}},{"cell_type":"markdown","source":"### Converting DataFrames to Tensors (10 pts)\n\nComplete the code in the cell below in order to define a Scikit Learn preprocessing pipeline for your features that has the following steps.\n\n1. Uses a `StandardScaler` to standardize the features.\n2. Uses a `FunctionTransformer` to convert a Numpy Array to a Torch Tensor with a `torch.float32` dtype.\n\nNext, define a Scikit Learn preprocessing pipeline for your target that has the following steps.\n\n1. Uses a `FunctionTransformer` to convert a Pandas Series to a Numpy Array.\n2. Uses a `FunctionTransformer` to convert a NumPy Array to a Torch Tensor with a `torch.int64` dtype.\n\n\n\n","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler, FunctionTransformer\nfrom sklearn.pipeline import Pipeline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:02.075904Z","iopub.execute_input":"2025-12-07T13:47:02.076179Z","iopub.status.idle":"2025-12-07T13:47:02.080109Z","shell.execute_reply.started":"2025-12-07T13:47:02.076159Z","shell.execute_reply":"2025-12-07T13:47:02.079370Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"features_preprocessing_pipeline = pipeline.make_pipeline(\n    preprocessing.StandardScaler(),\n    preprocessing.FunctionTransformer(\n        lambda X: torch.tensor(X, dtype=torch.float32),\n        validate=False,\n    ),\n)\n\ntarget_preprocessing_pipeline = pipeline.make_pipeline(\n    preprocessing.FunctionTransformer(\n        lambda y: y.to_numpy(),\n        validate=False,\n    ),\n    preprocessing.FunctionTransformer(\n        lambda y: torch.tensor(y, dtype=torch.int64),\n        validate=False,\n    ),\n)\n\ntrain_features_tensor = (\n    features_preprocessing_pipeline.fit_transform(\n        train_features_df\n    )\n)\n\ntrain_target_tensor = (\n    target_preprocessing_pipeline.fit_transform(\n        train_labels\n    )\n)\n\nval_features_tensor = (\n    features_preprocessing_pipeline.transform(\n        val_features_df\n    )\n)\n\nval_target_tensor = (\n    target_preprocessing_pipeline.transform(\n        val_labels\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:02.393075Z","iopub.execute_input":"2025-12-07T13:47:02.393341Z","iopub.status.idle":"2025-12-07T13:47:02.557831Z","shell.execute_reply.started":"2025-12-07T13:47:02.393322Z","shell.execute_reply":"2025-12-07T13:47:02.556908Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 7560 by 784!\nprint(train_features_tensor.shape)\n\n# should be float32!\nprint(train_features_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:05.793044Z","iopub.execute_input":"2025-12-07T13:47:05.793325Z","iopub.status.idle":"2025-12-07T13:47:05.798050Z","shell.execute_reply.started":"2025-12-07T13:47:05.793304Z","shell.execute_reply":"2025-12-07T13:47:05.797311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 7560!\nprint(train_target_tensor.shape)\n\n# should be int64!\nprint(train_target_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:06.168872Z","iopub.execute_input":"2025-12-07T13:47:06.169124Z","iopub.status.idle":"2025-12-07T13:47:06.173126Z","shell.execute_reply.started":"2025-12-07T13:47:06.169108Z","shell.execute_reply":"2025-12-07T13:47:06.172429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 840 by 784!\nprint(val_features_tensor.shape)\n\n# should be float32!\nprint(val_features_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:06.416863Z","iopub.execute_input":"2025-12-07T13:47:06.417283Z","iopub.status.idle":"2025-12-07T13:47:06.421344Z","shell.execute_reply.started":"2025-12-07T13:47:06.417263Z","shell.execute_reply":"2025-12-07T13:47:06.420630Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# should be 840!\nprint(val_target_tensor.shape)\n\n# should be int64!\nprint(val_target_tensor.dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:06.693448Z","iopub.execute_input":"2025-12-07T13:47:06.693692Z","iopub.status.idle":"2025-12-07T13:47:06.697666Z","shell.execute_reply.started":"2025-12-07T13:47:06.693673Z","shell.execute_reply":"2025-12-07T13:47:06.696990Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create TensorDatasets (5 pts)\n\nComplete the code in the cell below to create `utils.data.TensorDatasets` from the training and validation tensors.","metadata":{}},{"cell_type":"code","source":"train_dataset = utils.data.TensorDataset(\n    train_features_tensor,\n    train_target_tensor\n)\n\nval_dataset = utils.data.TensorDataset(\n    val_features_tensor,\n    val_target_tensor\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:09.212481Z","iopub.execute_input":"2025-12-07T13:47:09.212746Z","iopub.status.idle":"2025-12-07T13:47:09.218332Z","shell.execute_reply.started":"2025-12-07T13:47:09.212727Z","shell.execute_reply":"2025-12-07T13:47:09.217565Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Create DataLoaders (5 pts)\n\nComplete the code in the cell below to create `utils.data.DataLoaders` for the training and validation datasets.\n\n","metadata":{}},{"cell_type":"code","source":"train_dataloader = utils.data.DataLoader(\n    train_dataset,\n    batch_size=128,\n    shuffle=True\n)\n\n\nval_dataloader = utils.data.DataLoader(\n    val_dataset,\n    batch_size=128,\n    shuffle=False\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:11.173032Z","iopub.execute_input":"2025-12-07T13:47:11.173308Z","iopub.status.idle":"2025-12-07T13:47:11.177574Z","shell.execute_reply.started":"2025-12-07T13:47:11.173287Z","shell.execute_reply":"2025-12-07T13:47:11.176957Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 4. Define a model, optimizer, and LR scheduler (20 pts)\n\nUsing only the functions provided below define a self-normalizing MLP with three hidden layers. Configure the `optim.AdamW` optimizer, and configure the `optim.lr_scheduler.ReduceLROnPlateau` learning rate scheduler.","metadata":{}},{"cell_type":"code","source":"import torch.optim as optim","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:12.007164Z","iopub.execute_input":"2025-12-07T13:47:12.007404Z","iopub.status.idle":"2025-12-07T13:47:12.011112Z","shell.execute_reply.started":"2025-12-07T13:47:12.007388Z","shell.execute_reply":"2025-12-07T13:47:12.010354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def initialize_linear_layer(\n    in_features,\n    out_features,\n    init_strategy_=nn.init.kaiming_uniform_,\n    init_strategy_kwargs=None,\n    ):\n    linear_layer = nn.Linear(in_features, out_features)\n\n    if init_strategy_ is not None:\n        if init_strategy_kwargs is None:\n            init_strategy_kwargs = {}\n        init_strategy_(linear_layer.weight, **init_strategy_kwargs)\n        linear_layer.bias.data.fill_(0.0)\n\n    return linear_layer\n\n\ndef make_mlp_classifier(\n    input_size,\n    hidden_sizes=None,\n    output_size=2,\n    activation_fn=None,\n    init_strategy_=nn.init.kaiming_uniform_,\n    init_strategy_kwargs=None,\n    batch_normalization=False\n    ):\n    \n    modules = []\n    hidden_sizes = [] if hidden_sizes is None else hidden_sizes\n    for hidden_size in hidden_sizes:\n        hidden_layer = initialize_linear_layer(\n            input_size,\n            hidden_size,\n            init_strategy_,\n            init_strategy_kwargs,\n        )\n        modules.append(hidden_layer)\n\n        # batch normalization goes after the linear layer...\n        if batch_normalization:\n            modules.append(nn.BatchNorm1d(hidden_size))\n\n        # ...but before the activation_fn!\n        if activation_fn is not None:\n            modules.append(activation_fn)\n        input_size=hidden_size\n\n    output_layer = initialize_linear_layer(\n            input_size,\n            output_size,\n            init_strategy_,\n            init_strategy_kwargs,\n    )\n    modules.append(output_layer)\n    model_fn = nn.Sequential(*modules)\n    return nn.CrossEntropyLoss(), model_fn","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:13.487103Z","iopub.execute_input":"2025-12-07T13:47:13.487598Z","iopub.status.idle":"2025-12-07T13:47:13.494070Z","shell.execute_reply.started":"2025-12-07T13:47:13.487573Z","shell.execute_reply":"2025-12-07T13:47:13.493276Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"input_size = train_features_tensor.shape[1]\noutput_size = int(train_target_tensor.max().item() + 1)\nhidden_size = int((2 / 3) * (input_size + output_size))\nhidden_sizes = [hidden_size, hidden_size, hidden_size]\n\ncriterion, model_fn = make_mlp_classifier(\n    input_size,\n    hidden_sizes,\n    output_size,\n    activation_fn=nn.SELU(),\n    init_strategy_=nn.init.kaiming_normal_,\n    init_strategy_kwargs={\"nonlinearity\": \"linear\"}\n)\nmodel_fn = model_fn.to(DEVICE)\n\noptimizer = optim.AdamW(\n    model_fn.parameters(),\n    lr=1e-3,\n    weight_decay=1e-2\n)\n\nlr_scheduler = (\n    optim.lr_scheduler\n         .ReduceLROnPlateau(\n             optimizer,\n             mode=\"min\",\n             factor=0.5,\n             patience=5,\n             verbose=True,\n         )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:47:16.623219Z","iopub.execute_input":"2025-12-07T13:47:16.623792Z","iopub.status.idle":"2025-12-07T13:47:16.819979Z","shell.execute_reply.started":"2025-12-07T13:47:16.623767Z","shell.execute_reply":"2025-12-07T13:47:16.819174Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 5. Train your model (20 pts)\n\nUsing the training loop provided below, train your MLP for 100 epochs and plot the training and validation curves as well as the learning rate.","metadata":{}},{"cell_type":"code","source":"def compute_average_loss_per_batch(dataloader, criterion, model_fn):\n    device = next(model_fn.parameters()).device    \n    total_loss = torch.zeros(1, 1, device=device)\n    num_batches = len(dataloader)\n    for features, targets in dataloader:\n\n        features, targets = features.to(device), targets.to(device)\n        \n        predictions = model_fn(features)\n        batch_loss = criterion(predictions, targets)\n        total_loss += batch_loss\n    average_loss_per_batch = total_loss / num_batches\n    return average_loss_per_batch\n\n\ndef clip_gradients_(\n    clip_grad_strategy,\n    model_fn,\n    clip_value=None,\n    error_if_nonfinite=False,\n    max_norm=None,\n    norm_type=2.0):\n    if clip_grad_strategy == \"value\" and clip_value is not None:\n        nn.utils.clip_grad_value_(\n            model_fn.parameters(),\n            clip_value\n        )\n    elif clip_grad_strategy == \"norm\" and max_norm is not None:\n        nn.utils.clip_grad_norm_(\n            model_fn.parameters(),\n            max_norm,\n            norm_type,\n            error_if_nonfinite\n        )\n    else:\n        raise NotImplementedError()\n\n\ndef evaluate(model_fn, dataloader, metric, device=\"cpu\"):\n    model_fn.eval()\n    metric.reset()\n    with torch.inference_mode():\n        for X_batch, y_batch in dataloader:\n            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n            y_pred = model_fn(X_batch)\n            metric.update(y_pred, y_batch)\n    return metric.compute()\n\n\ndef fit(\n    criterion,\n    model_fn,\n    optimizer,\n    train_dataloader,\n    val_dataloader,\n    device=\"cpu\",\n    clip_grad_strategy=None,\n    clip_value=None,\n    error_if_nonfinite=False,\n    lr_scheduler=None,\n    log_epochs=1,\n    max_epochs=1,\n    max_norm=None,\n    norm_type=2.0\n    ):\n\n    history = {\n        \"epoch\": [],\n        \"average_train_loss\": [],\n        \"average_val_loss\": [],\n        \"lr\": [],\n    }\n    num_train_batches = len(train_dataloader)\n    for epoch in range(max_epochs):\n        total_train_loss = torch.zeros(1, 1, device=device)\n        model_fn = model_fn.train()\n        for features, targets in train_dataloader:\n\n            # forward pass\n            features, targets = features.to(device), targets.to(device)\n            predictions = model_fn(features)\n            batch_loss = criterion(predictions, targets)\n            total_train_loss += batch_loss\n\n            # backward pass\n            optimizer.zero_grad()\n            batch_loss.backward()\n            if clip_grad_strategy is not None:\n                clip_gradients_(\n                    clip_grad_strategy,\n                    model_fn,\n                    clip_value,\n                    error_if_nonfinite,\n                    max_norm,\n                    norm_type\n                )\n            optimizer.step()\n\n        history[\"epoch\"].append(epoch)\n\n        average_train_loss_per_batch = total_train_loss / num_train_batches\n        history[\"average_train_loss\"].append(average_train_loss_per_batch.item())\n\n        model_fn = model_fn.eval()\n        with torch.inference_mode():\n            average_val_loss_per_batch = compute_average_loss_per_batch(\n                val_dataloader,\n                criterion,\n                model_fn\n            )\n        history[\"average_val_loss\"].append(average_val_loss_per_batch.item())\n\n        # update the learning rate after every training epoch\n        if lr_scheduler is not None:\n            lr_scheduler.step(average_val_loss_per_batch.item())\n            history[\"lr\"].append(lr_scheduler._last_lr[-1])\n    \n        if (epoch + 1) % log_epochs == 0:\n            print(\n                f\"Epoch {epoch},\",\n                f\"Average train Loss {average_train_loss_per_batch.item():.4f},\",\n                f\"Average val Loss {average_val_loss_per_batch.item():.4f}\"\n            )\n\n    history_df = (\n        pd.DataFrame.from_dict(history)\n                    .set_index(\"epoch\")\n    )\n\n    return history_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:58:43.197338Z","iopub.execute_input":"2025-12-07T13:58:43.197971Z","iopub.status.idle":"2025-12-07T13:58:43.208954Z","shell.execute_reply.started":"2025-12-07T13:58:43.197945Z","shell.execute_reply":"2025-12-07T13:58:43.208147Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"history_df = fit(\n    criterion=criterion,\n    model_fn=model_fn,\n    optimizer=optimizer,\n    train_dataloader=train_dataloader,\n    val_dataloader=val_dataloader,\n    device=DEVICE,\n    clip_grad_strategy=None,   \n    lr_scheduler=lr_scheduler,\n    log_epochs=1,              \n    max_epochs=100,            \n    max_norm=None,             \n    norm_type=2.0\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T13:58:43.606561Z","iopub.execute_input":"2025-12-07T13:58:43.607031Z","iopub.status.idle":"2025-12-07T13:59:02.571489Z","shell.execute_reply.started":"2025-12-07T13:58:43.607005Z","shell.execute_reply":"2025-12-07T13:59:02.570641Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 6. Prepare the test dataset (10 pts)\n\nComplete the code in the cell below to prepare the test features and create the test dataloader.","metadata":{}},{"cell_type":"code","source":"test_dataset = features_preprocessing_pipeline.transform(\n    test_features_df\n)\n\ntest_dataloader = utils.data.DataLoader(\n    test_dataset,\n    batch_size=128,\n    shuffle=False\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:03:29.097072Z","iopub.execute_input":"2025-12-07T14:03:29.097629Z","iopub.status.idle":"2025-12-07T14:03:29.261619Z","shell.execute_reply.started":"2025-12-07T14:03:29.097606Z","shell.execute_reply":"2025-12-07T14:03:29.260808Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 7. Generate predictions (10 pts)\n\nUsing the function defined below, complete the code to generate predictions for the test dataset using your trained model.","metadata":{}},{"cell_type":"code","source":"def predict(model_fn, dataloader, device=\"cpu\"):\n    model_fn = model_fn.eval()\n    \n    batch_predictions = []\n    with torch.inference_mode():\n        for X in dataloader:\n            X = X.to(device)\n            logits = model_fn(X)\n\n            # convert logits to probabilities\n            probs = logits.softmax(dim=1)\n            preds = torch.argmax(probs, dim=1)\n            batch_predictions.append(preds.cpu())\n\n    predictions = torch.cat(batch_predictions, dim=0)\n    \n    return predictions.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:05:38.451678Z","iopub.execute_input":"2025-12-07T14:05:38.452205Z","iopub.status.idle":"2025-12-07T14:05:38.456903Z","shell.execute_reply.started":"2025-12-07T14:05:38.452180Z","shell.execute_reply":"2025-12-07T14:05:38.456259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_predictions = predict(\n    model_fn=model_fn,\n    dataloader=test_dataloader,\n    device=DEVICE\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:06:39.088234Z","iopub.execute_input":"2025-12-07T14:06:39.089012Z","iopub.status.idle":"2025-12-07T14:06:39.373049Z","shell.execute_reply.started":"2025-12-07T14:06:39.088983Z","shell.execute_reply":"2025-12-07T14:06:39.372402Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 8. Submit predictions to the competition (5 pts)","metadata":{}},{"cell_type":"code","source":"_ = (\n    pd.read_csv(\n        DATA_DIR / \"sample_submission.csv\",\n        index_col=\"ImageId\",\n    ).assign(\n        Label=test_predictions\n    ).to_csv(\n        OUTPUT_DIR / \"submission.csv\",\n        index=True,\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:18:43.497581Z","iopub.execute_input":"2025-12-07T14:18:43.497997Z","iopub.status.idle":"2025-12-07T14:18:43.559454Z","shell.execute_reply.started":"2025-12-07T14:18:43.497970Z","shell.execute_reply":"2025-12-07T14:18:43.558666Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%bash\n\ncat submission.csv | head -n 5","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:18:44.129919Z","iopub.execute_input":"2025-12-07T14:18:44.130188Z","iopub.status.idle":"2025-12-07T14:18:44.142468Z","shell.execute_reply.started":"2025-12-07T14:18:44.130166Z","shell.execute_reply":"2025-12-07T14:18:44.141725Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## 9. Unsupervised pretraining (25 pts, Optional)\n\nLeverage the unlabeled dataset created above to perform unsupervised pretraining and assess whether using unsupervised pretraining improves the performance of your model.\n\nPlot the training and validation loss curves for your original model, and plot the training and validation loss curves for your new model that uses unsupervised pretraining. Which model do you prefer and why?","metadata":{}},{"cell_type":"code","source":"encoder = decomposition.PCA(\n    n_components=64,          \n    random_state=RANDOM_STATE\n)\n\n_ = encoder.fit(\n    pd.concat(\n        [train_features_df, unlabeled_features_df],\n        axis=0\n    )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:11:09.325620Z","iopub.execute_input":"2025-12-07T14:11:09.325947Z","iopub.status.idle":"2025-12-07T14:11:12.272435Z","shell.execute_reply.started":"2025-12-07T14:11:09.325920Z","shell.execute_reply":"2025-12-07T14:11:12.271772Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoded_train_features = encoder.transform(train_features_df)\nencoded_val_features = encoder.transform(val_features_df)\n\nencoded_train_features_df = pd.DataFrame(\n    encoded_train_features,\n    index=train_features_df.index\n)\n\nencoded_val_features_df = pd.DataFrame(\n    encoded_val_features,\n    index=val_features_df.index\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:12:41.621023Z","iopub.execute_input":"2025-12-07T14:12:41.621642Z","iopub.status.idle":"2025-12-07T14:12:41.700400Z","shell.execute_reply.started":"2025-12-07T14:12:41.621617Z","shell.execute_reply":"2025-12-07T14:12:41.699741Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoded_train_features_tensor = torch.tensor(\n    encoded_train_features_df.to_numpy(),\n    dtype=torch.float32,\n)\n\nencoded_val_features_tensor = torch.tensor(\n    encoded_val_features_df.to_numpy(),\n    dtype=torch.float32,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:13:27.313870Z","iopub.execute_input":"2025-12-07T14:13:27.314154Z","iopub.status.idle":"2025-12-07T14:13:27.319206Z","shell.execute_reply.started":"2025-12-07T14:13:27.314134Z","shell.execute_reply":"2025-12-07T14:13:27.318524Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"_encoded_train_dataset = utils.data.TensorDataset(\n    encoded_train_features_tensor,\n    train_target_tensor\n)\n\n_encoded_val_dataset = utils.data.TensorDataset(\n    encoded_val_features_tensor,\n    val_target_tensor\n)\n\nencoded_train_dataloader = utils.data.DataLoader(\n    _encoded_train_dataset,\n    batch_size=128,\n    shuffle=True\n)\n\nencoded_val_dataloader = utils.data.DataLoader(\n    _encoded_val_dataset,\n    batch_size=128,\n    shuffle=False\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:14:05.329013Z","iopub.execute_input":"2025-12-07T14:14:05.329280Z","iopub.status.idle":"2025-12-07T14:14:05.334044Z","shell.execute_reply.started":"2025-12-07T14:14:05.329261Z","shell.execute_reply":"2025-12-07T14:14:05.333197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Specifying sizes\nencoded_input_size = encoded_train_features_tensor.shape[1]\nencoded_output_size = int(train_target_tensor.max().item() + 1)\n\nencoded_hidden_size = int((2 / 3) * (encoded_input_size + encoded_output_size))\nencoded_hidden_sizes = [\n    encoded_hidden_size,\n    encoded_hidden_size,\n    encoded_hidden_size,\n]\n\ncriterion, model_fn = make_mlp_classifier(\n    input_size=encoded_input_size,\n    hidden_sizes=encoded_hidden_sizes,\n    output_size=encoded_output_size,\n    activation_fn=nn.SELU(),\n    init_strategy_=nn.init.kaiming_normal_,\n    init_strategy_kwargs={\"nonlinearity\": \"linear\"},\n)\nmodel_fn = model_fn.to(DEVICE)\n\noptimizer = optim.AdamW(\n    model_fn.parameters(),\n    lr=1e-3,\n    weight_decay=1e-2,\n)\n\n# Added part\nlr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(\n    optimizer,\n    mode=\"min\",\n    factor=0.5,\n    patience=5,\n    verbose=True,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:16:10.784689Z","iopub.execute_input":"2025-12-07T14:16:10.785265Z","iopub.status.idle":"2025-12-07T14:16:10.793294Z","shell.execute_reply.started":"2025-12-07T14:16:10.785227Z","shell.execute_reply":"2025-12-07T14:16:10.792539Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"unsupervised_pretraining_history_df = fit(\n    criterion=criterion,\n    model_fn=model_fn,\n    optimizer=optimizer,\n    train_dataloader=encoded_train_dataloader,\n    val_dataloader=encoded_val_dataloader,\n    device=DEVICE,\n    clip_grad_strategy=None,\n    lr_scheduler=lr_scheduler,\n    log_epochs=1,\n    max_epochs=100,\n    max_norm=None,\n    norm_type=2.0\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-07T14:16:23.271114Z","iopub.execute_input":"2025-12-07T14:16:23.271660Z","iopub.status.idle":"2025-12-07T14:16:38.875128Z","shell.execute_reply.started":"2025-12-07T14:16:23.271636Z","shell.execute_reply":"2025-12-07T14:16:38.874052Z"}},"outputs":[],"execution_count":null}]}