{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"@webio":{"lastCommId":null,"lastKernelId":null},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3175,"databundleVersionId":44352,"sourceType":"competition"},{"sourceId":10805162,"sourceType":"datasetVersion","datasetId":6706770}],"dockerImageVersionId":30887,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"id":"7d32b7f4","cell_type":"markdown","source":"# ConvNeXt transfer learning on the Galaxy Zoo dataset\n\n### Author: SoulOfSword","metadata":{}},{"id":"02604d36-dd29-4a9b-b382-febc9d0003f6","cell_type":"markdown","source":"## Import Packages","metadata":{}},{"id":"c6ab7359","cell_type":"code","source":"import warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import figure, show\nimport os\nimport time\nimport glob\nimport zipfile\nimport requests\nimport shutil\nfrom sklearn.model_selection import train_test_split\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torchvision\nfrom torchvision import transforms\nimport torchvision.models as models\nfrom torch.utils.data import DataLoader, Dataset, random_split\nfrom PIL import Image\nfrom sklearn.metrics import accuracy_score, mean_squared_error\n\n%matplotlib inline","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:02:06.170056Z","iopub.execute_input":"2025-02-22T11:02:06.170419Z","iopub.status.idle":"2025-02-22T11:02:14.802308Z","shell.execute_reply.started":"2025-02-22T11:02:06.170383Z","shell.execute_reply":"2025-02-22T11:02:14.801438Z"}},"outputs":[],"execution_count":null},{"id":"4a48e695-2b55-4080-a2ef-2e91723239ea","cell_type":"markdown","source":"## Get the data nicely","metadata":{}},{"id":"52453d1e-ad67-4b87-ae53-c2578375b026","cell_type":"code","source":"input_dir = \"/kaggle/input/galaxy-zoo-the-galaxy-challenge\"\noutput_dir = \"/kaggle/working/data\"\n\nzip_files = glob.glob(input_dir + \"/*.zip\")\n\nfor zip_file in zip_files:\n    with zipfile.ZipFile(zip_file, 'r') as z:\n        z.extractall(output_dir)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:02:14.803475Z","iopub.execute_input":"2025-02-22T11:02:14.803837Z","iopub.status.idle":"2025-02-22T11:02:57.939830Z","shell.execute_reply.started":"2025-02-22T11:02:14.803815Z","shell.execute_reply":"2025-02-22T11:02:57.938744Z"}},"outputs":[],"execution_count":null},{"id":"17f45c53-9eb5-44c2-8376-69e0c3de240b","cell_type":"code","source":"# create new subdirectories for images (if they don't exist -- not sure why sometimes Kaggle removes them and sometimes not)\ntrain_dir = os.path.join(output_dir, 'images_training_rev1')\ntest_dir = os.path.join(output_dir, 'images_test_rev1')\nos.makedirs(train_dir, exist_ok=True)\nos.makedirs(test_dir, exist_ok=True)\n\n# Dfine image file extensions to check against (will be only jpg here but hey never hurts to have some redundancy)\nimage_extensions = {'.jpg', '.jpeg', '.png', '.bmp', '.gif'}\n\n# Iterate over files in the base directory\nfor filename in os.listdir(output_dir):\n    file_path = os.path.join(output_dir, filename)\n    # skip directories\n    if os.path.isfile(file_path):\n        # Check if its a CSV file. If so, leave it \n        if filename.lower().endswith('.csv'):\n            continue\n\n        # get file extension\n        ext = os.path.splitext(filename)[1].lower()\n\n        # if the file is an image, decide where to move it based on filename\n        if ext in image_extensions:\n            if 'train' in filename.lower():\n                destination = os.path.join(train_dir, filename)\n            elif 'test' in filename.lower():\n                destination = os.path.join(test_dir, filename)\n            else:\n                # If no clear indicator print a message to skip the file\n                print(f\"Skipping {filename}: no train/test indicator found.\")\n                continue\n            # Move the file\n            shutil.move(file_path, destination)\n            print(f\"Moved {filename} to {destination}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:02:57.942197Z","iopub.execute_input":"2025-02-22T11:02:57.942450Z","iopub.status.idle":"2025-02-22T11:02:57.949912Z","shell.execute_reply.started":"2025-02-22T11:02:57.942427Z","shell.execute_reply":"2025-02-22T11:02:57.949328Z"}},"outputs":[],"execution_count":null},{"id":"6167e864-148d-4c12-af95-ed8242b5f374","cell_type":"markdown","source":"### Optional: I made a python executable, load it here","metadata":{}},{"id":"dc3c9a80-f9be-4f02-8875-c7753da422c9","cell_type":"markdown","source":"!cp /kaggle/input/convnext-galaxyzoo-python-executable/galaxyzoo_multinode.py /kaggle/working/galaxyzoo_multinode.py","metadata":{"execution":{"iopub.status.busy":"2025-02-20T16:11:21.607938Z","iopub.execute_input":"2025-02-20T16:11:21.608278Z","iopub.status.idle":"2025-02-20T16:11:21.756350Z","shell.execute_reply.started":"2025-02-20T16:11:21.608244Z","shell.execute_reply":"2025-02-20T16:11:21.755127Z"}}},{"id":"8fdd4c93","cell_type":"code","source":"output_dir = \"/kaggle/working/data/\"\ntrain_solutions = pd.read_csv(output_dir+'training_solutions_rev1.csv')\ntrain_solutions.head()","metadata":{"scrolled":true,"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:02:57.951269Z","iopub.execute_input":"2025-02-22T11:02:57.951530Z","iopub.status.idle":"2025-02-22T11:02:59.298023Z","shell.execute_reply.started":"2025-02-22T11:02:57.951496Z","shell.execute_reply":"2025-02-22T11:02:59.297072Z"}},"outputs":[],"execution_count":null},{"id":"d57259e1-4392-4da9-ac55-58e172112081","cell_type":"markdown","source":"## Messing around with the model","metadata":{}},{"id":"402a282c","cell_type":"code","source":"convnext = models.convnext_tiny(pretrained=True)\nnum_classes = len(train_solutions.columns)-1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:02:59.298903Z","iopub.execute_input":"2025-02-22T11:02:59.299237Z","iopub.status.idle":"2025-02-22T11:03:00.820642Z","shell.execute_reply.started":"2025-02-22T11:02:59.299207Z","shell.execute_reply":"2025-02-22T11:03:00.819684Z"}},"outputs":[],"execution_count":null},{"id":"36b9b8ae","cell_type":"code","source":"convnext.classifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:03:00.821425Z","iopub.execute_input":"2025-02-22T11:03:00.821645Z","iopub.status.idle":"2025-02-22T11:03:00.825600Z","shell.execute_reply.started":"2025-02-22T11:03:00.821625Z","shell.execute_reply":"2025-02-22T11:03:00.824821Z"}},"outputs":[],"execution_count":null},{"id":"5b59d3d5","cell_type":"code","source":"# freeze all parameters\nfor param in convnext.parameters():\n    param.requires_grad = False\n\n# modify final layer\nin_features = convnext.classifier[2].in_features\nconvnext.classifier[2] = nn.Linear(in_features, num_classes)\n\n# unfreeze only the final layer parameters\nfor param in convnext.classifier.parameters():\n    param.requires_grad = True\n\n# uncomment to confirm only final layer is trainable\n# for name, param in convnext.named_parameters():\n#     print(name, param.requires_grad)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:03:00.826362Z","iopub.execute_input":"2025-02-22T11:03:00.826748Z","iopub.status.idle":"2025-02-22T11:03:00.857769Z","shell.execute_reply.started":"2025-02-22T11:03:00.826726Z","shell.execute_reply":"2025-02-22T11:03:00.856881Z"}},"outputs":[],"execution_count":null},{"id":"7108ca85-6747-44df-adea-49af9415a5ef","cell_type":"markdown","source":"# TRAINING!! (and validation)","metadata":{}},{"id":"70868542","cell_type":"code","source":"convnext = models.convnext_small(pretrained=True)\nnum_classes = len(train_solutions.columns)-1\n\n# Freeze all parameters\nfor param in convnext.parameters():\n    param.requires_grad = False\n\n# Replace the final layer\nin_features = convnext.classifier[2].in_features\nconvnext.classifier[2] = nn.Linear(in_features, num_classes)\n\n# Unfreeze only the classifier's parameters\nfor param in convnext.classifier.parameters():\n    param.requires_grad = True\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nconvnext = convnext.to(device)\n\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406],\n                         std=[0.229, 0.224, 0.225]) # means and stds of pretrained ImageNet model\n])\n\nclass GalaxyZooTrainDataset(Dataset):\n    def __init__(self, csv_file, image_dir, transform=None):\n        self.df = pd.read_csv(csv_file)\n        self.image_dir = image_dir\n        self.transform = transform\n        #self.length = len(df)\n\n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        # first column is 'GalaxyId'\n        galaxy_id = int(row.iloc[0])\n        image_path = os.path.join(self.image_dir, f\"{galaxy_id}.jpg\")\n        image = Image.open(image_path).convert('RGB')# /255\n        if self.transform:\n            image = self.transform(image)\n        # get labels from the remaining columns\n        labels = row.iloc[1:].values.astype(np.float32)\n        return image, labels\n\ntrain_dataset_full = GalaxyZooTrainDataset(\n    csv_file=output_dir+'training_solutions_rev1.csv',\n    image_dir=output_dir+'images_training_rev1',\n    transform=transform\n) # Create the full training dataset\n\n# make a validation set to.. well, validate\ntotal = len(train_dataset_full)\ntrain_size = int(0.8 * total)\nval_size = int(0.2 * total)\nunused_size = total - (train_size + val_size)\n\n# random_split to partition the dataset into three disjoint subsets\ntrain_subset, val_subset, _ = random_split(train_dataset_full, [train_size, val_size, unused_size])\n\n# creates DataLoaders\ntrain_loader = DataLoader(train_subset, batch_size=256, shuffle=True, num_workers=8, pin_memory=True)\nval_loader = DataLoader(val_subset, batch_size=256, shuffle=False, num_workers=8, pin_memory=True)\n\ncriterion = nn.BCEWithLogitsLoss() # tried KLDivLoss first, realized the classes dont all sum to 1.0. fixed\noptimizer = torch.optim.Adam(convnext.parameters(), lr=1e-4)\n\nnum_epochs = 30\n\ntrain_losses = []\nval_losses = []\n\nfor epoch in range(num_epochs):\n    # training loop\n    convnext.train()\n    running_loss = 0.0\n    last_print_time = time.time()\n    for batch_idx, (images, labels) in enumerate(train_loader):\n        images = images.to(device)\n        labels = labels.to(device)\n        \n        optimizer.zero_grad()\n        outputs = convnext(images)\n        #log_probs = F.log_softmax(outputs, dim=1) # this was needed for KLDivLoss, not anymore. if needed, change loss to criterion(log_probs, labels)\n        loss = criterion(outputs, labels)\n        loss.backward()\n        optimizer.step()\n        \n        running_loss += loss.item() * images.size(0)\n        if batch_idx % 50 == 0:\n            current_time = time.time()\n            elapsed = current_time - last_print_time\n            print(f\"\\n[{elapsed:.1f}s elapsed] Epoch [{epoch+1}/{num_epochs}], Batch [{batch_idx}/{len(train_loader)}], Loss: {loss.item():.4f}\\n\")\n            last_print_time = current_time\n    \n    avg_train_loss = running_loss / len(train_loader.dataset)\n    train_losses.append(avg_train_loss)\n    print(f\"Epoch [{epoch+1}/{num_epochs}] Training Loss: {avg_train_loss:.4f}\")\n    \n    # validation loop\n    convnext.eval()\n    val_loss = 0.0\n    with torch.no_grad():\n        for batch_idx, (images, labels) in enumerate(val_loader):\n            images = images.to(device)\n            labels = labels.to(device)\n            outputs = convnext(images)\n            #log_probs = F.log_softmax(outputs, dim=1)\n            loss = criterion(outputs, labels)\n            val_loss += loss.item() * images.size(0)\n    avg_val_loss = val_loss / len(val_loader.dataset)\n    val_losses.append(avg_val_loss)\n    print(f\"Epoch [{epoch+1}/{num_epochs}] Validation Loss: {avg_val_loss:.4f}\")\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-22T11:03:00.858603Z","iopub.execute_input":"2025-02-22T11:03:00.858827Z","execution_failed":"2025-02-22T11:09:45.249Z"}},"outputs":[],"execution_count":null},{"id":"aace22fd","cell_type":"code","source":"fig = figure(figsize=(8,8), facecolor='w')\nframe = fig.add_subplot(1,1,1)\nframe.plot(range(1, num_epochs + 1), train_losses, label='Training Loss', color=\"b\", lw=3)\nframe.plot(range(1, num_epochs + 1), val_losses, label='Validation Loss', color=\"orange\", lw=3)\nframe.set_xlabel(r\"Epoch\", fontsize = 15)\nframe.set_ylabel(r\"Loss\", fontsize = 15)\n#frame.set_yscale('log')\nframe.set_title('Training vs. Validation Loss', fontsize = 18)\nframe.legend(prop={'size': 18})\nframe.grid()\nshow()\nfig.savefig('loss_curve.png')","metadata":{"trusted":true,"execution":{"execution_failed":"2025-02-22T11:09:45.249Z"}},"outputs":[],"execution_count":null},{"id":"5612391a-26df-4162-a47b-1586e85d0b79","cell_type":"markdown","source":"## Test your trained model","metadata":{}},{"id":"ce090dab","cell_type":"code","source":"class GalaxyZooTestDataset(Dataset):\n    def __init__(self, image_dir, transform=None):\n        self.image_dir = image_dir\n        self.image_files = sorted([f for f in os.listdir(image_dir) if f.endswith('.jpg')])\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.image_files)\n\n    def __getitem__(self, idx):\n        filename = self.image_files[idx]\n        image_path = os.path.join(self.image_dir, filename)\n        image = Image.open(image_path).convert('RGB')\n        if self.transform:\n            image = self.transform(image)\n        galaxy_id = os.path.splitext(filename)[0]\n        return image, galaxy_id\n\ntest_dataset = GalaxyZooTestDataset(image_dir=output_dir+'images_test_rev1', transform=transform)\ntest_loader = DataLoader(test_dataset, batch_size=512, shuffle=False, num_workers=4)\n\nconvnext.eval()\nall_predictions = []\nall_galaxy_ids = []\n\nwith torch.no_grad():\n    for images, galaxy_ids in test_loader:\n        images = images.to(device)\n        outputs = convnext(images) # raw logits\n        probs = torch.sigmoid(outputs) # convert logits -> [0, 1] for each output\n        all_predictions.append(probs.cpu().numpy())\n        all_galaxy_ids.extend(galaxy_ids)\n\npredictions = np.concatenate(all_predictions, axis=0)\n\ncolumns = ['GalaxyId']\nquestions = {1: 3, 2: 2, 3: 2, 4: 2, 5: 4, 6: 2, 7: 3, 8: 7, 9: 3, 10: 3, 11: 6}\nfor q, count in questions.items():\n    for i in range(1, count + 1):\n        columns.append(f'Class{q}.{i}')\n\nnum_pred_columns = sum(questions.values())\nassert predictions.shape[1] == num_pred_columns, \"Mismatch between predictions and expected class columns.\"\n\nsubmission_df = pd.DataFrame(predictions, columns=columns[1:])\nsubmission_df.insert(0, 'GalaxyId', all_galaxy_ids)\nprint(submission_df.head())\nsubmission_df.to_csv('/kaggle/working/submission.csv', index=False)","metadata":{"trusted":true,"execution":{"execution_failed":"2025-02-22T11:09:45.249Z"}},"outputs":[],"execution_count":null},{"id":"111c360e-23c0-4fe5-8a4a-ab2111e62f00","cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"id":"8d986a23-8bef-4c4b-8625-75fb9e0a721f","cell_type":"markdown","source":"!python /kaggle/working/galaxyzoo_multinode.py","metadata":{}}]}