{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":3175,"databundleVersionId":44352,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install peft","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install --upgrade peft","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T17:15:54.274378Z","iopub.execute_input":"2024-11-10T17:15:54.275227Z","iopub.status.idle":"2024-11-10T17:16:05.880712Z","shell.execute_reply.started":"2024-11-10T17:15:54.275187Z","shell.execute_reply":"2024-11-10T17:16:05.879502Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nfrom transformers import AutoImageProcessor, AutoModelForImageClassification\nfrom tqdm import tqdm\nimport torch\nimport torch.nn as nn\nfrom tqdm import tqdm\nfrom transformers import ResNetModel, ResNetConfig\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom torch.utils.data import DataLoader, Dataset\nimport os\nfrom PIL import Image\nimport torch\nfrom torchvision import transforms","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2024-11-10T17:16:17.263147Z","iopub.execute_input":"2024-11-10T17:16:17.264185Z","iopub.status.idle":"2024-11-10T17:16:17.701587Z","shell.execute_reply.started":"2024-11-10T17:16:17.264126Z","shell.execute_reply":"2024-11-10T17:16:17.700745Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!unzip /kaggle/input/galaxy-zoo-the-galaxy-challenge/training_solutions_rev1.zip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:12:02.569422Z","iopub.execute_input":"2024-11-10T16:12:02.570245Z","iopub.status.idle":"2024-11-10T16:12:03.762334Z","shell.execute_reply.started":"2024-11-10T16:12:02.570207Z","shell.execute_reply":"2024-11-10T16:12:03.761221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!unzip /kaggle/input/galaxy-zoo-the-galaxy-challenge/images_training_rev1.zip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:12:03.763808Z","iopub.execute_input":"2024-11-10T16:12:03.764121Z","iopub.status.idle":"2024-11-10T16:12:21.739533Z","shell.execute_reply.started":"2024-11-10T16:12:03.764088Z","shell.execute_reply":"2024-11-10T16:12:21.738300Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!unzip /kaggle/input/galaxy-zoo-the-galaxy-challenge/images_test_rev1.zip","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:12:21.741888Z","iopub.execute_input":"2024-11-10T16:12:21.742764Z","iopub.status.idle":"2024-11-10T16:12:45.071608Z","shell.execute_reply.started":"2024-11-10T16:12:21.742699Z","shell.execute_reply":"2024-11-10T16:12:45.070734Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/working/training_solutions_rev1.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:12:45.073141Z","iopub.execute_input":"2024-11-10T16:12:45.073559Z","iopub.status.idle":"2024-11-10T16:12:45.383236Z","shell.execute_reply.started":"2024-11-10T16:12:45.073510Z","shell.execute_reply":"2024-11-10T16:12:45.382445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:18:35.068840Z","iopub.execute_input":"2024-11-10T16:18:35.069233Z","iopub.status.idle":"2024-11-10T16:18:35.106514Z","shell.execute_reply.started":"2024-11-10T16:18:35.069194Z","shell.execute_reply":"2024-11-10T16:18:35.105542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:31:59.943373Z","iopub.execute_input":"2024-11-10T16:31:59.943820Z","iopub.status.idle":"2024-11-10T16:31:59.979167Z","shell.execute_reply.started":"2024-11-10T16:31:59.943767Z","shell.execute_reply":"2024-11-10T16:31:59.978240Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"processor = AutoImageProcessor.from_pretrained(\"microsoft/resnet-50\")\nmodel = AutoModelForImageClassification.from_pretrained(\"microsoft/resnet-50\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:13:59.291844Z","iopub.execute_input":"2024-11-10T16:13:59.292519Z","iopub.status.idle":"2024-11-10T16:14:00.689113Z","shell.execute_reply.started":"2024-11-10T16:13:59.292456Z","shell.execute_reply":"2024-11-10T16:14:00.688187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nfolder_path = \"/kaggle/working/images_training_rev1\"  # Update this with your folder path\n\nimage_files = [os.path.join(folder_path, file) for file in os.listdir(folder_path) if file.endswith(('.png', '.jpg', '.jpeg'))]\n\nfor image_path in image_files:\n    print(image_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:19:24.925575Z","iopub.execute_input":"2024-11-10T16:19:24.925940Z","iopub.status.idle":"2024-11-10T16:19:25.635382Z","shell.execute_reply.started":"2024-11-10T16:19:24.925906Z","shell.execute_reply":"2024-11-10T16:19:25.634522Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom PIL import Image\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom torchvision import transforms, models\nfrom torch import nn, optim\n\n# Set device\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:24:51.670142Z","iopub.execute_input":"2024-11-10T16:24:51.670537Z","iopub.status.idle":"2024-11-10T16:24:51.724064Z","shell.execute_reply.started":"2024-11-10T16:24:51.670498Z","shell.execute_reply":"2024-11-10T16:24:51.722987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class GalaxyZooDataset(Dataset):\n    def __init__(self, csv_file, images_folder, transform=None):\n        self.data = pd.read_csv(csv_file)\n        self.images_folder = images_folder\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        # Retrieve image and label\n        row = self.data.iloc[idx]\n        galaxy_id = str(int(row['GalaxyID']))\n        image_path = os.path.join(self.images_folder, f\"{galaxy_id}.jpg\")  # Update extension if different\n        if not os.path.exists(image_path):\n            print(f\"Image {image_path} not found.\")\n        # Load image\n        image = Image.open(image_path).convert(\"RGB\")\n        if self.transform:\n            image = self.transform(image)\n        \n        # Get labels as a tensor\n        labels = torch.tensor(row[1:].values, dtype=torch.float)\n        \n        return image, labels\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:44:42.296391Z","iopub.execute_input":"2024-11-10T16:44:42.297306Z","iopub.status.idle":"2024-11-10T16:44:42.305388Z","shell.execute_reply.started":"2024-11-10T16:44:42.297260Z","shell.execute_reply":"2024-11-10T16:44:42.304317Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\nclass GalaxyZooDataset(Dataset):\n    def __init__(self, data, images_folder, transform=None):\n        self.data = data  # Data is now a DataFrame\n        self.images_folder = images_folder\n        self.transform = transform\n\n    def __len__(self):\n        return len(self.data)\n\n    def __getitem__(self, idx):\n        row = self.data.iloc[idx]\n        galaxy_id = str(int(row['GalaxyID']))\n        image_path = os.path.join(self.images_folder, f\"{galaxy_id}.jpg\")  # Update extension if different\n        \n        if not os.path.exists(image_path):\n            print(f\"Image {image_path} not found.\")\n            image = Image.new('RGB', (224, 224), color=(255, 255, 255)) \n        else:\n            image = Image.open(image_path).convert(\"RGB\")\n\n        if self.transform:\n            image = self.transform(image)\n        \n        labels = torch.tensor(row[1:].values, dtype=torch.float)\n        \n        return image, labels\n\ntransform = transforms.Compose([\n    transforms.Resize((224, 224)),\n    transforms.ToTensor(),\n    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n])\n\ndf = pd.read_csv(\"/kaggle/working/training_solutions_rev1.csv\")\n\ntrain_df, test_df = train_test_split(df, test_size=0.2, random_state=42)  # 80% train, 20% test\n\ntrain_dataset = GalaxyZooDataset(data=train_df, images_folder=\"/kaggle/working/images_training_rev1\", transform=transform)\ntest_dataset = GalaxyZooDataset(data=test_df, images_folder=\"/kaggle/working/images_training_rev1\", transform=transform)\n\ntrain_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)\ntest_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T17:16:44.999854Z","iopub.execute_input":"2024-11-10T17:16:45.001301Z","iopub.status.idle":"2024-11-10T17:16:45.586844Z","shell.execute_reply.started":"2024-11-10T17:16:45.001244Z","shell.execute_reply":"2024-11-10T17:16:45.585881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = models.resnet50(pretrained=True)\nmodel.fc = nn.Linear(model.fc.in_features, 37)  # Adjust output layer (update 37 to your actual number of classes)\nmodel = model.to(device)\n\ncriterion = nn.BCEWithLogitsLoss()  # Use BCEWithLogitsLoss for multi-label classification\noptimizer = optim.Adam(model.parameters(), lr=1e-4)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T17:17:15.514928Z","iopub.execute_input":"2024-11-10T17:17:15.515631Z","iopub.status.idle":"2024-11-10T17:17:16.064844Z","shell.execute_reply.started":"2024-11-10T17:17:15.515588Z","shell.execute_reply":"2024-11-10T17:17:16.063818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_model(model, dataloader):\n    model.eval()\n    total_loss = 0.0\n    with torch.no_grad():\n        for images, labels in dataloader:\n            images, labels = images.to(device), labels.to(device)\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            total_loss += loss.item()\n    return total_loss / len(dataloader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T16:34:25.040113Z","iopub.execute_input":"2024-11-10T16:34:25.040509Z","iopub.status.idle":"2024-11-10T16:34:25.046518Z","shell.execute_reply.started":"2024-11-10T16:34:25.040457Z","shell.execute_reply":"2024-11-10T16:34:25.045600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Before training\nmodel = models.resnet50(pretrained=True)\nmodel.fc = nn.Linear(model.fc.in_features, 37) \nmodel = model.to(device)\nevaluate_model(model,test_loader)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-10T17:20:44.266807Z","iopub.execute_input":"2024-11-10T17:20:44.267202Z","iopub.status.idle":"2024-11-10T17:22:03.484408Z","shell.execute_reply.started":"2024-11-10T17:20:44.267161Z","shell.execute_reply":"2024-11-10T17:22:03.483490Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def train_model(model, dataloader, optimizer, num_epochs=5):\n    model.train()\n    for epoch in range(num_epochs):\n        running_loss = 0.0\n        print(f\"Epoch {epoch+1}/{num_epochs}\")\n        \n        for images, labels in tqdm(dataloader, desc=f\"Training Epoch {epoch+1}\", leave=False):\n            images, labels = images.to(device), labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images)\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item()\n        \n        print(f\"Loss: {running_loss / len(dataloader):.4f}\")\n\ntrain_model(model, train_loader, optimizer)\n\nfine_tuned_loss = evaluate_model(model, test_loader)\nprint(f\"Fine-tuned Model Loss: {fine_tuned_loss:.4f}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#After normal training\nmodel.eval()\nevaluate_model(model,test_loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import peft","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T07:22:54.528593Z","iopub.execute_input":"2024-11-11T07:22:54.529074Z","iopub.status.idle":"2024-11-11T07:22:54.533967Z","shell.execute_reply.started":"2024-11-11T07:22:54.529031Z","shell.execute_reply":"2024-11-11T07:22:54.532781Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model_name = \"microsoft/resnet-50\"\nconfig = ResNetConfig.from_pretrained(model_name)\nmodel = ResNetModel.from_pretrained(model_name, config=config)\nmodel = model.to(device)\n\n# Step 2: Apply LoRA to the model\nlora_config = LoRAConfig(\n    r=4,  # Low-rank dimension (tune this as needed)\n    lora_alpha=16,  # Scaling factor (tune this as needed)\n    target_modules=[\"layer1\", \"layer2\", \"layer3\", \"layer4\"],  # LoRA target layers\n    lora_dropout=0.1  # Dropout rate for LoRA\n)\nmodel = get_peft_model(model, lora_config)\nmodel = model.to(device)\n\n# Step 3: Define a new optimizer for LoRA fine-tuning\noptimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n\n# Define the loss function\ncriterion = nn.CrossEntropyLoss()\n\n# Step 4: LoRA Fine-tuning function with tqdm\ndef train_model_with_lora(model, dataloader, optimizer, num_epochs=5):\n    model.train()\n    for epoch in range(num_epochs):\n        running_loss = 0.0\n        print(f\"Epoch {epoch+1}/{num_epochs}\")\n        \n        # Wrap the dataloader with tqdm for batch progress tracking\n        for images, labels in tqdm(dataloader, desc=f\"LoRA Fine-tuning Epoch {epoch+1}\", leave=False):\n            images, labels = images.to(device), labels.to(device)\n            \n            optimizer.zero_grad()\n            outputs = model(images).logits  # .logits for ResNet output in Hugging Face format\n            loss = criterion(outputs, labels)\n            loss.backward()\n            optimizer.step()\n            \n            running_loss += loss.item()\n        \n        # Print epoch-level loss\n        print(f\"Loss: {running_loss / len(dataloader):.4f}\")\n\n# Step 5: Run the LoRA fine-tuning\ntrain_model_with_lora(model, train_loader, optimizer)\n\n# Step 6: Evaluate the LoRA fine-tuned model on the test data\nlora_fine_tuned_loss = evaluate_model(model, test_loader)\nprint(f\"LoRA Fine-tuned Model Loss: {lora_fine_tuned_loss:.4f}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-11T07:26:45.190425Z","iopub.execute_input":"2024-11-11T07:26:45.191208Z","iopub.status.idle":"2024-11-11T07:26:45.234325Z","shell.execute_reply.started":"2024-11-11T07:26:45.191168Z","shell.execute_reply":"2024-11-11T07:26:45.233000Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#After Lora Finetuning\nmodel.eval()\nevaluate_model(model,test_loader)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}