{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":3175,"databundleVersionId":44352}],"dockerImageVersionId":31328,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Loading and Labelling the Data","metadata":{}},{"cell_type":"code","source":"import zipfile\nimport os\n\n# List of files to unzip\nzip_files = [\n    'images_training_rev1.zip', \n    'images_test_rev1.zip', \n    'training_solutions_rev1.zip'\n]\n\nbase_input_path = '/kaggle/input/competitions/galaxy-zoo-the-galaxy-challenge'\nbase_extract_path = '/kaggle/working/'\n\ntarget_thresh = 0.8 #threshold for the data\nRANDOM_SEED = 42\n\nfor file in zip_files:\n    zip_path = os.path.join(base_input_path, file)\n    \n    # Create a specific folder for images, but keep the CSV in the main working dir\n    if 'images' in file:\n        folder_name = file.replace('.zip', '')\n        extract_path = os.path.join(base_extract_path, folder_name)\n    else:\n        extract_path = base_extract_path\n        \n    os.makedirs(extract_path, exist_ok=True)\n\n    print(f\"Unzipping {file}...\")\n    with zipfile.ZipFile(zip_path, 'r') as zip_ref:\n        zip_ref.extractall(extract_path)\n    print(\"All files extracted successfully.\")\n\n    location_path = os.path.join(base_extract_path, folder_name, folder_name)\n    if os.path.exists(location_path):\n        files = os.listdir(location_path)\n        print(f\"✅ Folder found!\")\n        print(f\"Total files in training folder: {len(files)}\")\n        if len(files) > 0:\n            print(f\"Sample filename: {files[0]}\")\n    else:\n        print(\"❌ Folder not found. Check your extraction path.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:03:52.324602Z","iopub.execute_input":"2026-04-04T09:03:52.324862Z","iopub.status.idle":"2026-04-04T09:04:27.092106Z","shell.execute_reply.started":"2026-04-04T09:03:52.324841Z","shell.execute_reply":"2026-04-04T09:04:27.091441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Hierarchical Thresholding\n\ndef thresh_labels(row, thresh):\n    #the standards for the thresholds are 0.8 (from Willett et al. (2013), 0.5, and 0.7 (for the further tasks))\n    # Scaled in this definition since lower quality images can be insightful testing data\n    # Rule 1: High Consensus Smooth = Elliptical\n    if row['Class1.1'] >= thresh:\n        return 'Elliptical'\n    \n    # Rule 2: High Consensus Features + Arms = Spiral\n    # (Note: We ignore the small % of 'Smooth' votes here)\n    if row['Class1.2'] >= thresh and row['Class4.1'] >= ((5/8)*thresh):\n        return 'Spiral'\n    \n    # Rule 3: High Consensus Artifact or 'Odd' = Irregular\n    if row['Class1.3'] >= thresh or row['Class6.1'] >= ((7/8)*thresh):\n        return 'Irregular'\n    \n    # Rule 4: If no consensus, throw it out! (Prevents noisy training)\n    return 'Uncertain'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:04:27.093457Z","iopub.execute_input":"2026-04-04T09:04:27.093686Z","iopub.status.idle":"2026-04-04T09:04:27.098258Z","shell.execute_reply.started":"2026-04-04T09:04:27.093660Z","shell.execute_reply":"2026-04-04T09:04:27.097483Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\ntraining_sol = pd.read_csv('/kaggle/working/training_solutions_rev1.csv')\n\n# 3. Apply the function to create a new column\n# 'args' passes the thresh value to your function\ntraining_sol['label'] = training_sol.apply(thresh_labels, axis=1, thresh=target_thresh)\n\n# 4. Filter out the 'Uncertain' rows to get your final labeled DF\ntrain_sol_labeled = training_sol[training_sol['label'] != 'Uncertain'].copy()\n\n# 5. Check the results\nprint(f\"Original size: {len(training_sol)}\")\nprint(f\"Labeled size: {len(train_sol_labeled)}\")\nprint(train_sol_labeled['label'].value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:04:27.099096Z","iopub.execute_input":"2026-04-04T09:04:27.099305Z","iopub.status.idle":"2026-04-04T09:04:27.676690Z","shell.execute_reply.started":"2026-04-04T09:04:27.099285Z","shell.execute_reply":"2026-04-04T09:04:27.675986Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## The Stratified 80/10/10 Split","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\n# Create the 80/10/10 split\n# 1. First split: 80% Train, 20% \"Remainder\"\ntrain_df, temp_df = train_test_split(\n    train_sol_labeled, \n    test_size=0.2, \n    stratify=train_sol_labeled['label'], \n    random_state=RANDOM_SEED\n)\n\n# 2. Second split: Split the 20% \"Remainder\" into half (10% Val, 10% Test)\n# FIX: Use temp_df here, not the full train_sol_labeled\nval_df, test_df = train_test_split(\n    temp_df, \n    test_size=0.5, \n    stratify=temp_df['label'], \n    random_state=RANDOM_SEED\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:04:27.677433Z","iopub.execute_input":"2026-04-04T09:04:27.677630Z","iopub.status.idle":"2026-04-04T09:04:27.699786Z","shell.execute_reply.started":"2026-04-04T09:04:27.677610Z","shell.execute_reply":"2026-04-04T09:04:27.699223Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Image Integrity","metadata":{}},{"cell_type":"code","source":"import cv2\nimport os\nfrom tqdm import tqdm # Great for tracking progress on 20k images\n\ndef check_image_integrity(df, img_path):\n    corrupted_ids = []\n    missing_ids = []\n    \n    print(f\"Checking integrity for {len(df)} images...\")\n    \n    for idx, row in tqdm(df.iterrows(), total=len(df)):\n        galaxy_id = int(row['GalaxyID'])\n        full_path = os.path.join(img_path, f\"{galaxy_id}.jpg\")\n        \n        # 1. Check if file exists\n        if not os.path.exists(full_path):\n            missing_ids.append(galaxy_id)\n            continue\n            \n        # 2. Try to decode the image\n        img = cv2.imread(full_path)\n        \n        # 3. Check if image is corrupted or empty\n        if img is None or img.size == 0:\n            corrupted_ids.append(galaxy_id)\n            \n    return missing_ids, corrupted_ids\n\n# Run the check\nmissing, corrupted = check_image_integrity(train_sol_labeled, '/kaggle/working/images_training_rev1/images_training_rev1')\n\nprint(f\"\\nResults:\")\nprint(f\"Missing Files: {len(missing)}\")\nprint(f\"Corrupted Files: {len(corrupted)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:04:27.701148Z","iopub.execute_input":"2026-04-04T09:04:27.701340Z","iopub.status.idle":"2026-04-04T09:04:40.709332Z","shell.execute_reply.started":"2026-04-04T09:04:27.701321Z","shell.execute_reply":"2026-04-04T09:04:40.708534Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bad_ids = missing + corrupted\n\nif bad_ids:\n    print(f\"Removing {len(bad_ids)} problematic images from splits...\")\n    train_df = train_df[~train_df['GalaxyID'].isin(bad_ids)]\n    val_df = val_df[~val_df['GalaxyID'].isin(bad_ids)]\n    test_df = test_df[~test_df['GalaxyID'].isin(bad_ids)]\nelse:\n    print(\"All images verified! Dataset is healthy.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:04:40.710222Z","iopub.execute_input":"2026-04-04T09:04:40.710453Z","iopub.status.idle":"2026-04-04T09:04:40.715014Z","shell.execute_reply.started":"2026-04-04T09:04:40.710428Z","shell.execute_reply":"2026-04-04T09:04:40.714234Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Calculate \"Space-Specific\" Normalization","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport cv2\nimport os\nfrom tqdm import tqdm\n\ndef calculate_galaxy_stats(df, img_path):\n    # Initialize sums for Mean and Std\n    # We use 3 channels (R, G, B)\n    pixel_sum = np.zeros(3)\n    pixel_sq_sum = np.zeros(3)\n    pixel_count = 0\n    \n    print(\"Calculating Space-Specific Normalization Stats...\")\n    \n    for _, row in tqdm(df.iterrows(), total=len(df)):\n        img_id = int(row['GalaxyID'])\n        img = cv2.imread(os.path.join(img_path, f\"{img_id}.jpg\"))\n        \n        if img is not None:\n            # Convert BGR (OpenCV) to RGB and normalize to [0, 1]\n            img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0\n            \n            # Add to global sums\n            pixel_sum += np.mean(img, axis=(0, 1))\n            pixel_sq_sum += np.mean(np.square(img), axis=(0, 1))\n            pixel_count += 1\n            \n    # Final Mean and Std calculation\n    mean = pixel_sum / pixel_count\n    std = np.sqrt((pixel_sq_sum / pixel_count) - np.square(mean))\n    \n    return mean, std\n\n# Run on Training Set Only\nactual_img_path = '/kaggle/working/images_training_rev1/images_training_rev1'\ntrain_mean, train_std = calculate_galaxy_stats(train_df, actual_img_path)\n\nprint(f\"\\nSpace-Specific Mean: {train_mean}\")\nprint(f\"Space-Specific Std:  {train_std}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:10:17.909103Z","iopub.execute_input":"2026-04-04T09:10:17.909833Z","iopub.status.idle":"2026-04-04T09:12:28.345789Z","shell.execute_reply.started":"2026-04-04T09:10:17.909809Z","shell.execute_reply":"2026-04-04T09:12:28.344747Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Saving the CSVs","metadata":{}},{"cell_type":"code","source":"# 1. Save the Split CSVs\ntrain_df.to_csv('train_split.csv', index=False)\nval_df.to_csv('val_split.csv', index=False)\ntest_df.to_csv('test_split.csv', index=False)\n\n# 2. Save your Space-Specific Normalization Stats (for easy copy-pasting)\nstats = {\n    \"mean\": train_mean.tolist(),\n    \"std\": train_std.tolist()\n}\n\nimport json\nwith open('norm_stats.json', 'w') as f:\n    json.dump(stats, f)\n\nprint(\"Successfully exported Master Index and Normalization Stats!\")\nprint(f\"Final Training Samples: {len(train_df)}\")\nprint(f\"Final Validation Samples: {len(val_df)}\")\nprint(f\"Final Testing Samples: {len(test_df)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-04-04T09:14:29.618018Z","iopub.execute_input":"2026-04-04T09:14:29.618325Z","iopub.status.idle":"2026-04-04T09:14:30.034171Z","shell.execute_reply.started":"2026-04-04T09:14:29.618299Z","shell.execute_reply":"2026-04-04T09:14:30.033456Z"}},"outputs":[],"execution_count":null}]}