{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":8089,"databundleVersionId":44321,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13794204,"sourceType":"datasetVersion","datasetId":8782152},{"sourceId":13908503,"sourceType":"datasetVersion","datasetId":8861836}],"dockerImageVersionId":31193,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install stardist\n!pip install gputools","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-29T12:44:11.153061Z","iopub.execute_input":"2025-11-29T12:44:11.153815Z","iopub.status.idle":"2025-11-29T12:44:27.21152Z","shell.execute_reply.started":"2025-11-29T12:44:11.153787Z","shell.execute_reply":"2025-11-29T12:44:27.210762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm\nfrom skimage.io import imread, imsave\nfrom skimage.color import rgb2gray\nfrom sklearn.model_selection import train_test_split\nfrom stardist.models import Config2D, StarDist2D\nfrom csbdeep.utils import normalize\nimport albumentations as A\nimport zipfile\nimport shutil\nfrom pathlib import Path\nfrom skimage.segmentation import mark_boundaries\nfrom skimage.measure import regionprops\n\n# ==========================================\n# CẤU HÌNH\n# ==========================================\nDATASET_PATH = '/kaggle/input/data-science-bowl-2018/stage1_train.zip'\nEXTRACT_PATH = 'stage1_train/'\nTEST_PATH = '/kaggle/input/bubble/frames/'\nMODEL_NAME = 'stardist_overlap_v1'\nMODEL_DIR = 'models'\n\n# ==========================================\n# 1. GIẢI NÉN DỮ LIỆU\n# ==========================================\ndef extract_dataset():\n    \"\"\"Giải nén dataset nếu chưa có\"\"\"\n    if not os.path.exists(EXTRACT_PATH):\n        print(\"📦 Đang giải nén dữ liệu...\")\n        if os.path.exists(DATASET_PATH):\n            with zipfile.ZipFile(DATASET_PATH, 'r') as zip_ref:\n                zip_ref.extractall(EXTRACT_PATH)\n            print(\"✅ Giải nén hoàn tất!\")\n        else:\n            print(\"⚠️ CẢNH BÁO: Không tìm thấy file zip DSB2018!\")\n            return False\n    return True\n\n# ==========================================\n# 2. LOAD & PREPROCESS DỮ LIỆU\n# ==========================================\ndef preprocess_image(img):\n    \"\"\"Xử lý ảnh: bỏ alpha, chuyển grayscale, đảo màu nếu cần\"\"\"\n    if len(img.shape) == 3 and img.shape[2] == 4:\n        img = img[:,:,:3]\n    \n    if len(img.shape) == 3:\n        img = rgb2gray(img)\n    \n    if np.mean(img) > 0.5:\n        img = 1.0 - img\n    \n    return img\n\ndef load_training_data(base_path):\n    \"\"\"Load dữ liệu training với xử lý tối ưu\"\"\"\n    train_ids = next(os.walk(base_path))[1]\n    X, Y = [], []\n    \n    print(f\"🔄 Đang load {len(train_ids)} ảnh...\")\n    \n    for id_ in tqdm(train_ids):\n        path = os.path.join(base_path, id_)\n        \n        img_path = os.path.join(path, 'images', f'{id_}.png')\n        if not os.path.exists(img_path):\n            continue\n        \n        img = imread(img_path)\n        img = preprocess_image(img)\n        \n        if np.isnan(img).any() or np.isinf(img).any():\n            continue\n        \n        mask_path = os.path.join(path, 'masks')\n        if not os.path.exists(mask_path):\n            continue\n        \n        mask_files = os.listdir(mask_path)\n        if len(mask_files) == 0:\n            continue\n        \n        mask_accum = np.zeros(img.shape[:2], dtype=np.int32)\n        for i, mask_file in enumerate(mask_files):\n            m = imread(os.path.join(mask_path, mask_file))\n            m = m > 0\n            mask_accum = np.maximum(mask_accum, m * (i + 1))\n        \n        if mask_accum.max() > 0:\n            X.append(img)\n            Y.append(mask_accum)\n    \n    print(f\"✅ Load thành công {len(X)} ảnh hợp lệ\")\n    return X, Y\n\n# ==========================================\n# 3. AUGMENTATION\n# ==========================================\ndef get_augmentation():\n    \"\"\"Pipeline augmentation mạnh cho chồng lấn\"\"\"\n    return A.Compose([\n        A.HorizontalFlip(p=0.5),\n        A.VerticalFlip(p=0.5),\n        A.RandomRotate90(p=0.5),\n        A.ElasticTransform(alpha=1, sigma=50, p=0.4),\n        A.RandomBrightnessContrast(brightness_limit=0.2, contrast_limit=0.2, p=0.5),\n        A.GaussianBlur(blur_limit=(3, 5), p=0.2),\n        A.RandomGamma(gamma_limit=(80, 120), p=0.3),\n    ])\n\ndef augmenter(x, y):\n    \"\"\"Wrapper cho augmentation\"\"\"\n    aug = get_augmentation()\n    augmented = aug(image=x.astype(np.float32), mask=y.astype(np.int32))\n    return augmented['image'], augmented['mask']\n\n# ==========================================\n# 4. TẠO & TRAIN MODEL\n# ==========================================\ndef create_model():\n    \"\"\"Tạo model StarDist với config ổn định nhất\"\"\"\n    model_path = os.path.join(MODEL_DIR, MODEL_NAME)\n    if os.path.exists(model_path):\n        shutil.rmtree(model_path)\n    \n    conf = Config2D(\n        n_rays=64,\n        grid=(2,2),\n        unet_n_depth=4,\n        unet_n_filter_base=48,\n        use_gpu=True,\n        n_channel_in=1,\n        train_patch_size=(256, 256),\n        train_batch_size=4,\n        train_learning_rate=0.0003,\n        train_n_val_patches=None,\n        train_reduce_lr={'factor': 0.5, 'patience': 10},\n    )\n    \n    return StarDist2D(conf, name=MODEL_NAME, basedir=MODEL_DIR)\n\ndef train_model(X_trn, Y_trn, X_val, Y_val):\n    \"\"\"Train model với config an toàn\"\"\"\n    model = create_model()\n    \n    print(\"🚀 Bắt đầu training với config mặc định...\")\n    print(f\"📊 Train samples: {len(X_trn)}, Val samples: {len(X_val)}\")\n    \n    history = model.train(\n        X_trn, Y_trn,\n        validation_data=(X_val, Y_val),\n        augmenter=augmenter,\n        epochs=80,\n        steps_per_epoch=100\n    )\n    \n    print(\"⚖️ Đang tối ưu hóa ngưỡng...\")\n    model.optimize_thresholds(X_val, Y_val)\n    \n    return model, history\n\n\ndef analyze_single_object(mask, obj_id):\n    \"\"\"Phân tích chi tiết 1 vật thể\"\"\"\n    obj_mask = (mask == obj_id)\n    \n    # Tính pixel\n    pixel_count = np.sum(obj_mask)\n    \n    # Tính các thuộc tính hình học\n    props = regionprops(obj_mask.astype(int))[0] if np.any(obj_mask) else None\n    \n    if props:\n        return {\n            'pixel_count': pixel_count,\n            'centroid_y': round(props.centroid[0], 2),\n            'centroid_x': round(props.centroid[1], 2),\n            'area': props.area,\n            'perimeter': round(props.perimeter, 2),\n            'eccentricity': round(props.eccentricity, 3),\n            'solidity': round(props.solidity, 3),\n            'bbox_min_y': props.bbox[0],\n            'bbox_min_x': props.bbox[1],\n            'bbox_max_y': props.bbox[2],\n            'bbox_max_x': props.bbox[3],\n            'bbox_width': props.bbox[3] - props.bbox[1],\n            'bbox_height': props.bbox[2] - props.bbox[0],\n            'equivalent_diameter': round(props.equivalent_diameter, 2)\n        }\n    else:\n        return None\n\n# ==========================================\n# 6. VẼ SỐ THỨ TỰ CỰC RÕ RÀNG\n# ==========================================\ndef create_numbered_visualization(img_processed, labels, num_objects, img_name, output_dir):\n    \"\"\"Tạo ảnh với số thứ tự nhỏ gọn trên boundary overlay\"\"\"\n    \n    # Tạo figure với 3 subplot ngang\n    fig, axes = plt.subplots(1, 3, figsize=(18, 6))\n    \n    # ===== 1. ẢNH GỐC =====\n    axes[0].imshow(img_processed, cmap='gray')\n    axes[0].set_title(f'Original Image', fontsize=12, fontweight='bold')\n    axes[0].axis('off')\n    \n    # ===== 2. COLOR-CODED =====\n    axes[1].imshow(labels, cmap='nipy_spectral', interpolation='nearest')\n    axes[1].set_title(f'Color-coded ({num_objects} bubbles)', fontsize=12, fontweight='bold')\n    axes[1].axis('off')\n    \n    # ===== 3. BOUNDARY OVERLAY VỚI ID NHỎ =====\n    overlay = mark_boundaries(img_processed, labels, color=(1, 0, 0), mode='thick')\n    axes[2].imshow(overlay)\n    \n    # Đánh số ID nhỏ gọn tại centroid của mỗi bubble\n    for obj_id in range(1, num_objects + 1):\n        obj_mask = (labels == obj_id)\n        if np.any(obj_mask):\n            props = regionprops(obj_mask.astype(int))[0]\n            cy, cx = props.centroid\n            \n            # Vẽ số ID nhỏ, rõ ràng\n            axes[2].text(cx, cy, str(obj_id), \n                        color='yellow', \n                        fontsize=3,  # Font nhỏ\n                        fontweight='bold',\n                        ha='center', \n                        va='center',\n                        bbox=dict(boxstyle='circle,pad=0.2', \n                                 facecolor='red', \n                                 edgecolor='white', \n                                 linewidth=0.5, \n                                 alpha=0.8))\n    \n    axes[2].set_title(f'Boundary Overlay with IDs', fontsize=12, fontweight='bold')\n    axes[2].axis('off')\n    \n    # Tiêu đề chung\n    fig.suptitle(f'{img_name} - Detected {num_objects} bubbles', \n                 fontsize=14, fontweight='bold', y=0.98)\n    \n    # Lưu figure\n    output_path = os.path.join(output_dir, f'numbered_{img_name}')\n    plt.tight_layout()\n    plt.savefig(output_path, dpi=150, bbox_inches='tight', facecolor='white')\n    plt.close()\n    \n    print(f\"   ✅ Đã lưu: {output_path}\")\n\n# 7. TEST VÀ XUẤT KẾT QUẢ CHI TIẾT\ndef test_on_bubble_dataset(model, test_path, output_dir='test_results'):\n    \"\"\"Test model và xuất kết quả chi tiết cho từng bubble\"\"\"\n    if not os.path.exists(test_path):\n        print(f\"⚠️ Không tìm thấy thư mục test: {test_path}\")\n        return\n    \n    # Tạo thư mục output\n    os.makedirs(output_dir, exist_ok=True)\n    vis_dir = os.path.join(output_dir, 'numbered_visualizations')\n    os.makedirs(vis_dir, exist_ok=True)\n    \n    test_images = sorted([f for f in os.listdir(test_path) \n                         if f.endswith(('.png', '.jpg', '.jpeg', '.tif'))])\n    \n    if len(test_images) == 0:\n        print(f\"⚠️ Không tìm thấy ảnh nào trong {test_path}\")\n        return\n    \n    print(f\"\\n🧪 Bắt đầu test trên {len(test_images)} ảnh...\")\n    \n    all_results = []\n    prob_thresh = 0.4\n    nms_thresh = 0.3\n    \n    for img_name in tqdm(test_images):\n        img_path = os.path.join(test_path, img_name)\n        \n        # Load và preprocess\n        img = imread(img_path)\n        img_processed = preprocess_image(img)\n        img_norm = normalize(img_processed, 1, 99.8, axis=(0, 1))\n        img_norm = np.clip(img_norm, 0, 1)\n        \n        # Predict\n        labels, details = model.predict_instances(\n            img_norm,\n            prob_thresh=prob_thresh,\n            nms_thresh=nms_thresh\n        )\n        \n        num_objects = len(details['prob'])\n        \n        # ===== PHÂN TÍCH TỪNG VẬT THỂ =====\n        # print(f\"\\n📸 {img_name}: Phát hiện {num_objects} bubbles\")\n        \n        for obj_id in range(1, num_objects + 1):\n            obj_info = analyze_single_object(labels, obj_id)\n            \n            if obj_info:\n                obj_info.update({\n                    'image': img_name,\n                    'object_id': obj_id,\n                    'confidence': round(details['prob'][obj_id - 1], 3)\n                })\n                \n                all_results.append(obj_info)\n                \n                # print(f\"  🔵 Bubble #{obj_id}: {obj_info['pixel_count']}px | \"\n                #       f\"Ø{obj_info['equivalent_diameter']}px | \"\n                #       f\"conf={obj_info['confidence']}\")\n        \n        # ===== TẠO VISUALIZATION VỚI SỐ RÕ RÀNG =====\n        create_numbered_visualization(img_processed, labels, num_objects, img_name, vis_dir)\n    \n    # ===== XUẤT KẾT QUẢ =====\n    df = pd.DataFrame(all_results)\n    csv_path = os.path.join(output_dir, 'detailed_pixel_counts.csv')\n    df.to_csv(csv_path, index=False)\n    \n    # Thống kê\n    summary = df.groupby('image').agg({\n        'object_id': 'count',\n        'pixel_count': ['mean', 'std', 'min', 'max', 'sum'],\n        'equivalent_diameter': ['mean', 'std'],\n        'confidence': 'mean'\n    }).round(2)\n    summary.columns = ['_'.join(col).strip() for col in summary.columns.values]\n    summary_path = os.path.join(output_dir, 'summary_by_image.csv')\n    summary.to_csv(summary_path)\n    \n    # In thống kê\n    print(f\"\\n\" + \"=\"*70)\n    print(\"📊 THỐNG KÊ TỔNG QUAN\")\n    print(\"=\"*70)\n    print(f\"✅ Tổng số ảnh: {len(test_images)}\")\n    print(f\"✅ Tổng số bubbles: {len(df)}\")\n    print(f\"✅ Trung bình bubbles/ảnh: {len(df)/len(test_images):.1f}\")\n    print(f\"\\n🔵 PIXEL COUNT:\")\n    print(f\"   - Trung bình: {df['pixel_count'].mean():.1f} px\")\n    print(f\"   - Min/Max: {df['pixel_count'].min()} / {df['pixel_count'].max()} px\")\n    print(f\"   - Median: {df['pixel_count'].median():.1f} px\")\n    print(\"=\"*70)\n    \n    print(f\"\\n📁 KẾT QUẢ ĐÃ LƯU TẠI:\")\n    print(f\"   📊 {csv_path}\")\n    print(f\"   📊 {summary_path}\")\n    print(f\"   🖼️  {vis_dir}/\")\n    \n    return df\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T12:45:18.722397Z","iopub.execute_input":"2025-11-29T12:45:18.72292Z","iopub.status.idle":"2025-11-29T12:45:18.751475Z","shell.execute_reply.started":"2025-11-29T12:45:18.722897Z","shell.execute_reply":"2025-11-29T12:45:18.750795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"extract_dataset()\nX_raw, Y_raw = load_training_data(EXTRACT_PATH)\n\nprint(\"🔄 Đang chuẩn hóa dữ liệu...\")\nX_norm = []\nfor x in X_raw:\n    x_norm = normalize(x, 1, 99.8, axis=(0, 1))\n    x_norm = np.clip(x_norm, 0, 1)\n    if not np.isnan(x_norm).any() and not np.isinf(x_norm).any():\n        X_norm.append(x_norm)\n\nprint(f\"✅ Có {len(X_norm)} ảnh sau chuẩn hóa\")\n\nX_trn, X_val, Y_trn, Y_val = train_test_split(\n    X_norm, Y_raw, test_size=0.1, random_state=42\n)\n\nmodel, history = train_model(X_trn, Y_trn, X_val, Y_val)\n\n# Vẽ loss\nplt.figure(figsize=(10, 5))\nplt.plot(history.history['loss'], label='Train Loss', linewidth=2)\nplt.plot(history.history['val_loss'], label='Val Loss', linewidth=2)\nplt.title('Training History', fontsize=14, fontweight='bold')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.grid(alpha=0.3)\nplt.savefig('training_loss.png', dpi=150, bbox_inches='tight')\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T12:46:25.128977Z","iopub.execute_input":"2025-11-29T12:46:25.129287Z","iopub.status.idle":"2025-11-29T13:04:12.109042Z","shell.execute_reply.started":"2025-11-29T12:46:25.129262Z","shell.execute_reply":"2025-11-29T13:04:12.108308Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Test\nprint(\"\\n\" + \"=\"*70)\nprint(\"🧪 TEST VỚI ĐÁNH SỐ TỪNG BUBBLE\")\nprint(\"=\"*70)\nresults_df = test_on_bubble_dataset(model, TEST_PATH)\n\n# Đóng gói model\nprint(\"\\n📦 Đang nén model...\")\nshutil.make_archive('best_bubble_model', 'zip', os.path.join(MODEL_DIR, MODEL_NAME))\nprint(\"✅ HOÀN TẤT! Kiểm tra thư mục 'test_results/numbered_visualizations/'\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T13:04:12.11035Z","iopub.execute_input":"2025-11-29T13:04:12.111127Z","iopub.status.idle":"2025-11-29T13:27:26.946921Z","shell.execute_reply.started":"2025-11-29T13:04:12.111105Z","shell.execute_reply":"2025-11-29T13:27:26.946233Z"},"collapsed":true,"jupyter":{"outputs_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!zip -r working.zip /kaggle/working/\n ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-29T13:29:36.820549Z","iopub.execute_input":"2025-11-29T13:29:36.820839Z","iopub.status.idle":"2025-11-29T13:30:13.395095Z","shell.execute_reply.started":"2025-11-29T13:29:36.820816Z","shell.execute_reply":"2025-11-29T13:30:13.394431Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}