{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31192,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# =========================================================================================\n# JERRY FEATURE DEV: PHYSICS-BASED & FPS-AWARE FEATURE ENGINEERING\n# (Adapted from mabe-extra-trees-gpu.ipynb)\n# =========================================================================================\n\nimport numpy as np\nimport pandas as pd\nimport itertools\nimport warnings\nimport json\nimport re\nimport gc\nfrom typing import Dict, Optional, Tuple\n\nwarnings.filterwarnings('ignore')\n\n# -----------------------------------------------------------------------\n# 1. FPS SCALING HELPERS (Vũ khí bí mật để xử lý video khác FPS)\n# -----------------------------------------------------------------------\n\ndef _scale(n_frames_at_30fps, fps, ref=30.0):\n    \"\"\"Scale frame count to current video's FPS.\"\"\"\n    return max(1, int(round(n_frames_at_30fps * float(fps) / ref)))\n\ndef _scale_signed(n_frames_at_30fps, fps, ref=30.0):\n    \"\"\"Signed version for lags/shifts.\"\"\"\n    if n_frames_at_30fps == 0: return 0\n    s = 1 if n_frames_at_30fps > 0 else -1\n    mag = max(1, int(round(abs(n_frames_at_30fps) * float(fps) / ref)))\n    return s * mag\n\ndef _fps_from_meta(meta_df, fallback_lookup, default_fps=30.0):\n    if 'frames_per_second' in meta_df.columns and pd.notnull(meta_df['frames_per_second']).any():\n        return float(meta_df['frames_per_second'].iloc[0])\n    vid = meta_df['video_id'].iloc[0]\n    return float(fallback_lookup.get(vid, default_fps))\n\n# -----------------------------------------------------------------------\n# 2. FEATURE GENERATORS (Physics & Motion)\n# -----------------------------------------------------------------------\n\ndef add_curvature_features(X, center_x, center_y, fps):\n    vel_x = center_x.diff()\n    vel_y = center_y.diff()\n    acc_x = vel_x.diff()\n    acc_y = vel_y.diff()\n    \n    # Curvature: độ cong của quỹ đạo\n    cross_prod = vel_x * acc_y - vel_y * acc_x\n    vel_mag = np.sqrt(vel_x**2 + vel_y**2)\n    curvature = np.abs(cross_prod) / (vel_mag**3 + 1e-6)\n\n    for w in [30, 60]:\n        ws = _scale(w, fps)\n        X[f'curv_mean_{w}'] = curvature.rolling(ws, min_periods=max(1, ws // 6)).mean()\n    return X\n\ndef add_multiscale_features(X, center_x, center_y, fps):\n    # Speed in cm/s (do pixel đã được normalized)\n    speed = np.sqrt(center_x.diff()**2 + center_y.diff()**2) * float(fps)\n\n    scales = [10, 40, 160]\n    for scale in scales:\n        ws = _scale(scale, fps)\n        if len(speed) >= ws:\n            X[f'sp_m{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).mean()\n            X[f'sp_s{scale}'] = speed.rolling(ws, min_periods=max(1, ws // 4)).std()\n    return X\n\ndef add_groom_microfeatures(X, df, fps):\n    \"\"\"Detect các chuyển động nhỏ đặc trưng của grooming\"\"\"\n    parts = df.columns.get_level_values(0)\n    if 'body_center' not in parts or 'nose' not in parts: return X\n\n    cx, cy = df['body_center']['x'], df['body_center']['y']\n    nx, ny = df['nose']['x'], df['nose']['y']\n\n    cs = (np.sqrt(cx.diff()**2 + cy.diff()**2) * float(fps)).fillna(0)\n    ns = (np.sqrt(nx.diff()**2 + ny.diff()**2) * float(fps)).fillna(0)\n\n    w30 = _scale(30, fps)\n    # Tỷ lệ chuyển động mũi / thân (Grooming: Mũi động, thân tĩnh)\n    X['head_body_decouple'] = (ns / (cs + 1e-3)).clip(0, 10).rolling(w30, min_periods=1).median()\n    \n    # Bán kính mũi so với thân (biến động khi rúc đầu)\n    r = np.sqrt((nx - cx)**2 + (ny - cy)**2)\n    X['nose_rad_std'] = r.rolling(w30, min_periods=1).std().fillna(0)\n    return X\n\ndef add_interaction_features(X, mouse_pair, avail_A, avail_B, fps):\n    \"\"\"Tính các feature xã hội quan trọng: Chase, Approach\"\"\"\n    if 'body_center' not in avail_A or 'body_center' not in avail_B: return X\n\n    # Vị trí tương đối\n    rel_x = mouse_pair['A']['body_center']['x'] - mouse_pair['B']['body_center']['x']\n    rel_y = mouse_pair['A']['body_center']['y'] - mouse_pair['B']['body_center']['y']\n    rel_dist = np.sqrt(rel_x**2 + rel_y**2)\n\n    # Vận tốc\n    A_vx = mouse_pair['A']['body_center']['x'].diff()\n    A_vy = mouse_pair['A']['body_center']['y'].diff()\n    B_vx = mouse_pair['B']['body_center']['x'].diff()\n    B_vy = mouse_pair['B']['body_center']['y'].diff()\n\n    # Ai đang dẫn đầu? (Leader-Follower dynamics)\n    # Chiếu vận tốc lên vector nối tâm\n    A_lead = (A_vx * rel_x + A_vy * rel_y) / (np.sqrt(A_vx**2 + A_vy**2) * rel_dist + 1e-6)\n    B_lead = (B_vx * (-rel_x) + B_vy * (-rel_y)) / (np.sqrt(B_vx**2 + B_vy**2) * rel_dist + 1e-6)\n\n    # Chase metric: Khoảng cách giảm VÀ B đang dẫn đầu (A đuổi B)\n    approach = -rel_dist.diff()\n    chase = approach * B_lead\n    \n    ws = _scale(30, fps)\n    X[f'chase_{30}'] = chase.rolling(ws, min_periods=1).mean()\n    \n    # Correlation vận tốc (Di chuyển cùng nhau?)\n    ws_cor = _scale(60, fps)\n    A_sp = np.sqrt(A_vx**2 + A_vy**2)\n    B_sp = np.sqrt(B_vx**2 + B_vy**2)\n    X[f'sp_cor{60}'] = A_sp.rolling(ws_cor, min_periods=1).corr(B_sp)\n\n    return X\n\n# -----------------------------------------------------------------------\n# 3. CORE TRANSFORM FUNCTIONS (GỌI TỪ FILE TRAIN/INFERENCE)\n# -----------------------------------------------------------------------\n\ndef transform_single(single_mouse, body_parts_tracked, fps):\n    \"\"\"Main Feature Engineering cho hành vi SINGLE\"\"\"\n    available_body_parts = single_mouse.columns.get_level_values(0)\n\n    # 1. Distances between all body parts\n    X = pd.DataFrame({\n        f\"{p1}+{p2}\": np.square(single_mouse[p1] - single_mouse[p2]).sum(axis=1, skipna=False)\n        for p1, p2 in itertools.combinations(body_parts_tracked, 2)\n        if p1 in available_body_parts and p2 in available_body_parts\n    })\n    \n    # 2. Body Angle\n    if all(p in available_body_parts for p in ['nose', 'body_center', 'tail_base']):\n        v1 = single_mouse['nose'] - single_mouse['body_center']\n        v2 = single_mouse['tail_base'] - single_mouse['body_center']\n        X['body_ang'] = (v1['x'] * v2['x'] + v1['y'] * v2['y']) / (\n            np.sqrt(v1['x']**2 + v1['y']**2) * np.sqrt(v2['x']**2 + v2['y']**2) + 1e-6)\n\n    # 3. Motion Stats (FPS-scaled)\n    if 'body_center' in available_body_parts:\n        cx, cy = single_mouse['body_center']['x'], single_mouse['body_center']['y']\n        \n        for w in [5, 15, 30, 60]:\n            ws = _scale(w, fps)\n            roll = dict(min_periods=1, center=True)\n            X[f'cx_m{w}'] = cx.rolling(ws, **roll).mean()\n            X[f'act{w}'] = np.sqrt(cx.diff().rolling(ws, min_periods=1).var() +\n                                   cy.diff().rolling(ws, min_periods=1).var()) # Activity level\n\n        # Advanced Physics\n        X = add_curvature_features(X, cx, cy, fps)\n        X = add_multiscale_features(X, cx, cy, fps)\n        X = add_groom_microfeatures(X, single_mouse, fps)\n\n    # 4. Lagged features (Duration-aware)\n    if all(p in available_body_parts for p in ['nose', 'tail_base']):\n        nt_dist = np.sqrt((single_mouse['nose']['x'] - single_mouse['tail_base']['x'])**2 +\n                          (single_mouse['nose']['y'] - single_mouse['tail_base']['y'])**2)\n        for lag in [10, 20]:\n            l = _scale(lag, fps)\n            X[f'nt_df{lag}'] = nt_dist - nt_dist.shift(l) # Elongation change (Stretch/Compress)\n\n    return X.astype(np.float32, copy=False)\n\n\ndef transform_pair(mouse_pair, body_parts_tracked, fps):\n    \"\"\"Main Feature Engineering cho hành vi PAIR\"\"\"\n    avail_A = mouse_pair['A'].columns.get_level_values(0)\n    avail_B = mouse_pair['B'].columns.get_level_values(0)\n\n    # 1. Cross-Mouse Distances\n    X = pd.DataFrame({\n        f\"12+{p1}+{p2}\": np.square(mouse_pair['A'][p1] - mouse_pair['B'][p2]).sum(axis=1, skipna=False)\n        for p1, p2 in itertools.product(body_parts_tracked, repeat=2)\n        if p1 in avail_A and p2 in avail_B\n    })\n\n    # 2. Nose-Nose Dynamics (Quan trọng nhất cho Sniff/Attack)\n    if 'nose' in avail_A and 'nose' in avail_B:\n        nn = np.sqrt((mouse_pair['A']['nose']['x'] - mouse_pair['B']['nose']['x'])**2 +\n                     (mouse_pair['A']['nose']['y'] - mouse_pair['B']['nose']['y'])**2)\n        for lag in [10, 20]:\n            l = _scale(lag, fps)\n            X[f'nn_ch{lag}']  = nn - nn.shift(l) # Đang đến gần nhau hay ra xa?\n            is_cl = (nn < 10.0).astype(float)\n            X[f'cl_ps{lag}']  = is_cl.rolling(l, min_periods=1).mean() # Thời gian tiếp xúc gần\n\n    # 3. Advanced Interaction\n    X = add_interaction_features(X, mouse_pair, avail_A, avail_B, fps)\n\n    # 4. Relative Orientation\n    if all(p in avail_A for p in ['nose', 'tail_base']) and all(p in avail_B for p in ['nose', 'tail_base']):\n        dir_A = mouse_pair['A']['nose'] - mouse_pair['A']['tail_base']\n        dir_B = mouse_pair['B']['nose'] - mouse_pair['B']['tail_base']\n        # Cosine similarity giữa 2 vector thân\n        X['rel_ori'] = (dir_A['x'] * dir_B['x'] + dir_A['y'] * dir_B['y']) / (\n            np.sqrt(dir_A['x']**2 + dir_A['y']**2) * np.sqrt(dir_B['x']**2 + dir_B['y']**2) + 1e-6)\n\n    return X.astype(np.float32, copy=False)\n\n# -----------------------------------------------------------------------\n# 4. DATA GENERATOR (Parser thông minh)\n# -----------------------------------------------------------------------\n\ndef generate_mouse_data(dataset, traintest, traintest_directory=None, generate_single=True, generate_pair=True):\n    \"\"\"\n    Iterator thông minh: Đọc file tracking -> Tách luồng Single/Pair -> Yield data\n    \"\"\"\n    assert traintest in ['train', 'test']\n    if traintest_directory is None:\n        traintest_directory = f\"/kaggle/input/MABe-mouse-behavior-detection/{traintest}_tracking\"\n\n    # Regex helper để resolve mouse_id (vấn đề đau đầu nhất của dataset này)\n    def _to_num(x):\n        if isinstance(x, (int, np.integer)): return int(x)\n        m = re.search(r'(\\d+)$', str(x))\n        return int(m.group(1)) if m else None\n\n    drop_body_parts = ['spine_1', 'spine_2', 'tail_middle_1', 'tail_middle_2', 'tail_midpoint'] # Giảm nhiễu\n\n    for _, row in dataset.iterrows():\n        lab_id, video_id = row.lab_id, row.video_id\n        fps = float(row.frames_per_second)\n\n        # Skip nếu video không có label string (lỗi dataset)\n        if not isinstance(row.behaviors_labeled, str) and traintest == 'train': continue\n\n        # --- LOAD TRACKING ---\n        path = f\"{traintest_directory}/{lab_id}/{video_id}.parquet\"\n        try:\n            vid = pd.read_parquet(path)\n        except: continue\n            \n        # Pivot table để chuyển thành dạng [frame] x [mouse_id] x [body_part]\n        pvid = vid.pivot(columns=['mouse_id','bodypart'], index='video_frame', values=['x','y'])\n        del vid\n        pvid = pvid.reorder_levels([1,2,0], axis=1).T.sort_index().T\n        pvid = (pvid / float(row.pix_per_cm_approx)).astype('float32', copy=False) # Normalize sang cm\n\n        avail = list(pvid.columns.get_level_values('mouse_id').unique())\n        avail_set = set(avail) | set(map(str, avail)) | {f\"mouse{_to_num(a)}\" for a in avail if _to_num(a) is not None}\n\n        # Hàm map ID trong label (vd: \"mouse1\") sang ID trong tracking (vd: 0, 1)\n        def _resolve(agent_str):\n            m = re.search(r'(\\d+)$', str(agent_str))\n            cand = [agent_str]\n            if m:\n                n = int(m.group(1))\n                cand = [n, n-1, str(n), f\"mouse{n}\", agent_str]\n            for c in cand:\n                if c in avail_set:\n                    if c in set(avail): return c\n                    for a in avail:\n                        if str(a) == str(c) or f\"mouse{_to_num(a)}\" == str(c): return a\n            return None\n\n        # --- PARSE LABELS ---\n        if traintest == 'train':\n            annot = pd.read_parquet(path.replace('train_tracking', 'train_annotation'))\n        \n        # Parse behaviors_labeled json string\n        try:\n            vb = json.loads(row.behaviors_labeled if traintest == 'train' else row.behaviors_labeled)\n        except:\n            vb = []\n        vb = sorted(list({b.replace(\"'\", \"\") for b in vb}))\n        vb = pd.DataFrame([b.split(',') for b in vb], columns=['agent','target','action'])\n        if vb.empty: continue\n        \n        # Meta info\n        def _mk_meta(index, agent_id, target_id):\n            return pd.DataFrame({\n                'video_id': video_id, 'agent_id': agent_id, 'target_id': target_id,\n                'video_frame': index.astype('int32'), 'frames_per_second': np.float32(fps)\n            })\n\n        # --- YIELD SINGLE DATA ---\n        if generate_single:\n            vb_single = vb.query(\"target == 'self'\")\n            for agent_str in pd.unique(vb_single['agent']):\n                col_lab = _resolve(agent_str)\n                if col_lab is None: continue\n                \n                actions = sorted(vb_single.loc[vb_single['agent'].eq(agent_str), 'action'].unique().tolist())\n                if not actions: continue\n\n                single = pvid.loc[:, col_lab]\n                meta_df = _mk_meta(single.index, agent_str, 'self')\n\n                if traintest == 'train':\n                    # Tạo Binary Labels cho từng frame\n                    y = pd.DataFrame(False, index=single.index.astype('int32'), columns=actions)\n                    a_num = _to_num(col_lab)\n                    a_sub = annot.query(\"(agent_id == @a_num) & (target_id == @a_num)\")\n                    for i in range(len(a_sub)):\n                        ar = a_sub.iloc[i]\n                        a = str(ar.action).lower()\n                        if a in y.columns:\n                            y.loc[int(ar['start_frame']):int(ar['stop_frame']), a] = True\n                    yield 'single', single, meta_df, y\n                else:\n                    yield 'single', single, meta_df, actions\n\n        # --- YIELD PAIR DATA ---\n        if generate_pair:\n            vb_pair = vb.query(\"target != 'self'\")\n            if len(vb_pair) > 0:\n                allowed_pairs = set(map(tuple, vb_pair[['agent','target']].itertuples(index=False, name=None)))\n                \n                # Permutation tất cả các cặp có thể\n                for agent_num, target_num in itertools.permutations(np.unique(pvid.columns.get_level_values('mouse_id')), 2):\n                    agent_str = f\"mouse{_to_num(agent_num)}\"\n                    target_str = f\"mouse{_to_num(target_num)}\"\n                    \n                    if (agent_str, target_str) not in allowed_pairs: continue\n\n                    a_col = _resolve(agent_str)\n                    b_col = _resolve(target_str)\n                    if a_col is None or b_col is None: continue\n\n                    actions = sorted(vb_pair.query(\"(agent == @agent_str) & (target == @target_str)\")['action'].unique().tolist())\n                    if not actions: continue\n\n                    pair_xy = pd.concat([pvid[a_col], pvid[b_col]], axis=1, keys=['A','B'])\n                    meta_df = _mk_meta(pair_xy.index, agent_str, target_str)\n\n                    if traintest == 'train':\n                        a_num = _to_num(a_col); b_num = _to_num(b_col)\n                        y = pd.DataFrame(False, index=pair_xy.index.astype('int32'), columns=actions)\n                        a_sub = annot.query(\"(agent_id == @a_num) & (target_id == @b_num)\")\n                        for i in range(len(a_sub)):\n                            ar = a_sub.iloc[i]\n                            a = str(ar.action).lower()\n                            if a in y.columns:\n                                y.loc[int(ar['start_frame']):int(ar['stop_frame']), a] = True\n                        yield 'pair', pair_xy, meta_df, y\n                    else:\n                        yield 'pair', pair_xy, meta_df, actions\n\nprint(\"✅ Đã load xong các hàm Feature Engineering (Phiên bản Extra-Trees GPU).\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-12-02T14:44:02.191566Z","iopub.execute_input":"2025-12-02T14:44:02.191882Z","iopub.status.idle":"2025-12-02T14:44:04.299532Z","shell.execute_reply.started":"2025-12-02T14:44:02.191858Z","shell.execute_reply":"2025-12-02T14:44:04.298716Z"}},"outputs":[],"execution_count":null}]}