{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **EDA Link Notes**\n\nhttp://www.kaggle.com/code/neevvv/edanote/notebook","metadata":{}},{"cell_type":"markdown","source":"# Load Data","metadata":{}},{"cell_type":"code","source":"#Preprocessing Preparation\nimport pandas as pd\nimport numpy as np\nimport pyarrow\nimport random\nimport glob\nimport os\nfrom sklearn.model_selection import GroupKFold\nfrom sklearn.preprocessing import LabelEncoder\n\npd.set_option('display.max_columns', None)\n\nprint(\"Success\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-11-09T11:47:53.095407Z","iopub.execute_input":"2025-11-09T11:47:53.095727Z","iopub.status.idle":"2025-11-09T11:47:54.239729Z","shell.execute_reply.started":"2025-11-09T11:47:53.095705Z","shell.execute_reply":"2025-11-09T11:47:54.238776Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"path_in = \"/kaggle/input/MABe-mouse-behavior-detection/\"\n\nmetadata = os.path.join(path_in, \"train.csv\")\nannotations = os.path.join(path_in, \"train_annotation/\")\ntrain_tracking = os.path.join(path_in, \"train_tracking/\")\n\npattern_all_file = os.path.join(annotations, \"*\")\ncontent_ann = glob.glob(pattern_all_file)\n\nprint(\"\\nHasil Pengecekan Glob\")\nif len(content_ann) > 0:\n    print(f\"Brhasil menemukan {len(content_ann)} item: \")\n    for item in content_ann[:10]:\n        print(item)\nelse:\n    print(\"Warning: Folder kosong\")\n\n#Konfigurasi Kolom\ntarget = 'action'\ngroup = 'video_id'\n\n#Konfirmasi Path\nprint(f\"Path Metadata: {metadata}\")\nprint(f\"Path Folder Anotasi: {annotations}\")\nprint(f\"Path Folder Train Tracking: {train_tracking}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T11:47:57.965723Z","iopub.execute_input":"2025-11-09T11:47:57.966161Z","iopub.status.idle":"2025-11-09T11:47:58.001325Z","shell.execute_reply.started":"2025-11-09T11:47:57.966138Z","shell.execute_reply":"2025-11-09T11:47:58.000064Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Load Data Metadata\ndf_meta = pd.read_csv(metadata)\nprint(\"Load file metadata (train.csv) \")\n\nprint(f\"Bentuk data metadata: {df_meta.shape}\")\nprint(\"Example data Metadata: \")\ndf_meta.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T11:48:03.486073Z","iopub.execute_input":"2025-11-09T11:48:03.486479Z","iopub.status.idle":"2025-11-09T11:48:03.678292Z","shell.execute_reply.started":"2025-11-09T11:48:03.486453Z","shell.execute_reply":"2025-11-09T11:48:03.677169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#Load Data Annotations\npattern_subfol = os.path.join(annotations, \"*/\")\nlist_subfol = glob.glob(pattern_subfol)\n\nprint(f\"Found {len(list_subfol)} Sub-Folder Annotations: \")\n\nall_files = []\nfor folder in list_subfol:\n    print(f\" - {folder.split('/')[-2]}\")\n\n\nprint(\"\\nTake 3 random files from each subfolder.\")\nfor subfol in list_subfol:\n    pattern_parquet = os.path.join(subfol, \"*.parquet\")\n    parquet_files = glob.glob(pattern_parquet)\n\n    if len(parquet_files) > 0:\n        n_files = min(len(parquet_files), 3)\n        sample_files = random.sample(parquet_files, n_files)\n        all_files.extend(sample_files)\n        print(f\" - Take {len(sample_files)} files from {subfol.split('/')[-2]} skip\")\n    else:\n        print(f\" - There are no .parquet files in {subfol.splt('/')[-2]}, skip\")\n\nprint(f\"\\nTotal files to be loaded from all folders: {len(all_files)}\")\n\nlist_dfs = []\nfor f in all_files:\n    # list_dfs.append(pd.read_parquet(f))\n    df_temp = pd.read_parquet(f)\n\n    name_file = os.path.basename(f)\n    video_id = os.path.splitext(name_file)[0]\n    df_temp[group] = video_id\n    list_dfs.append(df_temp)\n\nif list_dfs:\n    df_labels = pd.concat(list_dfs, ignore_index=True)\n\n    print(f\"\\nCombined annotation data format (target): {df_labels.shape}\")\n    print(\"Example data annotations (target) after add 'video_id': \")\n    df_labels.head()\n\n    print(\"\\nDistribusi 'action' (target) kita (versi teks): \")\n    print(df_labels[target].value_counts(normalize=True).mul(100).round(2).astype(str) + '%')\nelse:\n    print(\"\\nERROR: No annotation data loaded\")\n    print(\"Check whether the annotation folder contains Parquet files.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T11:48:10.933303Z","iopub.execute_input":"2025-11-09T11:48:10.933683Z","iopub.status.idle":"2025-11-09T11:48:11.951557Z","shell.execute_reply.started":"2025-11-09T11:48:10.933658Z","shell.execute_reply":"2025-11-09T11:48:11.950411Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Preprocessin & Feature Engneering","metadata":{}},{"cell_type":"code","source":"df_process = df_labels.copy()\n\nprint(\"\\nCombining metadata (df_meta) into the label data.\")\n\nif group not in df_process.columns:\n    print(f\"ERROR: Column ‘{group}’ (‘video_id’) not found in label data\")\n    print(\"Unable to merge metadata. Check your parquet file.\")\nelse:\n    meta_features = [\n        group,\n        'frames_per_second',\n        'arena_type',\n        'mouse1_sex',\n        'arena_shape'\n    ]\n    df_meta_subset = df_meta[meta_features].drop_duplicates().copy()\n\n    df_process[group] = df_process[group].astype(str)\n    df_meta_subset[group] = df_meta_subset[group].astype(str)\n    \n    df_process = pd.merge(\n        df_process,\n        df_meta_subset,\n        on=group,\n        how='left'\n    )\n    print('Merge Metadata Success')\n\ndf_process['duration_frames'] = df_process['stop_frame'] - df_process['start_frame']\n\n#Normalisazi FPS\nif 'frames_per_second' in df_process.columns:\n    df_process['duration_seconds'] = df_process['duration_frames'] / df_process['frames_per_second']\nelse:\n    print(\"Warning: The ‘frames_per_second’ column was not found. ‘duration_seconds’ was not created.\")\n\n#CHECK NaN\nnan_counts = df_process.isnull().sum()\nnan_counts_report = nan_counts[nan_counts > 0]\ntotal_nan = nan_counts.sum()\nprint(f\"Total jumlah NaN yang ditemukan: {total_nan}\")\n\nif len(nan_counts_report) > 0:\n    print(\"NaN FOUND IN COLUMNS: \")\n    print(nan_counts_report)\nelse:\n    print(\"Check Success: NaN Not Found\")\n\n#Missing Values\nif 'target_id' in df_process.columns and df_process['target_id'].isnull().any():\n    df_process['target_id'] = df_process['target_id'].fillna('None')\n    print(\"NaN in 'target_id' filled with 'None'.\")\n\n#Duratioin Seconds\nif 'duration_seconds' in df_process.columns and df_process['duration_seconds'].isnull().any():\n    median_duration = df_process['duration_seconds'].median()\n    df_process['duration_seconds'] = df_process['duration_seconds'].fillna(median_duration)\n    print(f\"NaN in 'duration_seconds' filled with the median: {median_duration:.2f}\")\n\n\nfor col in ['arena_type', 'mouse1_sex', 'arena_shape']:\n    if col in df_process.columns and df_process[col].isnull().any():\n        mode_value = df_process[col].mode()[0]\n        df_process[col] = df_process[col].fillna(mode_value)\n        print(f\"NaN in '{col}' filled with modus: {mode_value}\")\n\n#ENCODING\nencoders = {}\n\n#Categori (X)\ncategorical_features = ['agent_id', 'target_id', 'arena_type', 'mouse1_sex', 'arena_shape']\n\nfor col in categorical_features:\n    if col in df_process.columns:\n        le = LabelEncoder()\n        df_process[f'{col}_encoded'] = le.fit_transform(df_process[col])\n        encoders[col] = le\n        print(f\"Columns '{col}' in Encode.\")\n\n#Target (Y)\nle_action = LabelEncoder()\ndf_process[f'{target}_encoded'] = le_action.fit_transform(df_process[target])\nencoders[target] = le_action\nprint(f\"Column Target '{target}_encoded' in Encode \")\n\ndf_process.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T12:27:45.963466Z","iopub.execute_input":"2025-11-09T12:27:45.964666Z","iopub.status.idle":"2025-11-09T12:27:46.025857Z","shell.execute_reply.started":"2025-11-09T12:27:45.964630Z","shell.execute_reply":"2025-11-09T12:27:46.024768Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# GROUP K-FOLD","metadata":{}},{"cell_type":"code","source":"feature_columns = [\n    'start_frame',\n    'stop_frame',\n    'duration_frames',\n    'duration_seconds',\n    'agent_id_encoded',\n    'target_id_encoded',\n    'arena_type_encoded',\n    'mouse1_sex_encoded',\n    'arena_shape_encoded'\n]\n\nfinal_features = [col for col in feature_columns if col in df_process.columns]\n\nif len(final_features) != len(feature_columns):\n    print(\"Warning: Some feature columns are not found in df_processed \")\n\nprint(f\"Features to be used (X): {final_features}\")\n\n#separate X, Y, and Groups\nx = df_process[final_features]\ny = df_process[f\"{target}_encoded\"]\ngroups = df_process[group]\n\nprint(f\"\\nX Shape (feature): {x.shape}\")\nprint(f\"Y Shape (target): {y.shape}\")\nprint(f\"Number of unique groups{group}: {groups.nunique()}\")\n\nif x.isnull().values.any():\n    print(\"\\nWARNING: There are still NaN values in the feature data (X)!.\")\n    print(x.isnull().sum())\nelse:\n    print(\"\\nCHCEK SUCCESS: There are no NaN values in the feature data (X).\")\n\nN_Splits = 5\ngkf = GroupKFold(n_splits=N_Splits)\n\n#VERIFIKASI DATA (GROUP K-FOLD)\nfor fold, (train_idx, val_idx) in enumerate(gkf.split(x, y, groups)):\n    print(f\"\\n=== FOLD {fold+1} ===\")\n    print(f\"Amount of training data: {len(train_idx)}\")\n    print(f\"Amount of validation data: {len(val_idx)}\")\n\n    train_videos = groups.iloc[train_idx].unique()\n    val_videos = groups.iloc[val_idx].unique()\n\n    overlap = np.intersect1d(train_videos, val_videos) #Search Irisan\n\n    if len(overlap) == 0:\n        print(f\"CHECK SUCCESS: There are no overlapping '{group}' in fold {fold+1}.\")\n    else:\n        print(f\"ERROR: '{GROUP_COL}' tumpang tindih! Cek kembali logika grup.\")\n\n    break","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-11-09T11:58:19.869775Z","iopub.execute_input":"2025-11-09T11:58:19.870796Z","iopub.status.idle":"2025-11-09T11:58:19.890395Z","shell.execute_reply.started":"2025-11-09T11:58:19.870763Z","shell.execute_reply":"2025-11-09T11:58:19.889380Z"}},"outputs":[],"execution_count":null},{"cell_type":"raw","source":"","metadata":{}}]}