{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":59156,"databundleVersionId":13874099,"sourceType":"competition"},{"sourceId":13391780,"sourceType":"datasetVersion","datasetId":8497698}],"dockerImageVersionId":31153,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Dataset Creation","metadata":{}},{"cell_type":"markdown","source":"#### The codes below were used to generate the datasets for each lab_id and for each mouse. The new data created in more human-readable and more easier to feed into model. The links for the Datasets is provided below :-\n#### https://www.kaggle.com/datasets/kushubhai/mabe-structured-dataset\n\n#### Also check out my visualization notebook for the competition :-\n#### https://www.kaggle.com/code/kushubhai/mabe-visualization","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport polars as pl\nimport json\nimport os\nimport gc\nimport itertools\nfrom functools import reduce\nfrom collections import defaultdict\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport matplotlib.animation as animation\nimport seaborn as sns\nfrom IPython.display import HTML\npd.options.display.max_columns = 100\npd.options.display.max_rows = 100\ntqdm.pandas()\nBASE_PATH = \"/kaggle/input/MABe-mouse-behavior-detection/\"\nTRAIN_TRACKING_DIR = os.path.join(BASE_PATH, \"train_tracking\")\nTEST_TRACKING_DIR = os.path.join(BASE_PATH, \"test_tracking\")\nANNOTATION_DIR = os.path.join(BASE_PATH, \"train_annotation\")\ntrain_df = pd.read_csv(os.path.join(BASE_PATH, \"train.csv\"))\ntest_df = pd.read_csv(os.path.join(BASE_PATH, \"test.csv\"))\nsample_submission_df = pd.read_csv(os.path.join(BASE_PATH, \"sample_submission.csv\"))\n\nOUTPUT_PATH = \"/kaggle/working/\"\nprint(\"ENVIRONMENT SETUP COMPLETE!\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-17T14:45:01.502521Z","iopub.execute_input":"2025-10-17T14:45:01.502854Z","iopub.status.idle":"2025-10-17T14:45:01.597683Z","shell.execute_reply.started":"2025-10-17T14:45:01.502830Z","shell.execute_reply":"2025-10-17T14:45:01.596586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = train_df[~train_df['lab_id'].str.startswith('MABe22_')].reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-17T14:45:01.599296Z","iopub.execute_input":"2025-10-17T14:45:01.599722Z","iopub.status.idle":"2025-10-17T14:45:01.609680Z","shell.execute_reply.started":"2025-10-17T14:45:01.599691Z","shell.execute_reply":"2025-10-17T14:45:01.608441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df[\"lab_id\"].value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-17T14:45:01.610629Z","iopub.execute_input":"2025-10-17T14:45:01.610965Z","iopub.status.idle":"2025-10-17T14:45:01.642593Z","shell.execute_reply.started":"2025-10-17T14:45:01.610940Z","shell.execute_reply":"2025-10-17T14:45:01.641372Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def process_mouse_dataframe(df, bodyparts):\n    \"\"\"Pivots, cleans, and vectorizes a dataframe for a single mouse.\"\"\"\n    \n    pivoted_df = df.pivot_table(\n        index='video_frame',\n        columns='bodypart',\n        values=['x', 'y']\n    )\n\n    pivoted_df.columns = [f'{col[1]}_{col[0]}' for col in pivoted_df.columns]\n    pivoted_df = pivoted_df.reset_index()\n\n    return pivoted_df\n\nfor _, row in tqdm(train_df.iterrows(), total=len(train_df)):\n    train_tracking_path = os.path.join(TRAIN_TRACKING_DIR, row['lab_id'], f\"{row['video_id']}.parquet\")\n    FINAL_OUTPUT_PATH = os.path.join(OUTPUT_PATH, row[\"lab_id\"])\n    os.makedirs(FINAL_OUTPUT_PATH, exist_ok=True)\n    try:\n        train_tracking_df = pd.read_parquet(train_tracking_path)\n        \n        # Find common bodyparts across all mice present in this file\n        mouse_bodyparts_combined = [\n            group['bodypart'].unique() for _, group in train_tracking_df.groupby('mouse_id')\n        ]\n        if not mouse_bodyparts_combined:\n            continue # Skip file if no data\n        common_bodyparts = reduce(np.intersect1d, mouse_bodyparts_combined)\n        \n        # Dynamically process for each mouse in the file\n        for mouse_id, mouse_df in train_tracking_df.groupby('mouse_id'):\n            MOUSE_OUTPUT_PATH = os.path.join(FINAL_OUTPUT_PATH, f\"Mouse_{mouse_id}\")\n            os.makedirs(MOUSE_OUTPUT_PATH, exist_ok=True)\n            if mouse_id in [1,2,3,4]:\n                # Call our single processing function\n                processed_df = process_mouse_dataframe(mouse_df, common_bodyparts)\n                parquet_file_format = f\"{row['video_id']}_new.parquet\"\n                parquet_file_path = os.path.join(MOUSE_OUTPUT_PATH, parquet_file_format)\n                processed_df.to_parquet(parquet_file_path)\n\n    except FileNotFoundError:\n        print(f\"File not found for video {row['video_id']}. Skipping.\")\n        pass","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-17T14:45:01.644274Z","iopub.execute_input":"2025-10-17T14:45:01.644782Z","iopub.status.idle":"2025-10-17T14:49:19.587666Z","shell.execute_reply.started":"2025-10-17T14:45:01.644751Z","shell.execute_reply":"2025-10-17T14:49:19.586265Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}