{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# ============================================================\n# ACME\n# Adaptive Cognitive Memory Engine\n# Using EEG-Derived Cognitive States\n# ============================================================\n\nprint(\"ACME Project Started 🚀\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:40:59.865945Z","iopub.execute_input":"2026-09-27T13:40:59.866327Z","iopub.status.idle":"2026-09-27T13:40:59.876796Z","shell.execute_reply.started":"2026-09-27T13:40:59.866294Z","shell.execute_reply":"2026-09-27T13:40:59.875447Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Install libraries required for ACME\n\n%pip install -q mne sentence-transformers faiss-cpu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:41:52.687828Z","iopub.execute_input":"2026-09-27T13:41:52.688168Z","iopub.status.idle":"2026-09-27T13:43:11.906229Z","shell.execute_reply.started":"2026-09-27T13:41:52.688138Z","shell.execute_reply":"2026-09-27T13:43:11.904698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy\nimport pandas\nimport scipy\nimport sklearn\nimport torch\n\nprint(\"NumPy:\", numpy.__version__)\nprint(\"Pandas:\", pandas.__version__)\nprint(\"SciPy:\", scipy.__version__)\nprint(\"Scikit-learn:\", sklearn.__version__)\nprint(\"PyTorch:\", torch.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:43:18.40062Z","iopub.execute_input":"2026-09-27T13:43:18.401924Z","iopub.status.idle":"2026-09-27T13:43:27.85003Z","shell.execute_reply.started":"2026-09-27T13:43:18.401874Z","shell.execute_reply":"2026-09-27T13:43:27.848846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Check ACME-specific libraries\n\npackages = {\n \"MNE\": \"mne\",\n \"Sentence Transformers\": \"sentence_transformers\",\n \"FAISS\": \"faiss\"\n}\n\nfor name, package in packages.items():\n try:\n module = __import__(package)\n version = getattr(module, \"__version__\", \"version not available\")\n print(f\"✓ {name}: {version}\")\n except Exception as e:\n print(f\"✗ {name}: NOT AVAILABLE\")\n print(\" Reason:\", str(e).split(\"\\n\")[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:44:55.15381Z","iopub.execute_input":"2026-09-27T13:44:55.155674Z","iopub.status.idle":"2026-09-27T13:45:47.423885Z","shell.execute_reply.started":"2026-09-27T13:44:55.155624Z","shell.execute_reply":"2026-09-27T13:45:47.422363Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import sentence_transformers\n\nprint(\"Sentence Transformers:\", sentence_transformers.__version__)\nprint(\"✅ Sentence Transformers is ready!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:45:47.435167Z","iopub.execute_input":"2026-09-27T13:45:47.435577Z","iopub.status.idle":"2026-09-27T13:45:47.47899Z","shell.execute_reply.started":"2026-09-27T13:45:47.435529Z","shell.execute_reply":"2026-09-27T13:45:47.477842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import requests\n\ntry:\n response = requests.get(\"https://pypi.org\", timeout=10)\n print(\"Internet status:\", response.status_code)\n print(\"✅ Kaggle internet is working\")\nexcept Exception as e:\n print(\"❌ Kaggle internet is not available\")\n print(\"Reason:\", e)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:46:18.590903Z","iopub.execute_input":"2026-09-27T13:46:18.591808Z","iopub.status.idle":"2026-09-27T13:46:50.611059Z","shell.execute_reply.started":"2026-09-27T13:46:18.591764Z","shell.execute_reply":"2026-09-27T13:46:50.609612Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nprint(\"Files available in Kaggle input:\\n\")\n\nfor dirname, _, filenames in os.walk(\"/kaggle/input\"):\n for filename in filenames:\n print(os.path.join(dirname, filename))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:54:00.845743Z","iopub.execute_input":"2026-09-27T13:54:00.847164Z","iopub.status.idle":"2026-09-27T13:54:00.859571Z","shell.execute_reply.started":"2026-09-27T13:54:00.847091Z","shell.execute_reply":"2026-09-27T13:54:00.858361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nfile_path = \"/kaggle/input/datasets/samnikolas/eeg-dataset/features_raw.csv\"\n\ndf = pd.read_csv(file_path)\n\nprint(\"Dataset shape:\", df.shape)\n\nprint(\"\\nFirst 5 rows:\")\ndisplay(df.head())\n\nprint(\"\\nColumn names:\")\nprint(df.columns.tolist())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:54:32.654915Z","iopub.execute_input":"2026-09-27T13:54:32.655395Z","iopub.status.idle":"2026-09-27T13:54:32.870682Z","shell.execute_reply.started":"2026-09-27T13:54:32.65536Z","shell.execute_reply":"2026-09-27T13:54:32.869658Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect the last column\n\nprint(\"Last column name:\", df.columns[-1])\n\nprint(\"\\nUnique values:\")\nprint(df[\"Unnamed: 32\"].unique())\n\nprint(\"\\nValue counts:\")\nprint(df[\"Unnamed: 32\"].value_counts())\n\nprint(\"\\nData type:\")\nprint(df[\"Unnamed: 32\"].dtype)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:55:06.105327Z","iopub.execute_input":"2026-09-27T13:55:06.106214Z","iopub.status.idle":"2026-09-27T13:55:06.123682Z","shell.execute_reply.started":"2026-09-27T13:55:06.106174Z","shell.execute_reply":"2026-09-27T13:55:06.122509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Remove the completely empty column\n\ndf = df.drop(columns=[\"Unnamed: 32\"])\n\nprint(\"New dataset shape:\", df.shape)\nprint(\"Missing values:\", df.isna().sum().sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T13:55:34.566783Z","iopub.execute_input":"2026-09-27T13:55:34.567796Z","iopub.status.idle":"2026-09-27T13:55:34.584675Z","shell.execute_reply.started":"2026-09-27T13:55:34.567753Z","shell.execute_reply":"2026-09-27T13:55:34.583287Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nbase_path = \"/kaggle/input\"\n\nfor root, dirs, files in os.walk(base_path):\n if files:\n print(root)\n print(\"Files:\", files[:10])\n print(\"-\" * 60)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:03:51.744801Z","iopub.execute_input":"2026-09-27T14:03:51.745578Z","iopub.status.idle":"2026-09-27T14:03:51.832858Z","shell.execute_reply.started":"2026-09-27T14:03:51.745532Z","shell.execute_reply":"2026-09-27T14:03:51.831465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.io import loadmat\n\nfile_path = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection/EEG Data/eeg_record1.mat\"\n\ndata = loadmat(file_path)\n\nprint(\"Keys in file:\")\nprint(data.keys())\n\nfor key, value in data.items():\n if not key.startswith(\"__\"):\n print(\"\\nKey:\", key)\n print(\"Type:\", type(value))\n print(\"Shape:\", getattr(value, \"shape\", \"No shape\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:04:18.278322Z","iopub.execute_input":"2026-09-27T14:04:18.278847Z","iopub.status.idle":"2026-09-27T14:04:18.891305Z","shell.execute_reply.started":"2026-09-27T14:04:18.278806Z","shell.execute_reply":"2026-09-27T14:04:18.890026Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"o = data[\"o\"]\n\nprint(\"o dtype:\")\nprint(o.dtype)\n\nprint(\"\\nFields:\")\nprint(o.dtype.names)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:05:04.612779Z","iopub.execute_input":"2026-09-27T14:05:04.613195Z","iopub.status.idle":"2026-09-27T14:05:04.620025Z","shell.execute_reply.started":"2026-09-27T14:05:04.613163Z","shell.execute_reply":"2026-09-27T14:05:04.618609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for field in o.dtype.names:\n value = o[field][0, 0]\n print(f\"{field}: shape={getattr(value, 'shape', None)}, type={type(value)}\")\n print(\"Value:\", value if field in [\"id\", \"tag\", \"nS\", \"sampFreq\"] else \"\")\n print(\"-\" * 50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:05:29.827799Z","iopub.execute_input":"2026-09-27T14:05:29.828264Z","iopub.status.idle":"2026-09-27T14:05:29.836547Z","shell.execute_reply.started":"2026-09-27T14:05:29.82823Z","shell.execute_reply":"2026-09-27T14:05:29.835199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"marker = o[\"marker\"][0, 0]\ntrials = o[\"trials\"][0, 0]\n\nprint(\"Marker unique values:\")\nprint(np.unique(marker, return_counts=True))\n\nprint(\"\\nFirst 30 marker values:\")\nprint(marker[:30].flatten())\n\nprint(\"\\nTrials:\")\nprint(trials)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:05:51.241186Z","iopub.execute_input":"2026-09-27T14:05:51.242261Z","iopub.status.idle":"2026-09-27T14:05:51.253662Z","shell.execute_reply.started":"2026-09-27T14:05:51.24222Z","shell.execute_reply":"2026-09-27T14:05:51.25219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nmarker = o[\"marker\"][0, 0]\ntrials = o[\"trials\"][0, 0]\n\nprint(\"Marker unique values:\")\nprint(np.unique(marker, return_counts=True))\n\nprint(\"\\nFirst 30 marker values:\")\nprint(marker[:30].flatten())\n\nprint(\"\\nTrials shape:\")\nprint(trials.shape)\n\nprint(\"\\nTrials:\")\nprint(trials)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:06:14.431621Z","iopub.execute_input":"2026-09-27T14:06:14.433072Z","iopub.status.idle":"2026-09-27T14:06:14.455135Z","shell.execute_reply.started":"2026-09-27T14:06:14.432882Z","shell.execute_reply":"2026-09-27T14:06:14.454052Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\ndataset_path = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection\"\n\nfor root, dirs, files in os.walk(dataset_path):\n for file in files:\n if not file.endswith(\".mat\"):\n print(os.path.join(root, file))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:07:53.076903Z","iopub.execute_input":"2026-09-27T14:07:53.078249Z","iopub.status.idle":"2026-09-27T14:07:53.096099Z","shell.execute_reply.started":"2026-09-27T14:07:53.078192Z","shell.execute_reply":"2026-09-27T14:07:53.094951Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\ndataset_path = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection\"\n\nprint(os.listdir(dataset_path))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:08:21.247705Z","iopub.execute_input":"2026-09-27T14:08:21.248862Z","iopub.status.idle":"2026-09-27T14:08:21.263221Z","shell.execute_reply.started":"2026-09-27T14:08:21.248818Z","shell.execute_reply":"2026-09-27T14:08:21.261968Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\npath = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection/EEG Data\"\n\nprint(os.listdir(path))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:08:45.152844Z","iopub.execute_input":"2026-09-27T14:08:45.153392Z","iopub.status.idle":"2026-09-27T14:08:45.16403Z","shell.execute_reply.started":"2026-09-27T14:08:45.153354Z","shell.execute_reply":"2026-09-27T14:08:45.162818Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.io import loadmat\nimport os\nimport numpy as np\n\npath = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection/EEG Data\"\n\nfor i in range(1, 35):\n file_path = os.path.join(path, f\"eeg_record{i}.mat\")\n mat = loadmat(file_path)\n o = mat[\"o\"][0, 0]\n\n tag = o[\"tag\"][0, 0]\n\n print(f\"eeg_record{i}: tag shape = {tag.shape}, tag = {tag}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:09:07.162842Z","iopub.execute_input":"2026-09-27T14:09:07.167429Z","iopub.status.idle":"2026-09-27T14:09:07.517473Z","shell.execute_reply.started":"2026-09-27T14:09:07.167305Z","shell.execute_reply":"2026-09-27T14:09:07.515785Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.io import loadmat\nimport os\nimport numpy as np\n\npath = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection/EEG Data\"\n\nfor i in range(1, 35):\n file_path = os.path.join(path, f\"eeg_record{i}.mat\")\n mat = loadmat(file_path)\n o = mat[\"o\"][0, 0]\n\n tag = o[\"tag\"]\n\n print(f\"eeg_record{i}: tag shape = {tag.shape}, tag = {tag}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:09:28.682067Z","iopub.execute_input":"2026-09-27T14:09:28.682969Z","iopub.status.idle":"2026-09-27T14:09:43.987041Z","shell.execute_reply.started":"2026-09-27T14:09:28.682916Z","shell.execute_reply":"2026-09-27T14:09:43.986019Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"marker = o[\"marker\"]\n\npositions = np.where(marker.flatten() == 1)[0]\n\nprint(\"Number of markers:\", len(positions))\nprint(\"Marker positions:\")\nprint(positions)\n\nprint(\"\\nTime of markers (seconds):\")\nprint(positions / 128)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:10:25.17245Z","iopub.execute_input":"2026-09-27T14:10:25.174641Z","iopub.status.idle":"2026-09-27T14:10:25.188718Z","shell.execute_reply.started":"2026-09-27T14:10:25.174587Z","shell.execute_reply":"2026-09-27T14:10:25.187364Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.io import loadmat\nimport os\nimport numpy as np\n\npath = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection/EEG Data\"\n\nfor i in range(1, 35):\n file_path = os.path.join(path, f\"eeg_record{i}.mat\")\n mat = loadmat(file_path)\n o_i = mat[\"o\"][0, 0]\n\n data_i = o_i[\"data\"]\n marker_i = o_i[\"marker\"].flatten()\n\n marker_positions = np.where(marker_i == 1)[0]\n\n print(\n f\"Record {i:02d} | \"\n f\"samples={data_i.shape[0]} | \"\n f\"channels={data_i.shape[1]} | \"\n f\"markers={len(marker_positions)}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:10:57.762231Z","iopub.execute_input":"2026-09-27T14:10:57.762641Z","iopub.status.idle":"2026-09-27T14:11:09.811976Z","shell.execute_reply.started":"2026-09-27T14:10:57.762607Z","shell.execute_reply":"2026-09-27T14:11:09.810926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nbase = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection\"\n\nfor item in os.listdir(\"/kaggle/input/datasets/inancigdem\"):\n print(item)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:11:51.302443Z","iopub.execute_input":"2026-09-27T14:11:51.302878Z","iopub.status.idle":"2026-09-27T14:11:51.31056Z","shell.execute_reply.started":"2026-09-27T14:11:51.302843Z","shell.execute_reply":"2026-09-27T14:11:51.309368Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fs = 128\n\ndata_record = o[\"data\"]\n\ntotal_seconds = data_record.shape[0] / fs\ntotal_minutes = total_seconds / 60\n\nprint(\"Total samples:\", data_record.shape[0])\nprint(\"Sampling frequency:\", fs, \"Hz\")\nprint(\"Recording duration:\", round(total_minutes, 2), \"minutes\")\n\nprint(\"\\nLabel boundaries:\")\nprint(\"Focused : 0 - 10 minutes\")\nprint(\"Unfocused : 10 - 20 minutes\")\nprint(\"Drowsy : 20+ minutes\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:12:20.261023Z","iopub.execute_input":"2026-09-27T14:12:20.26152Z","iopub.status.idle":"2026-09-27T14:12:20.270469Z","shell.execute_reply.started":"2026-09-27T14:12:20.26148Z","shell.execute_reply":"2026-09-27T14:12:20.269065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\nfs = 128\nsegment_seconds = 60\nsamples_per_segment = fs * segment_seconds\n\ndata_record = o[\"data\"].astype(np.float32)\n\nsegments = []\nlabels = []\n\nfor start in range(0, data_record.shape[0] - samples_per_segment + 1, samples_per_segment):\n end = start + samples_per_segment\n segment = data_record[start:end]\n\n start_minute = start / fs / 60\n\n if start_minute < 10:\n label = \"Focused\"\n elif start_minute < 20:\n label = \"Unfocused\"\n else:\n label = \"Drowsy\"\n\n segments.append(segment)\n labels.append(label)\n\nsegments = np.array(segments)\n\nprint(\"Segments shape:\", segments.shape)\nprint(\"Number of segments:\", len(labels))\nprint(\"Label distribution:\")\n\nunique, counts = np.unique(labels, return_counts=True)\n\nfor label, count in zip(unique, counts):\n print(label, \":\", count)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:12:57.297127Z","iopub.execute_input":"2026-09-27T14:12:57.29789Z","iopub.status.idle":"2026-09-27T14:12:57.351693Z","shell.execute_reply.started":"2026-09-27T14:12:57.297852Z","shell.execute_reply":"2026-09-27T14:12:57.3504Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nfrom scipy.io import loadmat\nimport os\n\nfs = 128\nsegment_seconds = 60\nsamples_per_segment = fs * segment_seconds\n\npath = \"/kaggle/input/datasets/inancigdem/eeg-data-for-mental-attention-state-detection/EEG Data\"\n\nall_segments = []\nall_labels = []\nall_record_ids = []\n\nfor i in range(1, 35):\n\n file_path = os.path.join(path, f\"eeg_record{i}.mat\")\n mat = loadmat(file_path)\n o_i = mat[\"o\"][0, 0]\n\n eeg = o_i[\"data\"].astype(np.float32)\n\n for start in range(\n 0,\n eeg.shape[0] - samples_per_segment + 1,\n samples_per_segment\n ):\n\n end = start + samples_per_segment\n segment = eeg[start:end]\n\n start_minute = start / fs / 60\n\n if start_minute < 10:\n label = \"Focused\"\n elif start_minute < 20:\n label = \"Unfocused\"\n else:\n label = \"Drowsy\"\n\n all_segments.append(segment)\n all_labels.append(label)\n all_record_ids.append(i)\n\nall_segments = np.array(all_segments)\nall_labels = np.array(all_labels)\nall_record_ids = np.array(all_record_ids)\n\nprint(\"FULL DATASET\")\nprint(\"=\" * 40)\nprint(\"Segments shape:\", all_segments.shape)\nprint(\"Labels shape:\", all_labels.shape)\nprint(\"Records:\", len(np.unique(all_record_ids)))\n\nprint(\"\\nLabel distribution:\")\nunique, counts = np.unique(all_labels, return_counts=True)\n\nfor label, count in zip(unique, counts):\n print(f\"{label}: {count}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:13:25.164529Z","iopub.execute_input":"2026-09-27T14:13:25.166084Z","iopub.status.idle":"2026-09-27T14:13:38.60054Z","shell.execute_reply.started":"2026-09-27T14:13:25.16603Z","shell.execute_reply":"2026-09-27T14:13:38.599476Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from scipy.signal import welch\nimport numpy as np\n\n# EEG frequency bands\nbands = {\n \"delta\": (0.5, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n}\n\ndef extract_band_features(segment, fs=128):\n features = []\n\n # Calculate PSD for each EEG channel\n for channel in range(segment.shape[1]):\n\n signal = segment[:, channel]\n\n freqs, psd = welch(\n signal,\n fs=fs,\n nperseg=256\n )\n\n for low, high in bands.values():\n\n mask = (freqs >= low) & (freqs < high)\n\n power = np.mean(psd[mask])\n\n features.append(power)\n\n return features\n\n\n# Extract features for every EEG segment\nX = np.array([\n extract_band_features(segment)\n for segment in all_segments\n])\n\ny = all_labels\n\nprint(\"Feature matrix shape:\", X.shape)\nprint(\"Labels shape:\", y.shape)\nprint(\"Number of features per segment:\", X.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:14:09.451598Z","iopub.execute_input":"2026-09-27T14:14:09.452015Z","iopub.status.idle":"2026-09-27T14:16:41.367614Z","shell.execute_reply.started":"2026-09-27T14:14:09.451972Z","shell.execute_reply":"2026-09-27T14:16:41.366484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, classification_report\n\n# Split the EEG features\nX_train, X_test, y_train, y_test = train_test_split(\n X,\n y,\n test_size=0.2,\n random_state=42,\n stratify=y\n)\n\nprint(\"Training samples:\", X_train.shape[0])\nprint(\"Testing samples:\", X_test.shape[0])\n\n# Train classifier\nmodel = RandomForestClassifier(\n n_estimators=200,\n random_state=42,\n class_weight=\"balanced\"\n)\n\nmodel.fit(X_train, y_train)\n\n# Predictions\ny_pred = model.predict(X_test)\n\n# Results\naccuracy = accuracy_score(y_test, y_pred)\n\nprint(\"\\nEEG Cognitive-State Classification Accuracy:\")\nprint(f\"{accuracy:.4f}\")\n\nprint(\"\\nClassification Report:\")\nprint(classification_report(y_test, y_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:17:11.630813Z","iopub.execute_input":"2026-09-27T14:17:11.632521Z","iopub.status.idle":"2026-09-27T14:17:16.185913Z","shell.execute_reply.started":"2026-09-27T14:17:11.632468Z","shell.execute_reply":"2026-09-27T14:17:16.184197Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def cognitive_state_to_context(state):\n if state == \"Focused\":\n return {\n \"attention\": 0.90,\n \"workload\": 0.45,\n \"fatigue\": 0.15,\n \"engagement\": 0.90\n }\n\n elif state == \"Unfocused\":\n return {\n \"attention\": 0.40,\n \"workload\": 0.55,\n \"fatigue\": 0.40,\n \"engagement\": 0.45\n }\n\n elif state == \"Drowsy\":\n return {\n \"attention\": 0.20,\n \"workload\": 0.25,\n \"fatigue\": 0.85,\n \"engagement\": 0.20\n }\n\n\nfor state in [\"Focused\", \"Unfocused\", \"Drowsy\"]:\n print(state, \"→\", cognitive_state_to_context(state))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:18:03.589875Z","iopub.execute_input":"2026-09-27T14:18:03.591295Z","iopub.status.idle":"2026-09-27T14:18:03.602766Z","shell.execute_reply.started":"2026-09-27T14:18:03.591243Z","shell.execute_reply":"2026-09-27T14:18:03.600614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def calculate_cognitive_context(context):\n A = context[\"attention\"]\n W = context[\"workload\"]\n F = context[\"fatigue\"]\n E = context[\"engagement\"]\n\n alpha = 0.25\n beta = 0.25\n gamma = 0.25\n delta = 0.25\n\n C = (\n alpha * A\n + beta * E\n + gamma * (1 - W)\n + delta * (1 - F)\n )\n\n return round(C, 4)\n\n\nfor state in [\"Focused\", \"Unfocused\", \"Drowsy\"]:\n context = cognitive_state_to_context(state)\n score = calculate_cognitive_context(context)\n\n print(f\"{state}: Cognitive Context Score = {score}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:18:29.709156Z","iopub.execute_input":"2026-09-27T14:18:29.709886Z","iopub.status.idle":"2026-09-27T14:18:29.718852Z","shell.execute_reply.started":"2026-09-27T14:18:29.709842Z","shell.execute_reply":"2026-09-27T14:18:29.717617Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import math\n\ndef recency_score(time_elapsed, decay_rate=0.05):\n return math.exp(-decay_rate * time_elapsed)\n\n\ndef memory_score(importance, time_elapsed, cognitive_context, value,\n w_importance=0.25,\n w_recency=0.25,\n w_context=0.25,\n w_value=0.25):\n\n R = recency_score(time_elapsed)\n\n score = (\n importance ** w_importance\n * R ** w_recency\n * cognitive_context ** w_context\n * value ** w_value\n )\n\n return round(score, 4)\n\n\n# Example memory\nimportance = 0.9\ntime_elapsed = 2\nvalue = 0.8\n\nfor state in [\"Focused\", \"Unfocused\", \"Drowsy\"]:\n\n context = cognitive_state_to_context(state)\n C = calculate_cognitive_context(context)\n\n score = memory_score(\n importance=importance,\n time_elapsed=time_elapsed,\n cognitive_context=C,\n value=value\n )\n\n print(f\"{state}: Memory Score = {score}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:18:57.632777Z","iopub.execute_input":"2026-09-27T14:18:57.633183Z","iopub.status.idle":"2026-09-27T14:18:57.642594Z","shell.execute_reply.started":"2026-09-27T14:18:57.633148Z","shell.execute_reply":"2026-09-27T14:18:57.641168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def memory_lifecycle(score, high_threshold=0.80, low_threshold=0.50):\n\n if score >= high_threshold:\n return \"STRENGTHEN\"\n\n elif score >= low_threshold:\n return \"RETAIN\"\n\n else:\n return \"REMOVE\"\n\n\nscores = {\n \"Focused\": 0.8497,\n \"Unfocused\": 0.7458,\n \"Drowsy\": 0.6783\n}\n\nfor state, score in scores.items():\n action = memory_lifecycle(score)\n\n print(\n f\"{state}: \"\n f\"Score = {score} → \"\n f\"Action = {action}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:19:22.295635Z","iopub.execute_input":"2026-09-27T14:19:22.29597Z","iopub.status.idle":"2026-09-27T14:19:22.304435Z","shell.execute_reply.started":"2026-09-27T14:19:22.295941Z","shell.execute_reply":"2026-09-27T14:19:22.302822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"memories = [\n {\n \"id\": 1,\n \"text\": \"The learner understands Python loops and can solve basic for-loop problems.\",\n \"importance\": 0.8,\n \"value\": 0.9\n },\n {\n \"id\": 2,\n \"text\": \"The learner struggles with recursion and needs to revise recursion from the basics.\",\n \"importance\": 0.9,\n \"value\": 0.95\n },\n {\n \"id\": 3,\n \"text\": \"The learner has practiced JavaScript ES6 arrow functions and map().\",\n \"importance\": 0.7,\n \"value\": 0.8\n },\n {\n \"id\": 4,\n \"text\": \"The learner needs more practice with SQL joins and aggregate queries.\",\n \"importance\": 0.85,\n \"value\": 0.9\n },\n {\n \"id\": 5,\n \"text\": \"The learner is comfortable with HTML and CSS fundamentals.\",\n \"importance\": 0.6,\n \"value\": 0.7\n }\n]\n\nprint(\"Number of memories:\", len(memories))\n\nfor memory in memories:\n print(f\"\\nMemory {memory['id']}:\")\n print(memory[\"text\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:19:48.84827Z","iopub.execute_input":"2026-09-27T14:19:48.848709Z","iopub.status.idle":"2026-09-27T14:19:48.858193Z","shell.execute_reply.started":"2026-09-27T14:19:48.848673Z","shell.execute_reply":"2026-09-27T14:19:48.85667Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\n\n# Load embedding model\nembedding_model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\n# Extract memory text\nmemory_texts = [memory[\"text\"] for memory in memories]\n\n# Generate embeddings\nmemory_embeddings = embedding_model.encode(\n memory_texts,\n normalize_embeddings=True\n)\n\nprint(\"Embedding matrix shape:\", memory_embeddings.shape)\nprint(\"Number of memories:\", len(memory_texts))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:20:27.116093Z","iopub.execute_input":"2026-09-27T14:20:27.117484Z","iopub.status.idle":"2026-09-27T14:22:37.365826Z","shell.execute_reply.started":"2026-09-27T14:20:27.117408Z","shell.execute_reply":"2026-09-27T14:22:37.363552Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics.pairwise import cosine_similarity\n\n# Extract memory text\nmemory_texts = [memory[\"text\"] for memory in memories]\n\n# Create TF-IDF representations\nvectorizer = TfidfVectorizer(\n stop_words=\"english\",\n ngram_range=(1, 2)\n)\n\nmemory_embeddings = vectorizer.fit_transform(memory_texts)\n\nprint(\"Memory matrix shape:\", memory_embeddings.shape)\nprint(\"Number of memories:\", len(memory_texts))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:23:08.369934Z","iopub.execute_input":"2026-09-27T14:23:08.370393Z","iopub.status.idle":"2026-09-27T14:23:08.431911Z","shell.execute_reply.started":"2026-09-27T14:23:08.370353Z","shell.execute_reply":"2026-09-27T14:23:08.430643Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics.pairwise import cosine_similarity\n\ndef retrieve_memories(query, top_k=3):\n query_vector = vectorizer.transform([query])\n\n similarities = cosine_similarity(\n query_vector,\n memory_embeddings\n )[0]\n\n ranked_indices = similarities.argsort()[::-1][:top_k]\n\n results = []\n\n for idx in ranked_indices:\n results.append({\n \"id\": memories[idx][\"id\"],\n \"text\": memories[idx][\"text\"],\n \"similarity\": round(float(similarities[idx]), 4)\n })\n\n return results\n\n\nquery = \"I need help with recursion\"\n\nresults = retrieve_memories(query)\n\nprint(\"Query:\", query)\nprint(\"\\nRetrieved memories:\")\n\nfor result in results:\n print(\n f\"\\nMemory {result['id']} \"\n f\"| Similarity: {result['similarity']}\"\n )\n print(result[\"text\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:23:42.066203Z","iopub.execute_input":"2026-09-27T14:23:42.066816Z","iopub.status.idle":"2026-09-27T14:23:42.083864Z","shell.execute_reply.started":"2026-09-27T14:23:42.066771Z","shell.execute_reply":"2026-09-27T14:23:42.081771Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def acme_retrieve(query, cognitive_state, top_k=3, eta=0.7):\n\n query_vector = vectorizer.transform([query])\n\n similarities = cosine_similarity(\n query_vector,\n memory_embeddings\n )[0]\n\n context = cognitive_state_to_context(cognitive_state)\n cognitive_context = calculate_cognitive_context(context)\n\n results = []\n\n for idx, similarity in enumerate(similarities):\n\n memory = memories[idx]\n\n # Calculate current memory strength\n memory_score_value = memory_score(\n importance=memory[\"importance\"],\n time_elapsed=2,\n cognitive_context=cognitive_context,\n value=memory[\"value\"]\n )\n\n # ACME retrieval score\n acme_score = (\n eta * similarity\n + (1 - eta) * memory_score_value\n )\n\n results.append({\n \"id\": memory[\"id\"],\n \"text\": memory[\"text\"],\n \"semantic_similarity\": round(float(similarity), 4),\n \"memory_score\": memory_score_value,\n \"acme_score\": round(float(acme_score), 4)\n })\n\n results.sort(\n key=lambda x: x[\"acme_score\"],\n reverse=True\n )\n\n return results[:top_k]\n\n\nquery = \"I need help with recursion\"\n\nfor state in [\"Focused\", \"Unfocused\", \"Drowsy\"]:\n\n print(\"\\n\" + \"=\" * 60)\n print(\"Cognitive State:\", state)\n\n results = acme_retrieve(\n query=query,\n cognitive_state=state,\n top_k=3\n )\n\n for result in results:\n print(\n f\"\\nMemory {result['id']}\"\n f\"\\nSemantic Similarity: {result['semantic_similarity']}\"\n f\"\\nMemory Score: {result['memory_score']}\"\n f\"\\nACME Score: {result['acme_score']}\"\n )\n print(result[\"text\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:24:18.13158Z","iopub.execute_input":"2026-09-27T14:24:18.132225Z","iopub.status.idle":"2026-09-27T14:24:18.151674Z","shell.execute_reply.started":"2026-09-27T14:24:18.132184Z","shell.execute_reply":"2026-09-27T14:24:18.150645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def compare_retrieval(query, cognitive_state):\n\n query_vector = vectorizer.transform([query])\n\n similarities = cosine_similarity(\n query_vector,\n memory_embeddings\n )[0]\n\n # Baseline: semantic similarity only\n baseline_indices = similarities.argsort()[::-1][:3]\n\n print(\"=\" * 65)\n print(\"QUERY:\", query)\n print(\"COGNITIVE STATE:\", cognitive_state)\n\n print(\"\\nBASELINE — Semantic Retrieval\")\n print(\"-\" * 40)\n\n for rank, idx in enumerate(baseline_indices, start=1):\n print(\n f\"{rank}. Memory {memories[idx]['id']} \"\n f\"| Similarity: {similarities[idx]:.4f}\"\n )\n\n # ACME retrieval\n acme_results = acme_retrieve(\n query=query,\n cognitive_state=cognitive_state,\n top_k=3\n )\n\n print(\"\\nACME — Cognitive-Aware Retrieval\")\n print(\"-\" * 40)\n\n for rank, result in enumerate(acme_results, start=1):\n print(\n f\"{rank}. Memory {result['id']} \"\n f\"| Semantic: {result['semantic_similarity']:.4f} \"\n f\"| ACME: {result['acme_score']:.4f}\"\n )\n\n\ncompare_retrieval(\n query=\"I need help with recursion\",\n cognitive_state=\"Drowsy\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:24:51.145202Z","iopub.execute_input":"2026-09-27T14:24:51.146542Z","iopub.status.idle":"2026-09-27T14:24:51.162358Z","shell.execute_reply.started":"2026-09-27T14:24:51.146485Z","shell.execute_reply":"2026-09-27T14:24:51.160603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"evaluation_queries = [\n {\n \"query\": \"I need help with recursion\",\n \"relevant_memory\": 2\n },\n {\n \"query\": \"How do I solve SQL joins?\",\n \"relevant_memory\": 4\n },\n {\n \"query\": \"I want to practice Python loops\",\n \"relevant_memory\": 1\n },\n {\n \"query\": \"Explain JavaScript map function\",\n \"relevant_memory\": 3\n },\n {\n \"query\": \"Help me with HTML and CSS\",\n \"relevant_memory\": 5\n }\n]\n\nprint(\"Number of evaluation queries:\", len(evaluation_queries))\n\nfor item in evaluation_queries:\n print(\n f\"\\nQuery: {item['query']}\"\n f\"\\nRelevant memory: {item['relevant_memory']}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:25:18.116558Z","iopub.execute_input":"2026-09-27T14:25:18.117685Z","iopub.status.idle":"2026-09-27T14:25:18.125985Z","shell.execute_reply.started":"2026-09-27T14:25:18.117645Z","shell.execute_reply":"2026-09-27T14:25:18.124969Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def evaluate_retrieval(cognitive_state=\"Focused\", k=3):\n\n baseline_hits = 0\n acme_hits = 0\n\n baseline_results = []\n acme_results_all = []\n\n for item in evaluation_queries:\n\n query = item[\"query\"]\n relevant_memory = item[\"relevant_memory\"]\n\n query_vector = vectorizer.transform([query])\n\n similarities = cosine_similarity(\n query_vector,\n memory_embeddings\n )[0]\n\n # Baseline ranking\n baseline_indices = similarities.argsort()[::-1][:k]\n\n baseline_ids = [\n memories[idx][\"id\"]\n for idx in baseline_indices\n ]\n\n # ACME ranking\n acme_results = acme_retrieve(\n query=query,\n cognitive_state=cognitive_state,\n top_k=k\n )\n\n acme_ids = [\n result[\"id\"]\n for result in acme_results\n ]\n\n baseline_hit = relevant_memory in baseline_ids\n acme_hit = relevant_memory in acme_ids\n\n baseline_hits += baseline_hit\n acme_hits += acme_hit\n\n baseline_results.append(baseline_ids)\n acme_results_all.append(acme_ids)\n\n print(f\"\\nQuery: {query}\")\n print(f\"Relevant Memory: {relevant_memory}\")\n print(f\"Baseline Top-{k}: {baseline_ids}\")\n print(f\"ACME Top-{k}: {acme_ids}\")\n\n # Recall@K for one relevant memory per query\n baseline_recall = baseline_hits / len(evaluation_queries)\n acme_recall = acme_hits / len(evaluation_queries)\n\n print(\"\\n\" + \"=\" * 60)\n print(f\"COGNITIVE STATE: {cognitive_state}\")\n print(\"=\" * 60)\n\n print(f\"Baseline Recall@{k}: {baseline_recall:.4f}\")\n print(f\"ACME Recall@{k}: {acme_recall:.4f}\")\n\n\nevaluate_retrieval(\n cognitive_state=\"Focused\",\n k=3\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:25:45.381791Z","iopub.execute_input":"2026-09-27T14:25:45.382922Z","iopub.status.idle":"2026-09-27T14:25:45.413214Z","shell.execute_reply.started":"2026-09-27T14:25:45.382881Z","shell.execute_reply":"2026-09-27T14:25:45.412207Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"evaluate_retrieval(\n cognitive_state=\"Unfocused\",\n k=3\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:26:24.396589Z","iopub.execute_input":"2026-09-27T14:26:24.39791Z","iopub.status.idle":"2026-09-27T14:26:24.418377Z","shell.execute_reply.started":"2026-09-27T14:26:24.397859Z","shell.execute_reply":"2026-09-27T14:26:24.417187Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"evaluate_retrieval(\n cognitive_state=\"Drowsy\",\n k=3\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:27:03.882541Z","iopub.execute_input":"2026-09-27T14:27:03.884003Z","iopub.status.idle":"2026-09-27T14:27:03.906044Z","shell.execute_reply.started":"2026-09-27T14:27:03.883937Z","shell.execute_reply":"2026-09-27T14:27:03.905213Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\n\nlifecycle_results = []\n\ntest_states = [\"Focused\", \"Unfocused\", \"Drowsy\"]\n\nfor state in test_states:\n\n context = cognitive_state_to_context(state)\n C = calculate_cognitive_context(context)\n\n for memory in memories:\n\n score = memory_score(\n importance=memory[\"importance\"],\n time_elapsed=2,\n cognitive_context=C,\n value=memory[\"value\"]\n )\n\n action = memory_lifecycle(score)\n\n lifecycle_results.append({\n \"Cognitive State\": state,\n \"Memory ID\": memory[\"id\"],\n \"Memory Score\": score,\n \"Lifecycle Action\": action\n })\n\nlifecycle_df = pd.DataFrame(lifecycle_results)\n\nprint(lifecycle_df.to_string(index=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:27:29.470784Z","iopub.execute_input":"2026-09-27T14:27:29.471569Z","iopub.status.idle":"2026-09-27T14:27:29.493537Z","shell.execute_reply.started":"2026-09-27T14:27:29.47153Z","shell.execute_reply":"2026-09-27T14:27:29.49231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"memory_templates = [\n \"The learner understands Python loops and can solve basic for-loop problems.\",\n \"The learner struggles with recursion and needs to revise recursion from the basics.\",\n \"The learner has practiced JavaScript ES6 arrow functions and map().\",\n \"The learner needs more practice with SQL joins and aggregate queries.\",\n \"The learner is comfortable with HTML and CSS fundamentals.\"\n]\n\nmemory_topics = [\n \"Python loops\",\n \"recursion\",\n \"JavaScript ES6\",\n \"SQL joins\",\n \"HTML CSS\"\n]\n\nexpanded_memories = []\n\nmemory_id = 1\n\nfor topic, template in zip(memory_topics, memory_templates):\n\n for variation in range(10):\n\n expanded_memories.append({\n \"id\": memory_id,\n \"topic\": topic,\n \"text\": template,\n \"importance\": round(\n 0.55 + 0.04 * (variation % 8), 2\n ),\n \"value\": round(\n 0.60 + 0.03 * (variation % 10), 2\n ),\n \"time_elapsed\": variation + 1\n })\n\n memory_id += 1\n\nprint(\"Total memories:\", len(expanded_memories))\n\nfor memory in expanded_memories[:10]:\n print(memory)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:28:00.988485Z","iopub.execute_input":"2026-09-27T14:28:00.988983Z","iopub.status.idle":"2026-09-27T14:28:01.001457Z","shell.execute_reply.started":"2026-09-27T14:28:00.988941Z","shell.execute_reply":"2026-09-27T14:28:00.999509Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Create embeddings for the 50-memory bank\nexpanded_texts = [m[\"text\"] for m in expanded_memories]\n\nexpanded_embeddings = vectorizer.transform(expanded_texts)\n\n\ndef semantic_retrieval(query, top_k=5):\n\n query_vector = vectorizer.transform([query])\n\n similarities = cosine_similarity(\n query_vector,\n expanded_embeddings\n )[0]\n\n indices = similarities.argsort()[::-1][:top_k]\n\n return [\n {\n \"id\": expanded_memories[i][\"id\"],\n \"similarity\": float(similarities[i])\n }\n for i in indices\n ]\n\n\ndef semantic_recency_retrieval(query, top_k=5):\n\n query_vector = vectorizer.transform([query])\n\n similarities = cosine_similarity(\n query_vector,\n expanded_embeddings\n )[0]\n\n scores = []\n\n for i, similarity in enumerate(similarities):\n\n recency = recency_score(\n expanded_memories[i][\"time_elapsed\"]\n )\n\n combined_score = (\n 0.7 * similarity\n + 0.3 * recency\n )\n\n scores.append(combined_score)\n\n indices = np.argsort(scores)[::-1][:top_k]\n\n return [\n {\n \"id\": expanded_memories[i][\"id\"],\n \"score\": float(scores[i])\n }\n for i in indices\n ]\n\n\ndef acme_retrieval_expanded(\n query,\n cognitive_state,\n top_k=5\n):\n\n query_vector = vectorizer.transform([query])\n\n similarities = cosine_similarity(\n query_vector,\n expanded_embeddings\n )[0]\n\n context = cognitive_state_to_context(cognitive_state)\n C = calculate_cognitive_context(context)\n\n scores = []\n\n for i, similarity in enumerate(similarities):\n\n memory = expanded_memories[i]\n\n recency = recency_score(\n memory[\"time_elapsed\"]\n )\n\n memory_strength = (\n memory[\"importance\"] ** 0.25\n * recency ** 0.25\n * C ** 0.25\n * memory[\"value\"] ** 0.25\n )\n\n acme_score = (\n 0.7 * similarity\n + 0.3 * memory_strength\n )\n\n scores.append(acme_score)\n\n indices = np.argsort(scores)[::-1][:top_k]\n\n return [\n {\n \"id\": expanded_memories[i][\"id\"],\n \"score\": float(scores[i])\n }\n for i in indices\n ]\n\n\nprint(\"Three retrieval methods created successfully.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:28:34.078967Z","iopub.execute_input":"2026-09-27T14:28:34.079364Z","iopub.status.idle":"2026-09-27T14:28:34.095897Z","shell.execute_reply.started":"2026-09-27T14:28:34.079332Z","shell.execute_reply":"2026-09-27T14:28:34.094631Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_queries = [\n (\"I need help with Python loops\", \"Python loops\"),\n (\"I am struggling with recursion\", \"recursion\"),\n (\"How can I practice JavaScript map?\", \"JavaScript ES6\"),\n (\"I need help understanding SQL joins\", \"SQL joins\"),\n (\"Help me revise HTML and CSS\", \"HTML CSS\")\n]\n\nstates = [\"Focused\", \"Unfocused\", \"Drowsy\"]\n\nresults = []\n\nfor query, expected_topic in test_queries:\n\n for state in states:\n\n # Baseline\n baseline = semantic_retrieval(query, top_k=5)\n\n # Semantic + recency\n recency_results = semantic_recency_retrieval(\n query,\n top_k=5\n )\n\n # ACME\n acme = acme_retrieval_expanded(\n query,\n cognitive_state=state,\n top_k=5\n )\n\n # Check whether retrieved memory belongs to correct topic\n baseline_topics = [\n expanded_memories[r[\"id\"] - 1][\"topic\"]\n for r in baseline\n ]\n\n recency_topics = [\n expanded_memories[r[\"id\"] - 1][\"topic\"]\n for r in recency_results\n ]\n\n acme_topics = [\n expanded_memories[r[\"id\"] - 1][\"topic\"]\n for r in acme\n ]\n\n results.append({\n \"query\": query,\n \"state\": state,\n \"baseline_hit\": expected_topic in baseline_topics,\n \"recency_hit\": expected_topic in recency_topics,\n \"acme_hit\": expected_topic in acme_topics\n })\n\nresults_df = pd.DataFrame(results)\n\nprint(results_df.to_string(index=False))\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"SUMMARY\")\nprint(\"=\" * 60)\n\nprint(\n \"Semantic Recall@5:\",\n round(results_df[\"baseline_hit\"].mean(), 4)\n)\n\nprint(\n \"Semantic + Recency Recall@5:\",\n round(results_df[\"recency_hit\"].mean(), 4)\n)\n\nprint(\n \"ACME Recall@5:\",\n round(results_df[\"acme_hit\"].mean(), 4)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:29:10.63333Z","iopub.execute_input":"2026-09-27T14:29:10.634547Z","iopub.status.idle":"2026-09-27T14:29:10.73926Z","shell.execute_reply.started":"2026-09-27T14:29:10.634501Z","shell.execute_reply":"2026-09-27T14:29:10.737953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import GroupShuffleSplit\n\n# Use the recording ID for every segment\ngroups = all_record_ids\n\nsplitter = GroupShuffleSplit(\n n_splits=1,\n test_size=0.2,\n random_state=42\n)\n\ntrain_idx, test_idx = next(\n splitter.split(X, y, groups=groups)\n)\n\nX_train_group = X[train_idx]\nX_test_group = X[test_idx]\n\ny_train_group = y[train_idx]\ny_test_group = y[test_idx]\n\ngroups_train = groups[train_idx]\ngroups_test = groups[test_idx]\n\nprint(\"Training samples:\", len(train_idx))\nprint(\"Testing samples:\", len(test_idx))\n\nprint(\"\\nTraining recordings:\")\nprint(np.unique(groups_train))\n\nprint(\"\\nTesting recordings:\")\nprint(np.unique(groups_test))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:30:53.206951Z","iopub.execute_input":"2026-09-27T14:30:53.207579Z","iopub.status.idle":"2026-09-27T14:30:53.220675Z","shell.execute_reply.started":"2026-09-27T14:30:53.207543Z","shell.execute_reply":"2026-09-27T14:30:53.219795Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, classification_report\n\ngroup_model = RandomForestClassifier(\n n_estimators=200,\n random_state=42,\n class_weight=\"balanced\"\n)\n\ngroup_model.fit(X_train_group, y_train_group)\n\ny_pred_group = group_model.predict(X_test_group)\n\ngroup_accuracy = accuracy_score(\n y_test_group,\n y_pred_group\n)\n\nprint(\"Recording-Level EEG Classification Accuracy:\")\nprint(f\"{group_accuracy:.4f}\")\n\nprint(\"\\nClassification Report:\")\nprint(\n classification_report(\n y_test_group,\n y_pred_group\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:31:15.923301Z","iopub.execute_input":"2026-09-27T14:31:15.924217Z","iopub.status.idle":"2026-09-27T14:31:18.641577Z","shell.execute_reply.started":"2026-09-27T14:31:15.924167Z","shell.execute_reply":"2026-09-27T14:31:18.640445Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"eeg_results = {\n \"segment_level\": {\n \"accuracy\": 0.8665,\n \"macro_f1\": 0.83\n },\n \"recording_level\": {\n \"accuracy\": 0.6957,\n \"macro_f1\": 0.63\n }\n}\n\nprint(\"ACME EEG Results\")\nprint(\"=\" * 40)\n\nfor experiment, metrics in eeg_results.items():\n print(f\"\\n{experiment.replace('_', ' ').title()}\")\n\n for metric, value in metrics.items():\n print(f\"{metric}: {value:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:31:51.466393Z","iopub.execute_input":"2026-09-27T14:31:51.467581Z","iopub.status.idle":"2026-09-27T14:31:51.475305Z","shell.execute_reply.started":"2026-09-27T14:31:51.46753Z","shell.execute_reply":"2026-09-27T14:31:51.473971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 38 — Retrieval Ablation Experiment\n# ============================================================\n\nablation_results = []\n\nfor query, expected_topic in test_queries:\n\n for state in states:\n\n # -------------------------\n # 1. Semantic retrieval\n # -------------------------\n semantic_results = semantic_retrieval(\n query,\n top_k=5\n )\n\n semantic_topics = [\n expanded_memories[r[\"id\"] - 1][\"topic\"]\n for r in semantic_results\n ]\n\n semantic_hit = expected_topic in semantic_topics\n\n # -------------------------\n # 2. Semantic + Recency\n # -------------------------\n recency_results = semantic_recency_retrieval(\n query,\n top_k=5\n )\n\n recency_topics = [\n expanded_memories[r[\"id\"] - 1][\"topic\"]\n for r in recency_results\n ]\n\n recency_hit = expected_topic in recency_topics\n\n # -------------------------\n # 3. Full ACME\n # -------------------------\n acme_results = acme_retrieval_expanded(\n query,\n cognitive_state=state,\n top_k=5\n )\n\n acme_topics = [\n expanded_memories[r[\"id\"] - 1][\"topic\"]\n for r in acme_results\n ]\n\n acme_hit = expected_topic in acme_topics\n\n ablation_results.append({\n \"query\": query,\n \"state\": state,\n \"semantic\": semantic_hit,\n \"semantic_recency\": recency_hit,\n \"acme\": acme_hit\n })\n\n\nablation_df = pd.DataFrame(ablation_results)\n\nprint(\"ABLATION RESULTS\")\nprint(\"=\" * 60)\nprint(ablation_df.to_string(index=False))\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"RECALL@5\")\nprint(\"=\" * 60)\n\nprint(\n \"Semantic:\",\n round(ablation_df[\"semantic\"].mean(), 4)\n)\n\nprint(\n \"Semantic + Recency:\",\n round(ablation_df[\"semantic_recency\"].mean(), 4)\n)\n\nprint(\n \"Full ACME:\",\n round(ablation_df[\"acme\"].mean(), 4)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:32:19.011687Z","iopub.execute_input":"2026-09-27T14:32:19.012148Z","iopub.status.idle":"2026-09-27T14:32:19.100911Z","shell.execute_reply.started":"2026-09-27T14:32:19.012084Z","shell.execute_reply":"2026-09-27T14:32:19.100042Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 39 — Harder Retrieval Benchmark\n# ============================================================\n\nhard_queries = [\n (\"I need to revise a topic I keep getting wrong\", \"recursion\"),\n (\"I want to practice coding fundamentals\", \"Python loops\"),\n (\"I need help with a programming concept\", \"JavaScript ES6\"),\n (\"I am struggling with database queries\", \"SQL joins\"),\n (\"I need to revise web development basics\", \"HTML CSS\")\n]\n\nprint(\"Hard evaluation queries:\")\nprint(\"=\" * 60)\n\nfor query, expected_topic in hard_queries:\n print(f\"\\nQuery: {query}\")\n print(f\"Relevant topic: {expected_topic}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:32:47.568426Z","iopub.execute_input":"2026-09-27T14:32:47.568774Z","iopub.status.idle":"2026-09-27T14:32:47.576331Z","shell.execute_reply.started":"2026-09-27T14:32:47.568742Z","shell.execute_reply":"2026-09-27T14:32:47.574943Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 40 — Hard Retrieval Evaluation using MRR\n# ============================================================\n\ndef reciprocal_rank(results, expected_topic):\n for rank, result in enumerate(results, start=1):\n memory_id = result[\"id\"]\n topic = expanded_memories[memory_id - 1][\"topic\"]\n\n if topic == expected_topic:\n return 1 / rank\n\n return 0.0\n\n\nmrr_results = []\n\nfor query, expected_topic in hard_queries:\n\n # -------------------------\n # Semantic Retrieval\n # -------------------------\n semantic_results = semantic_retrieval(\n query,\n top_k=5\n )\n\n semantic_rr = reciprocal_rank(\n semantic_results,\n expected_topic\n )\n\n # -------------------------\n # Semantic + Recency\n # -------------------------\n recency_results = semantic_recency_retrieval(\n query,\n top_k=5\n )\n\n recency_rr = reciprocal_rank(\n recency_results,\n expected_topic\n )\n\n # -------------------------\n # Full ACME\n # -------------------------\n for state in states:\n\n acme_results = acme_retrieval_expanded(\n query,\n cognitive_state=state,\n top_k=5\n )\n\n acme_rr = reciprocal_rank(\n acme_results,\n expected_topic\n )\n\n mrr_results.append({\n \"query\": query,\n \"state\": state,\n \"semantic_rr\": semantic_rr,\n \"semantic_recency_rr\": recency_rr,\n \"acme_rr\": acme_rr\n })\n\n\nmrr_df = pd.DataFrame(mrr_results)\n\nprint(\"=\" * 70)\nprint(\"HARD RETRIEVAL BENCHMARK — MRR\")\nprint(\"=\" * 70)\n\nprint(mrr_df.to_string(index=False))\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"MEAN RECIPROCAL RANK\")\nprint(\"=\" * 70)\n\nprint(\n \"Semantic:\",\n round(mrr_df[\"semantic_rr\"].mean(), 4)\n)\n\nprint(\n \"Semantic + Recency:\",\n round(mrr_df[\"semantic_recency_rr\"].mean(), 4)\n)\n\nprint(\n \"Full ACME:\",\n round(mrr_df[\"acme_rr\"].mean(), 4)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:33:13.265151Z","iopub.execute_input":"2026-09-27T14:33:13.266219Z","iopub.status.idle":"2026-09-27T14:33:13.349328Z","shell.execute_reply.started":"2026-09-27T14:33:13.266104Z","shell.execute_reply":"2026-09-27T14:33:13.348028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 41 — Cognitive-Aware Memory Lifecycle Experiment\n# ============================================================\n\nlifecycle_results = []\n\nfor state in states:\n\n context = cognitive_state_to_context(state)\n cognitive_context = calculate_cognitive_context(context)\n\n for memory in memories:\n\n score = memory_score(\n importance=memory[\"importance\"],\n time_elapsed=2,\n cognitive_context=cognitive_context,\n value=memory[\"value\"]\n )\n\n lifecycle = memory_lifecycle(score)\n\n lifecycle_results.append({\n \"state\": state,\n \"memory_id\": memory[\"id\"],\n \"importance\": memory[\"importance\"],\n \"cognitive_context\": cognitive_context,\n \"memory_score\": score,\n \"lifecycle\": lifecycle\n })\n\nlifecycle_df = pd.DataFrame(lifecycle_results)\n\nprint(\"=\" * 70)\nprint(\"COGNITIVE-AWARE MEMORY LIFECYCLE\")\nprint(\"=\" * 70)\n\nprint(lifecycle_df.to_string(index=False))\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"LIFECYCLE COUNTS\")\nprint(\"=\" * 70)\n\nprint(\n lifecycle_df.groupby(\n [\"state\", \"lifecycle\"]\n ).size()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:33:46.229379Z","iopub.execute_input":"2026-09-27T14:33:46.230648Z","iopub.status.idle":"2026-09-27T14:33:46.254359Z","shell.execute_reply.started":"2026-09-27T14:33:46.230588Z","shell.execute_reply":"2026-09-27T14:33:46.253286Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 42 — Cognitive Context Visualization\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\nstate_contexts = []\n\nfor state in states:\n context = cognitive_state_to_context(state)\n cognitive_context = calculate_cognitive_context(context)\n\n state_contexts.append({\n \"state\": state,\n \"cognitive_context\": cognitive_context\n })\n\ncontext_df = pd.DataFrame(state_contexts)\n\nplt.figure(figsize=(8, 5))\n\nplt.bar(\n context_df[\"state\"],\n context_df[\"cognitive_context\"]\n)\n\nplt.xlabel(\"Cognitive State\")\nplt.ylabel(\"Cognitive Context Score\")\nplt.title(\"Cognitive Context Across Learner States\")\nplt.ylim(0, 1)\n\nfor i, value in enumerate(context_df[\"cognitive_context\"]):\n plt.text(\n i,\n value + 0.02,\n f\"{value:.3f}\",\n ha=\"center\"\n )\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:34:15.49806Z","iopub.execute_input":"2026-09-27T14:34:15.498979Z","iopub.status.idle":"2026-09-27T14:34:15.775662Z","shell.execute_reply.started":"2026-09-27T14:34:15.49894Z","shell.execute_reply":"2026-09-27T14:34:15.774135Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 43 — Memory Score Across Cognitive States\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\nmemory_score_plot = []\n\nfor state in states:\n\n context = cognitive_state_to_context(state)\n cognitive_context = calculate_cognitive_context(context)\n\n for memory in memories:\n\n score = memory_score(\n importance=memory[\"importance\"],\n time_elapsed=2,\n cognitive_context=cognitive_context,\n value=memory[\"value\"]\n )\n\n memory_score_plot.append({\n \"state\": state,\n \"memory_id\": memory[\"id\"],\n \"memory_score\": score\n })\n\nmemory_score_df = pd.DataFrame(memory_score_plot)\n\nplt.figure(figsize=(9, 5))\n\nfor state in states:\n state_data = memory_score_df[\n memory_score_df[\"state\"] == state\n ]\n\n plt.plot(\n state_data[\"memory_id\"],\n state_data[\"memory_score\"],\n marker=\"o\",\n label=state\n )\n\nplt.xlabel(\"Memory ID\")\nplt.ylabel(\"Memory Score\")\nplt.title(\"Memory Strength Across Cognitive States\")\nplt.xticks(range(1, 6))\nplt.ylim(0, 1)\nplt.legend()\nplt.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:34:43.719964Z","iopub.execute_input":"2026-09-27T14:34:43.720455Z","iopub.status.idle":"2026-09-27T14:34:43.931053Z","shell.execute_reply.started":"2026-09-27T14:34:43.72042Z","shell.execute_reply":"2026-09-27T14:34:43.929843Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 44 — EEG Cognitive-State Confusion Matrix\n# ============================================================\n\nfrom sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay\nimport matplotlib.pyplot as plt\n\n# Predictions from the recording-level model\ny_test_pred = group_model.predict(X_test)\n\ncm = confusion_matrix(\n y_test,\n y_test_pred,\n labels=[\"Drowsy\", \"Focused\", \"Unfocused\"]\n)\n\ndisp = ConfusionMatrixDisplay(\n confusion_matrix=cm,\n display_labels=[\"Drowsy\", \"Focused\", \"Unfocused\"]\n)\n\nfig, ax = plt.subplots(figsize=(7, 6))\n\ndisp.plot(\n ax=ax,\n values_format=\"d\"\n)\n\nplt.title(\"EEG Cognitive-State Classification — Recording-Level Split\")\nplt.xlabel(\"Predicted State\")\nplt.ylabel(\"Actual State\")\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:35:34.31575Z","iopub.execute_input":"2026-09-27T14:35:34.316108Z","iopub.status.idle":"2026-09-27T14:35:34.572331Z","shell.execute_reply.started":"2026-09-27T14:35:34.316079Z","shell.execute_reply":"2026-09-27T14:35:34.571366Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 45 — EEG Cognitive-State Class Distribution\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\nclass_counts = pd.Series(y).value_counts()\n\n# Keep a consistent order\nclass_order = [\"Drowsy\", \"Focused\", \"Unfocused\"]\nclass_counts = class_counts.reindex(class_order)\n\nplt.figure(figsize=(8, 5))\n\nplt.bar(\n class_counts.index,\n class_counts.values\n)\n\nplt.xlabel(\"Cognitive State\")\nplt.ylabel(\"Number of EEG Segments\")\nplt.title(\"Distribution of EEG Cognitive-State Samples\")\n\nfor i, value in enumerate(class_counts.values):\n plt.text(\n i,\n value + 15,\n str(value),\n ha=\"center\"\n )\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:36:12.040644Z","iopub.execute_input":"2026-09-27T14:36:12.041287Z","iopub.status.idle":"2026-09-27T14:36:12.238807Z","shell.execute_reply.started":"2026-09-27T14:36:12.041242Z","shell.execute_reply":"2026-09-27T14:36:12.2377Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 46 — COMPLETE ACME PIPELINE\n# EEG → Cognitive State → Context → Memory → Retrieval\n# ============================================================\n\ndef run_acme_pipeline(eeg_segment, query, top_k=3):\n\n # --------------------------------------------------------\n # 1. Extract EEG features\n # --------------------------------------------------------\n segment_features = extract_eeg_features(\n eeg_segment\n )\n\n segment_features = segment_features.reshape(1, -1)\n\n # --------------------------------------------------------\n # 2. Predict cognitive state\n # --------------------------------------------------------\n predicted_state = group_model.predict(\n segment_features\n )[0]\n\n # --------------------------------------------------------\n # 3. Convert state → cognitive context\n # --------------------------------------------------------\n context = cognitive_state_to_context(\n predicted_state\n )\n\n cognitive_context = calculate_cognitive_context(\n context\n )\n\n # --------------------------------------------------------\n # 4. Retrieve relevant memories\n # --------------------------------------------------------\n retrieved_memories = acme_retrieve(\n query=query,\n cognitive_state=predicted_state,\n top_k=top_k\n )\n\n # --------------------------------------------------------\n # 5. Assign lifecycle status\n # --------------------------------------------------------\n for memory in retrieved_memories:\n\n lifecycle = memory_lifecycle(\n memory[\"acme_score\"]\n )\n\n memory[\"lifecycle\"] = lifecycle\n\n # --------------------------------------------------------\n # 6. Return complete ACME result\n # --------------------------------------------------------\n return {\n \"predicted_state\": predicted_state,\n \"cognitive_context\": cognitive_context,\n \"context\": context,\n \"retrieved_memories\": retrieved_memories\n }\n\n\nprint(\"Complete ACME pipeline created successfully.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:36:47.947273Z","iopub.execute_input":"2026-09-27T14:36:47.948606Z","iopub.status.idle":"2026-09-27T14:36:47.957443Z","shell.execute_reply.started":"2026-09-27T14:36:47.948563Z","shell.execute_reply":"2026-09-27T14:36:47.956338Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 47 — REAL EEG END-TO-END ACME DEMO\n# ============================================================\n\n# Select one real EEG segment from the dataset\nreal_eeg_segment = all_segments[0]\n\n# Learner query\nquery = \"I need help with recursion\"\n\n# Run complete ACME pipeline\nresult = run_acme_pipeline(\n eeg_segment=real_eeg_segment,\n query=query,\n top_k=3\n)\n\nprint(\"=\" * 70)\nprint(\"ACME END-TO-END DEMONSTRATION\")\nprint(\"=\" * 70)\n\nprint(\"\\nLearner Query:\")\nprint(query)\n\nprint(\"\\nPredicted Cognitive State:\")\nprint(result[\"predicted_state\"])\n\nprint(\"\\nCognitive Context:\")\nprint(result[\"cognitive_context\"])\n\nprint(\"\\nCognitive Context Components:\")\nprint(result[\"context\"])\n\nprint(\"\\nRetrieved Memories:\")\nprint(\"-\" * 70)\n\nfor memory in result[\"retrieved_memories\"]:\n print(f\"\\nMemory ID: {memory['id']}\")\n print(f\"Similarity: {memory['similarity']}\")\n print(f\"Memory Score: {memory['memory_score']}\")\n print(f\"ACME Score: {memory['acme_score']}\")\n print(f\"Lifecycle: {memory['lifecycle']}\")\n print(f\"Memory: {memory['text']}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ACME PIPELINE COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:37:24.697978Z","iopub.execute_input":"2026-09-27T14:37:24.698513Z","iopub.status.idle":"2026-09-27T14:37:24.711937Z","shell.execute_reply.started":"2026-09-27T14:37:24.698473Z","shell.execute_reply":"2026-09-27T14:37:24.710091Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 47 — CONNECT EEG FEATURE EXTRACTION\n# ============================================================\n\nfrom scipy.signal import welch\nimport numpy as np\n\ndef extract_eeg_features(segment, sfreq=128):\n\n bands = {\n \"delta\": (0.5, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n }\n\n features = []\n\n # segment shape = (samples, channels)\n for channel in range(segment.shape[1]):\n\n signal = segment[:, channel]\n\n frequencies, psd = welch(\n signal,\n fs=sfreq,\n nperseg=min(256, len(signal))\n )\n\n for low, high in bands.values():\n\n band_power = psd[\n (frequencies >= low) &\n (frequencies < high)\n ].mean()\n\n features.append(band_power)\n\n return np.array(features)\n\n\nprint(\"EEG feature extraction connected successfully.\")\nprint(\"Feature vector size:\", len(extract_eeg_features(all_segments[0])))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:37:50.368753Z","iopub.execute_input":"2026-09-27T14:37:50.369307Z","iopub.status.idle":"2026-09-27T14:37:50.480928Z","shell.execute_reply.started":"2026-09-27T14:37:50.369263Z","shell.execute_reply":"2026-09-27T14:37:50.480149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 48 — REAL EEG END-TO-END ACME DEMO\n# ============================================================\n\nreal_eeg_segment = all_segments[0]\n\nquery = \"I need help with recursion\"\n\nresult = run_acme_pipeline(\n eeg_segment=real_eeg_segment,\n query=query,\n top_k=3\n)\n\nprint(\"=\" * 70)\nprint(\"ACME END-TO-END DEMONSTRATION\")\nprint(\"=\" * 70)\n\nprint(\"\\nLearner Query:\")\nprint(query)\n\nprint(\"\\nPredicted Cognitive State:\")\nprint(result[\"predicted_state\"])\n\nprint(\"\\nCognitive Context:\")\nprint(result[\"cognitive_context\"])\n\nprint(\"\\nCognitive Context Components:\")\nprint(result[\"context\"])\n\nprint(\"\\nRetrieved Memories:\")\nprint(\"-\" * 70)\n\nfor memory in result[\"retrieved_memories\"]:\n print(f\"\\nMemory ID: {memory['id']}\")\n print(f\"Similarity: {memory['similarity']}\")\n print(f\"Memory Score: {memory['memory_score']}\")\n print(f\"ACME Score: {memory['acme_score']}\")\n print(f\"Lifecycle: {memory['lifecycle']}\")\n print(f\"Memory: {memory['text']}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ACME PIPELINE COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:38:21.365485Z","iopub.execute_input":"2026-09-27T14:38:21.365915Z","iopub.status.idle":"2026-09-27T14:38:21.489672Z","shell.execute_reply.started":"2026-09-27T14:38:21.365879Z","shell.execute_reply":"2026-09-27T14:38:21.488251Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 49 — DISPLAY ACME PIPELINE RESULT\n# ============================================================\n\nprint(\"=\" * 70)\nprint(\"ACME END-TO-END DEMONSTRATION\")\nprint(\"=\" * 70)\n\nprint(\"\\nLearner Query:\")\nprint(query)\n\nprint(\"\\nPredicted Cognitive State:\")\nprint(result[\"predicted_state\"])\n\nprint(\"\\nCognitive Context:\")\nprint(result[\"cognitive_context\"])\n\nprint(\"\\nCognitive Context Components:\")\nprint(result[\"context\"])\n\nprint(\"\\nRetrieved Memories:\")\nprint(\"-\" * 70)\n\nfor memory in result[\"retrieved_memories\"]:\n\n print(f\"\\nMemory ID: {memory.get('id', 'N/A')}\")\n print(f\"Memory: {memory.get('text', 'N/A')}\")\n print(f\"Similarity: {memory.get('similarity', 'N/A')}\")\n print(f\"Memory Score: {memory.get('memory_score', 'N/A')}\")\n print(f\"ACME Score: {memory.get('acme_score', 'N/A')}\")\n print(f\"Lifecycle: {memory.get('lifecycle', 'N/A')}\")\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ACME PIPELINE COMPLETE\")\nprint(\"=\" * 70)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:39:02.325479Z","iopub.execute_input":"2026-09-27T14:39:02.3258Z","iopub.status.idle":"2026-09-27T14:39:02.335373Z","shell.execute_reply.started":"2026-09-27T14:39:02.32577Z","shell.execute_reply":"2026-09-27T14:39:02.334104Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 50 — FIX MEMORY LIFECYCLE CALCULATION\n# ============================================================\n\nfor memory in result[\"retrieved_memories\"]:\n\n memory[\"lifecycle\"] = memory_lifecycle(\n memory[\"memory_score\"]\n )\n\nprint(\"=\" * 70)\nprint(\"CORRECTED ACME MEMORY LIFECYCLE\")\nprint(\"=\" * 70)\n\nfor memory in result[\"retrieved_memories\"]:\n\n print(f\"\\nMemory ID: {memory['id']}\")\n print(f\"Memory: {memory['text']}\")\n print(f\"Memory Score: {memory['memory_score']}\")\n print(f\"ACME Score: {memory['acme_score']}\")\n print(f\"Lifecycle: {memory['lifecycle']}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:39:34.646762Z","iopub.execute_input":"2026-09-27T14:39:34.64825Z","iopub.status.idle":"2026-09-27T14:39:34.657615Z","shell.execute_reply.started":"2026-09-27T14:39:34.648191Z","shell.execute_reply":"2026-09-27T14:39:34.656219Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 51 — FINAL EXPERIMENTAL RESULTS TABLE\n# ============================================================\n\nfinal_results = pd.DataFrame({\n \"Experiment\": [\n \"EEG Classification - Segment Split\",\n \"EEG Classification - Recording Split\",\n \"Retrieval - Semantic\",\n \"Retrieval - Semantic + Recency\",\n \"Retrieval - Full ACME\",\n \"Cognitive Context - Focused\",\n \"Cognitive Context - Unfocused\",\n \"Cognitive Context - Drowsy\"\n ],\n \"Metric\": [\n \"Accuracy / Macro F1\",\n \"Accuracy / Macro F1\",\n \"MRR\",\n \"MRR\",\n \"MRR\",\n \"Cognitive Context\",\n \"Cognitive Context\",\n \"Cognitive Context\"\n ],\n \"Result\": [\n \"0.8665 / 0.83\",\n \"0.6957 / 0.63\",\n \"0.40\",\n \"0.44\",\n \"0.44\",\n \"0.800\",\n \"0.475\",\n \"0.325\"\n ]\n})\n\nprint(\"=\" * 80)\nprint(\"ACME FINAL EXPERIMENTAL RESULTS\")\nprint(\"=\" * 80)\n\nprint(final_results.to_string(index=False))\n\n# Save results\nfinal_results.to_csv(\n \"ACME_final_results.csv\",\n index=False\n)\n\nprint(\"\\nSaved: ACME_final_results.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:40:08.358592Z","iopub.execute_input":"2026-09-27T14:40:08.359007Z","iopub.status.idle":"2026-09-27T14:40:08.382989Z","shell.execute_reply.started":"2026-09-27T14:40:08.358973Z","shell.execute_reply":"2026-09-27T14:40:08.381839Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 52 — ADAPTIVE RESPONSE GENERATION\n# ============================================================\n\ndef generate_adaptive_response(result, query):\n\n state = result[\"predicted_state\"]\n\n # Find the highest-ranked memory\n top_memory = result[\"retrieved_memories\"][0]\n\n if state == \"Focused\":\n\n response = (\n f\"You're currently in a focused state. \"\n f\"Let's work directly on your question: '{query}'. \"\n f\"Your previous memory indicates: \"\n f\"'{top_memory['text']}' \"\n f\"We can move directly into the next problem.\"\n )\n\n elif state == \"Unfocused\":\n\n response = (\n f\"Your current state suggests reduced focus. \"\n f\"Let's keep this simple and work through one step at a time. \"\n f\"Your relevant previous memory is: \"\n f\"'{top_memory['text']}'\"\n )\n\n else: # Drowsy\n\n response = (\n f\"Your current state suggests increased fatigue. \"\n f\"Let's avoid a long explanation and focus on one small concept. \"\n f\"Your relevant previous memory is: \"\n f\"'{top_memory['text']}'\"\n )\n\n return response\n\n\nadaptive_response = generate_adaptive_response(\n result,\n query\n)\n\nprint(\"=\" * 70)\nprint(\"ACME ADAPTIVE RESPONSE\")\nprint(\"=\" * 70)\n\nprint(\"\\nCognitive State:\")\nprint(result[\"predicted_state\"])\n\nprint(\"\\nQuery:\")\nprint(query)\n\nprint(\"\\nGenerated Response:\")\nprint(adaptive_response)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:41:02.250074Z","iopub.execute_input":"2026-09-27T14:41:02.251458Z","iopub.status.idle":"2026-09-27T14:41:02.260797Z","shell.execute_reply.started":"2026-09-27T14:41:02.251417Z","shell.execute_reply":"2026-09-27T14:41:02.259542Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 53 — ACME FINAL SYSTEM CHECK\n# ============================================================\n\nprint(\"=\" * 75)\nprint(\"ACME FINAL SYSTEM CHECK\")\nprint(\"=\" * 75)\n\nprint(\"\\n[1] EEG DATA\")\nprint(\"Segments:\", all_segments.shape[0])\nprint(\"Samples per segment:\", all_segments.shape[1])\nprint(\"EEG Channels:\", all_segments.shape[2])\n\nprint(\"\\n[2] EEG FEATURES\")\nprint(\"Feature vector size:\", X.shape[1])\n\nprint(\"\\n[3] EEG CLASSIFICATION\")\nprint(\"Recording-level Accuracy:\", eeg_results[\"recording_level\"][\"accuracy\"])\nprint(\"Recording-level Macro F1:\", eeg_results[\"recording_level\"][\"macro_f1\"])\n\nprint(\"\\n[4] COGNITIVE CONTEXT\")\nfor state in states:\n context = cognitive_state_to_context(state)\n score = calculate_cognitive_context(context)\n print(f\"{state}: {score}\")\n\nprint(\"\\n[5] RETRIEVAL\")\nprint(\"Semantic MRR: 0.40\")\nprint(\"Semantic + Recency MRR: 0.44\")\nprint(\"Full ACME MRR: 0.44\")\n\nprint(\"\\n[6] MEMORY LIFECYCLE\")\nprint(\"Focused: 3 Strengthen, 2 Retain\")\nprint(\"Unfocused: 5 Retain\")\nprint(\"Drowsy: 5 Retain\")\n\nprint(\"\\n[7] END-TO-END PIPELINE\")\nprint(\"Real EEG → Cognitive State → Context → Memory → Retrieval → Response\")\n\nprint(\"\\n\" + \"=\" * 75)\nprint(\"ACME PROJECT CORE PIPELINE VERIFIED\")\nprint(\"=\" * 75)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:41:38.256762Z","iopub.execute_input":"2026-09-27T14:41:38.257734Z","iopub.status.idle":"2026-09-27T14:41:38.26893Z","shell.execute_reply.started":"2026-09-27T14:41:38.257695Z","shell.execute_reply":"2026-09-27T14:41:38.26662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nfor root, dirs, files in os.walk(\"/kaggle/input\"):\n if \"stew\" in root.lower() or \"stew\" in \" \".join(files).lower():\n print(root)\n print(files[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:55:39.317176Z","iopub.execute_input":"2026-09-27T14:55:39.31767Z","iopub.status.idle":"2026-09-27T14:55:39.454434Z","shell.execute_reply.started":"2026-09-27T14:55:39.317634Z","shell.execute_reply":"2026-09-27T14:55:39.453404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nstew_path = \"/kaggle/input/competitions/eeg-stew-signal-enhancement-and-modeling-challeng/STEW Dataset\"\n\nfiles = sorted(os.listdir(stew_path))\n\nprint(\"Total files:\", len(files))\nprint(\"\\nFirst 10 files:\")\nfor f in files[:10]:\n print(f)\n\n# Inspect one low-workload and one high-workload recording\nfor filename in [\"sub22_lo.txt\", \"sub31_hi.txt\"]:\n filepath = os.path.join(stew_path, filename)\n\n print(\"\\n\" + \"=\"*60)\n print(filename)\n\n with open(filepath, \"r\", errors=\"ignore\") as f:\n for i in range(5):\n print(f.readline().strip())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:56:12.399995Z","iopub.execute_input":"2026-09-27T14:56:12.400419Z","iopub.status.idle":"2026-09-27T14:56:12.421728Z","shell.execute_reply.started":"2026-09-27T14:56:12.400386Z","shell.execute_reply":"2026-09-27T14:56:12.420429Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport os\n\nratings_path = os.path.join(stew_path, \"ratings.txt\")\n\nratings = pd.read_csv(\n ratings_path,\n sep=r\"\\s+\",\n header=None,\n engine=\"python\"\n)\n\nprint(\"Shape:\", ratings.shape)\nprint(\"\\nFirst 10 rows:\")\ndisplay(ratings.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:56:40.079146Z","iopub.execute_input":"2026-09-27T14:56:40.080553Z","iopub.status.idle":"2026-09-27T14:56:40.115978Z","shell.execute_reply.started":"2026-09-27T14:56:40.080502Z","shell.execute_reply":"2026-09-27T14:56:40.114953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(ratings.to_string(index=False, header=False))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:57:07.504756Z","iopub.execute_input":"2026-09-27T14:57:07.505372Z","iopub.status.idle":"2026-09-27T14:57:07.512716Z","shell.execute_reply.started":"2026-09-27T14:57:07.505338Z","shell.execute_reply":"2026-09-27T14:57:07.511674Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport os\n\nfor filename in [\"sub01_lo.txt\", \"sub01_hi.txt\"]:\n filepath = os.path.join(stew_path, filename)\n\n data = np.loadtxt(filepath)\n\n print(filename)\n print(\"Shape:\", data.shape)\n print(\"Channels:\", data.shape[1])\n print(\"Samples:\", data.shape[0])\n print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:57:35.769909Z","iopub.execute_input":"2026-09-27T14:57:35.771387Z","iopub.status.idle":"2026-09-27T14:57:35.966582Z","shell.execute_reply.started":"2026-09-27T14:57:35.771341Z","shell.execute_reply":"2026-09-27T14:57:35.965388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport os\n\n# Check sampling rate assumption from STEW dataset structure\n# 19200 samples = 150 seconds at 128 Hz\nsfreq = 128\nduration = 19200 / sfreq\n\nprint(\"Sampling frequency:\", sfreq, \"Hz\")\nprint(\"Recording duration:\", duration, \"seconds\")\nprint(\"Recording duration:\", duration / 60, \"minutes\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:57:58.550911Z","iopub.execute_input":"2026-09-27T14:57:58.551333Z","iopub.status.idle":"2026-09-27T14:57:58.558316Z","shell.execute_reply.started":"2026-09-27T14:57:58.551298Z","shell.execute_reply":"2026-09-27T14:57:58.556961Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEW: Create 30-second EEG segments\n# ============================================================\n\nsegment_duration = 30\nsamples_per_segment = sfreq * segment_duration\n\nstew_segments = []\nstew_labels = []\nstew_subjects = []\n\nfor filename in sorted(os.listdir(stew_path)):\n\n if not filename.endswith(\".txt\"):\n continue\n\n if filename == \"ratings.txt\":\n continue\n\n data = np.loadtxt(os.path.join(stew_path, filename))\n\n # Subject number\n subject_id = int(filename[3:5])\n\n # Workload condition from filename\n # lo = low workload, hi = high workload\n workload = \"Low\" if \"_lo\" in filename else \"High\"\n\n # Create non-overlapping 30-second segments\n n_segments = data.shape[0] // samples_per_segment\n\n for i in range(n_segments):\n start = i * samples_per_segment\n end = start + samples_per_segment\n\n segment = data[start:end]\n\n if segment.shape == (samples_per_segment, 14):\n stew_segments.append(segment)\n stew_labels.append(workload)\n stew_subjects.append(subject_id)\n\nstew_segments = np.array(stew_segments)\nstew_labels = np.array(stew_labels)\nstew_subjects = np.array(stew_subjects)\n\nprint(\"STEW segments:\", stew_segments.shape)\nprint(\"Labels:\", np.unique(stew_labels, return_counts=True))\nprint(\"Subjects:\", len(np.unique(stew_subjects)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:58:20.953915Z","iopub.execute_input":"2026-09-27T14:58:20.955184Z","iopub.status.idle":"2026-09-27T14:58:28.431841Z","shell.execute_reply.started":"2026-09-27T14:58:20.955111Z","shell.execute_reply":"2026-09-27T14:58:28.430466Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEW: EEG Frequency-Band Feature Extraction\n# ============================================================\n\nfrom scipy.signal import welch\n\nbands = {\n \"delta\": (0.5, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n}\n\ndef extract_stew_features(segment, sfreq=128):\n\n features = []\n\n for channel in range(segment.shape[1]):\n\n signal = segment[:, channel]\n\n frequencies, psd = welch(\n signal,\n fs=sfreq,\n nperseg=256\n )\n\n for low, high in bands.values():\n\n band_power = psd[\n (frequencies >= low) &\n (frequencies < high)\n ].mean()\n\n features.append(band_power)\n\n return np.array(features)\n\n\nstew_X = np.array([\n extract_stew_features(segment)\n for segment in stew_segments\n])\n\nstew_y = stew_labels\n\nprint(\"Feature matrix:\", stew_X.shape)\nprint(\"Labels:\", stew_y.shape)\nprint(\"Features per segment:\", stew_X.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:59:00.147919Z","iopub.execute_input":"2026-09-27T14:59:00.148333Z","iopub.status.idle":"2026-09-27T14:59:13.9341Z","shell.execute_reply.started":"2026-09-27T14:59:00.148299Z","shell.execute_reply":"2026-09-27T14:59:13.933274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEW: Subject-Level Workload Classification\n# ============================================================\n\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# Subject-level split\nsplitter = GroupShuffleSplit(\n n_splits=1,\n test_size=0.20,\n random_state=42\n)\n\ntrain_idx, test_idx = next(\n splitter.split(stew_X, stew_y, groups=stew_subjects)\n)\n\nX_train_stew = stew_X[train_idx]\nX_test_stew = stew_X[test_idx]\n\ny_train_stew = stew_y[train_idx]\ny_test_stew = stew_y[test_idx]\n\ngroups_train = stew_subjects[train_idx]\ngroups_test = stew_subjects[test_idx]\n\nprint(\"Training segments:\", len(train_idx))\nprint(\"Testing segments:\", len(test_idx))\nprint(\"Training subjects:\", len(np.unique(groups_train)))\nprint(\"Testing subjects:\", len(np.unique(groups_test)))\n\n# Train classifier\nstew_model = RandomForestClassifier(\n n_estimators=200,\n random_state=42,\n class_weight=\"balanced\"\n)\n\nstew_model.fit(X_train_stew, y_train_stew)\n\n# Prediction\nstew_pred = stew_model.predict(X_test_stew)\n\n# Results\naccuracy = accuracy_score(y_test_stew, stew_pred)\n\nprint(\"\\nSTEW Workload Classification Accuracy:\", round(accuracy, 4))\n\nprint(\"\\nClassification Report:\")\nprint(classification_report(y_test_stew, stew_pred))\n\nprint(\"\\nConfusion Matrix:\")\nprint(confusion_matrix(y_test_stew, stew_pred))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T14:59:40.671657Z","iopub.execute_input":"2026-09-27T14:59:40.672612Z","iopub.status.idle":"2026-09-27T14:59:41.483273Z","shell.execute_reply.started":"2026-09-27T14:59:40.672567Z","shell.execute_reply":"2026-09-27T14:59:41.482139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Combined Cognitive-State Estimation\n# Attention + Workload\n# ============================================================\n\ndef predict_attention_state(eeg_segment):\n \"\"\"\n Predict attention state using the trained\n Mental Attention EEG model.\n \"\"\"\n\n features = extract_eeg_features(eeg_segment, sfreq=128)\n features = features.reshape(1, -1)\n\n return group_model.predict(features)[0]\n\n\ndef predict_workload_state(stew_segment):\n \"\"\"\n Predict workload state using the trained\n STEW workload model.\n \"\"\"\n\n features = extract_stew_features(stew_segment, sfreq=128)\n features = features.reshape(1, -1)\n\n return stew_model.predict(features)[0]\n\n\ndef calculate_combined_cognitive_context(attention_state, workload_state):\n \"\"\"\n Convert learned Attention + Workload states\n into the ACME cognitive context score.\n \"\"\"\n\n attention_values = {\n \"Focused\": 0.90,\n \"Unfocused\": 0.40,\n \"Drowsy\": 0.20\n }\n\n workload_values = {\n \"Low\": 0.25,\n \"High\": 0.80\n }\n\n A = attention_values[attention_state]\n W = workload_values[workload_state]\n\n # At this stage, fatigue and engagement are\n # not directly learned from EEG yet.\n # We keep them neutral.\n F = 0.50\n E = 0.50\n\n alpha = beta = gamma = delta = 0.25\n\n C = (\n alpha * A +\n beta * E +\n gamma * (1 - W) +\n delta * (1 - F)\n )\n\n return round(C, 4)\n\n\n# ------------------------------------------------------------\n# Test using one real attention-dataset EEG segment\n# ------------------------------------------------------------\n\ntest_attention_segment = all_segments[0]\n\nattention_state = predict_attention_state(test_attention_segment)\n\nprint(\"Predicted Attention State:\", attention_state)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:00:25.883666Z","iopub.execute_input":"2026-09-27T15:00:25.885552Z","iopub.status.idle":"2026-09-27T15:00:26.039209Z","shell.execute_reply.started":"2026-09-27T15:00:25.885493Z","shell.execute_reply":"2026-09-27T15:00:26.037937Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Test STEW Workload Prediction\n# ============================================================\n\ntest_stew_segment = stew_segments[0]\n\nworkload_state = predict_workload_state(test_stew_segment)\n\nprint(\"Predicted Workload State:\", workload_state)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:00:51.007506Z","iopub.execute_input":"2026-09-27T15:00:51.008151Z","iopub.status.idle":"2026-09-27T15:00:51.105656Z","shell.execute_reply.started":"2026-09-27T15:00:51.008066Z","shell.execute_reply":"2026-09-27T15:00:51.104418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Combine Attention + Workload\n# ============================================================\n\ncognitive_context = calculate_combined_cognitive_context(\n attention_state,\n workload_state\n)\n\nprint(\"Attention State :\", attention_state)\nprint(\"Workload State :\", workload_state)\nprint(\"Cognitive Context:\", cognitive_context)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:01:15.964787Z","iopub.execute_input":"2026-09-27T15:01:15.965135Z","iopub.status.idle":"2026-09-27T15:01:15.971958Z","shell.execute_reply.started":"2026-09-27T15:01:15.965079Z","shell.execute_reply":"2026-09-27T15:01:15.970797Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Persistent Learner Memory Store\n# ============================================================\n\nimport json\nfrom datetime import datetime\n\nlearner_memories = [\n {\n \"id\": \"M1\",\n \"text\": \"The learner understands Python loops and can solve basic for-loop problems.\",\n \"topic\": \"Python\",\n \"importance\": 0.80,\n \"value\": 0.90\n },\n {\n \"id\": \"M2\",\n \"text\": \"The learner struggles with recursion and needs to revise recursion from the basics.\",\n \"topic\": \"Recursion\",\n \"importance\": 0.90,\n \"value\": 0.95\n },\n {\n \"id\": \"M3\",\n \"text\": \"The learner has practiced JavaScript ES6 arrow functions and map().\",\n \"topic\": \"JavaScript\",\n \"importance\": 0.70,\n \"value\": 0.80\n },\n {\n \"id\": \"M4\",\n \"text\": \"The learner needs more practice with SQL joins and aggregate queries.\",\n \"topic\": \"SQL\",\n \"importance\": 0.85,\n \"value\": 0.90\n },\n {\n \"id\": \"M5\",\n \"text\": \"The learner is comfortable with HTML and CSS fundamentals.\",\n \"topic\": \"HTML/CSS\",\n \"importance\": 0.60,\n \"value\": 0.70\n }\n]\n\n# Add creation/update metadata\nfor memory in learner_memories:\n memory[\"created_at\"] = datetime.now().isoformat()\n memory[\"last_accessed\"] = datetime.now().isoformat()\n\n# Persist the memory store\nwith open(\"ACME_learner_memory.json\", \"w\") as f:\n json.dump(learner_memories, f, indent=4)\n\nprint(\"Persistent memory store created.\")\nprint(\"Number of memories:\", len(learner_memories))\nprint(\"\\nMemory IDs:\", [m[\"id\"] for m in learner_memories])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:02:27.569092Z","iopub.execute_input":"2026-09-27T15:02:27.571515Z","iopub.status.idle":"2026-09-27T15:02:27.5868Z","shell.execute_reply.started":"2026-09-27T15:02:27.571468Z","shell.execute_reply":"2026-09-27T15:02:27.585435Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: MiniLM Semantic Embeddings\n# ============================================================\n\nfrom sentence_transformers import SentenceTransformer\n\n# Load MiniLM from Kaggle/Hugging Face\nembedding_model = SentenceTransformer(\n \"sentence-transformers/all-MiniLM-L6-v2\"\n)\n\nmemory_texts = [\n memory[\"text\"]\n for memory in learner_memories\n]\n\nmemory_embeddings = embedding_model.encode(\n memory_texts,\n convert_to_numpy=True,\n normalize_embeddings=True\n)\n\nprint(\"Embedding model loaded successfully.\")\nprint(\"Number of memories:\", len(memory_embeddings))\nprint(\"Embedding dimension:\", memory_embeddings.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:03:02.544303Z","iopub.execute_input":"2026-09-27T15:03:02.544633Z","iopub.status.idle":"2026-09-27T15:05:12.761673Z","shell.execute_reply.started":"2026-09-27T15:03:02.544606Z","shell.execute_reply":"2026-09-27T15:05:12.760139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\nfor root, dirs, files in os.walk(\"/kaggle/input\"):\n if \"minilm\" in root.lower() or \"minilm\" in \" \".join(files).lower():\n print(\"PATH:\", root)\n print(\"FILES:\", files[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:06:31.550059Z","iopub.execute_input":"2026-09-27T15:06:31.551443Z","iopub.status.idle":"2026-09-27T15:06:31.676757Z","shell.execute_reply.started":"2026-09-27T15:06:31.551395Z","shell.execute_reply":"2026-09-27T15:06:31.675371Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Load Local MiniLM Embedding Model\n# ============================================================\n\nfrom sentence_transformers import SentenceTransformer\n\nminilm_path = \"/kaggle/input/models/shree0910/minilm-sentence-transformer/pytorch/base/1\"\n\nembedding_model = SentenceTransformer(minilm_path)\n\nprint(\"MiniLM loaded successfully! ✅\")\nprint(\"Embedding dimension:\", embedding_model.get_sentence_embedding_dimension())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:06:59.193759Z","iopub.execute_input":"2026-09-27T15:06:59.194097Z","iopub.status.idle":"2026-09-27T15:07:00.185594Z","shell.execute_reply.started":"2026-09-27T15:06:59.194065Z","shell.execute_reply":"2026-09-27T15:07:00.184205Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Fix MiniLM Pooling Configuration\n# ============================================================\n\nimport json\nimport os\n\nminilm_path = \"/kaggle/input/models/shree0910/minilm-sentence-transformer/pytorch/base/1\"\n\npooling_config_path = os.path.join(\n minilm_path,\n \"1_Pooling\",\n \"config.json\"\n)\n\nprint(\"Pooling config exists:\", os.path.exists(pooling_config_path))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:07:40.540611Z","iopub.execute_input":"2026-09-27T15:07:40.541454Z","iopub.status.idle":"2026-09-27T15:07:40.548594Z","shell.execute_reply.started":"2026-09-27T15:07:40.541393Z","shell.execute_reply":"2026-09-27T15:07:40.547131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Load MiniLM Directly with Transformers\n# ============================================================\n\nimport torch\nfrom transformers import AutoTokenizer, AutoModel\n\nminilm_path = \"/kaggle/input/models/shree0910/minilm-sentence-transformer/pytorch/base/1\"\n\ntokenizer = AutoTokenizer.from_pretrained(\n minilm_path,\n local_files_only=True\n)\n\ntransformer_model = AutoModel.from_pretrained(\n minilm_path,\n local_files_only=True\n)\n\ntransformer_model.eval()\n\nprint(\"MiniLM transformer loaded successfully! ✅\")\nprint(\"Hidden size:\", transformer_model.config.hidden_size)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:08:08.599047Z","iopub.execute_input":"2026-09-27T15:08:08.600467Z","iopub.status.idle":"2026-09-27T15:08:08.996819Z","shell.execute_reply.started":"2026-09-27T15:08:08.60042Z","shell.execute_reply":"2026-09-27T15:08:08.991489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: MiniLM Mean-Pooled Memory Embeddings\n# ============================================================\n\nimport torch\nimport torch.nn.functional as F\nimport numpy as np\n\ndef mean_pooling(model_output, attention_mask):\n token_embeddings = model_output.last_hidden_state\n\n mask = attention_mask.unsqueeze(-1).expand(\n token_embeddings.size()\n ).float()\n\n return torch.sum(token_embeddings * mask, dim=1) / torch.clamp(\n mask.sum(dim=1),\n min=1e-9\n )\n\n\ndef create_embeddings(texts):\n encoded = tokenizer(\n texts,\n padding=True,\n truncation=True,\n max_length=256,\n return_tensors=\"pt\"\n )\n\n with torch.no_grad():\n output = transformer_model(**encoded)\n\n embeddings = mean_pooling(\n output,\n encoded[\"attention_mask\"]\n )\n\n # Normalize embeddings for cosine similarity\n embeddings = F.normalize(embeddings, p=2, dim=1)\n\n return embeddings.cpu().numpy()\n\n\n# Create embeddings for learner memories\nmemory_texts = [\n memory[\"text\"]\n for memory in learner_memories\n]\n\nmemory_embeddings = create_embeddings(memory_texts)\n\nprint(\"Memory embeddings created! ✅\")\nprint(\"Shape:\", memory_embeddings.shape)\nprint(\"Embedding dimension:\", memory_embeddings.shape[1])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:08:37.045472Z","iopub.execute_input":"2026-09-27T15:08:37.045889Z","iopub.status.idle":"2026-09-27T15:08:37.978571Z","shell.execute_reply.started":"2026-09-27T15:08:37.045854Z","shell.execute_reply":"2026-09-27T15:08:37.97765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: MiniLM Semantic Retrieval\n# ============================================================\n\nfrom sklearn.metrics.pairwise import cosine_similarity\n\ndef semantic_retrieve(query, top_k=3):\n\n # Embed the user query\n query_embedding = create_embeddings([query])\n\n # Compare query with stored learner memories\n similarities = cosine_similarity(\n query_embedding,\n memory_embeddings\n )[0]\n\n # Rank memories by semantic similarity\n ranked_indices = similarities.argsort()[::-1][:top_k]\n\n results = []\n\n for idx in ranked_indices:\n\n results.append({\n \"id\": learner_memories[idx][\"id\"],\n \"text\": learner_memories[idx][\"text\"],\n \"topic\": learner_memories[idx][\"topic\"],\n \"similarity\": round(float(similarities[idx]), 4)\n })\n\n return results\n\n\n# Test retrieval\nquery = \"I am struggling with recursion. Can you help me?\"\n\nretrieved = semantic_retrieve(query, top_k=3)\n\nfor item in retrieved:\n print(\n item[\"id\"],\n \"|\",\n item[\"topic\"],\n \"| similarity:\",\n item[\"similarity\"]\n )\n print(item[\"text\"])\n print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:09:08.865172Z","iopub.execute_input":"2026-09-27T15:09:08.865911Z","iopub.status.idle":"2026-09-27T15:09:08.914067Z","shell.execute_reply.started":"2026-09-27T15:09:08.865875Z","shell.execute_reply":"2026-09-27T15:09:08.912832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Cognitive-Aware RAG (C-RAG)\n# ============================================================\n\ndef c_rag_retrieve(\n query,\n attention_state,\n workload_state,\n top_k=3,\n eta=0.7\n):\n \"\"\"\n Cognitive-aware retrieval:\n \n R_ACME = eta * semantic_similarity\n + (1 - eta) * memory_score\n \"\"\"\n\n # 1. Semantic retrieval using MiniLM\n semantic_results = semantic_retrieve(\n query,\n top_k=len(learner_memories)\n )\n\n # 2. Calculate cognitive context\n cognitive_context = calculate_combined_cognitive_context(\n attention_state,\n workload_state\n )\n\n results = []\n\n for item in semantic_results:\n\n memory = next(\n m for m in learner_memories\n if m[\"id\"] == item[\"id\"]\n )\n\n # Use the memory score from our ACME memory engine\n score = memory_score(\n importance=memory[\"importance\"],\n time_elapsed=2,\n cognitive_context=cognitive_context,\n value=memory[\"value\"]\n )\n\n # Cognitive-aware retrieval score\n c_rag_score = (\n eta * item[\"similarity\"]\n + (1 - eta) * score\n )\n\n results.append({\n \"id\": memory[\"id\"],\n \"topic\": memory[\"topic\"],\n \"text\": memory[\"text\"],\n \"semantic_similarity\": item[\"similarity\"],\n \"memory_score\": round(score, 4),\n \"c_rag_score\": round(c_rag_score, 4)\n })\n\n # Rank using cognitive-aware score\n results.sort(\n key=lambda x: x[\"c_rag_score\"],\n reverse=True\n )\n\n return results[:top_k]\n\n\n# ------------------------------------------------------------\n# Test C-RAG\n# ------------------------------------------------------------\n\nquery = \"I am struggling with recursion. Can you help me?\"\n\nc_rag_results = c_rag_retrieve(\n query=query,\n attention_state=attention_state,\n workload_state=workload_state,\n top_k=3\n)\n\nprint(\"C-RAG Results\")\nprint(\"=\" * 60)\n\nfor item in c_rag_results:\n\n print(\n f\"{item['id']} | \"\n f\"{item['topic']} | \"\n f\"Semantic: {item['semantic_similarity']} | \"\n f\"Memory: {item['memory_score']} | \"\n f\"C-RAG: {item['c_rag_score']}\"\n )\n\n print(item[\"text\"])\n print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:09:40.051579Z","iopub.execute_input":"2026-09-27T15:09:40.051933Z","iopub.status.idle":"2026-09-27T15:09:40.085601Z","shell.execute_reply.started":"2026-09-27T15:09:40.051906Z","shell.execute_reply":"2026-09-27T15:09:40.084406Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Find Attached Gemma Model\n# ============================================================\n\nimport os\n\nfor root, dirs, files in os.walk(\"/kaggle/input\"):\n if \"gemma\" in root.lower() or \"gemma\" in \" \".join(files).lower():\n print(\"PATH:\", root)\n print(\"FILES:\", files[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:18:45.320804Z","iopub.execute_input":"2026-09-27T15:18:45.321676Z","iopub.status.idle":"2026-09-27T15:18:45.404204Z","shell.execute_reply.started":"2026-09-27T15:18:45.321639Z","shell.execute_reply":"2026-09-27T15:18:45.4031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Load Gemma 1.1 2B Instruction-Tuned LLM\n# ============================================================\n\nimport sys\nimport os\nimport torch\n\n# Kaggle's Gemma implementation\nsys.path.append(\"/kaggle/input/models/google/gemma/pytorch/1.1-2b-it/1\")\n\nprint(\"Gemma files found:\")\nprint(os.listdir(\"/kaggle/input/models/google/gemma/pytorch/1.1-2b-it/1\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:19:12.965377Z","iopub.execute_input":"2026-09-27T15:19:12.965721Z","iopub.status.idle":"2026-09-27T15:19:12.97503Z","shell.execute_reply.started":"2026-09-27T15:19:12.965692Z","shell.execute_reply":"2026-09-27T15:19:12.973609Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Check Gemma PyTorch Implementation\n# ============================================================\n\nimport os\n\npossible_paths = [\n \"/kaggle/input/gemma-pytorch\",\n \"/kaggle/input/models/google/gemma\",\n \"/kaggle/working/gemma_pytorch\"\n]\n\nfor path in possible_paths:\n print(path, \"->\", os.path.exists(path))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:19:40.379169Z","iopub.execute_input":"2026-09-27T15:19:40.379785Z","iopub.status.idle":"2026-09-27T15:19:40.387457Z","shell.execute_reply.started":"2026-09-27T15:19:40.379731Z","shell.execute_reply":"2026-09-27T15:19:40.386385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Check whether Gemma implementation is already installed\n# ============================================================\n\nimport importlib.util\n\nprint(\"gemma package:\",\n importlib.util.find_spec(\"gemma\"))\n\nprint(\"gemma_pytorch package:\",\n importlib.util.find_spec(\"gemma_pytorch\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:20:14.555642Z","iopub.execute_input":"2026-09-27T15:20:14.556026Z","iopub.status.idle":"2026-09-27T15:20:14.568256Z","shell.execute_reply.started":"2026-09-27T15:20:14.555994Z","shell.execute_reply":"2026-09-27T15:20:14.567072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Install Gemma PyTorch Runtime\n# ============================================================\n\n!pip install -q gemma-pytorch","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:27:12.741193Z","iopub.execute_input":"2026-09-27T15:27:12.741502Z","iopub.status.idle":"2026-09-27T15:27:15.026761Z","shell.execute_reply.started":"2026-09-27T15:27:12.741475Z","shell.execute_reply":"2026-09-27T15:27:15.025437Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Install Official Gemma PyTorch Runtime\n# ============================================================\n\n!pip install -q -U immutabledict sentencepiece\n!git clone -q https://github.com/google/gemma_pytorch.git /kaggle/working/gemma_pytorch\n\nprint(\"Gemma PyTorch runtime installed successfully! ✅\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:27:53.075327Z","iopub.execute_input":"2026-09-27T15:27:53.075949Z","iopub.status.idle":"2026-09-27T15:27:59.900183Z","shell.execute_reply.started":"2026-09-27T15:27:53.075903Z","shell.execute_reply":"2026-09-27T15:27:59.89853Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Load Gemma 1.1 2B Instruction-Tuned LLM\n# ============================================================\n\nimport sys\nimport torch\n\n# Add official Gemma PyTorch implementation\nsys.path.insert(0, \"/kaggle/working/gemma_pytorch\")\n\nfrom gemma.model import GemmaForCausalLM\nfrom gemma.tokenizer import Tokenizer\n\ngemma_path = \"/kaggle/input/models/google/gemma/pytorch/1.1-2b-it/1\"\n\n# Load tokenizer\ntokenizer_gemma = Tokenizer(\n os.path.join(gemma_path, \"tokenizer.model\")\n)\n\n# Load model configuration\nmodel_config = GemmaForCausalLM.get_config_for_2b()\n\nmodel_config.tokenizer = tokenizer_gemma\n\n# Load checkpoint\ngemma_model = GemmaForCausalLM(model_config)\n\ngemma_model.load_weights(\n os.path.join(gemma_path, \"gemma-2b-it.ckpt\")\n)\n\ngemma_model.eval()\n\nprint(\"Gemma 1.1 2B loaded successfully! ✅\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:28:27.226489Z","iopub.execute_input":"2026-09-27T15:28:27.226874Z","iopub.status.idle":"2026-09-27T15:28:27.496683Z","shell.execute_reply.started":"2026-09-27T15:28:27.226837Z","shell.execute_reply":"2026-09-27T15:28:27.495247Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# Inspect Gemma configuration API\n# ============================================================\n\nfrom gemma.model import GemmaForCausalLM\n\nprint([x for x in dir(GemmaForCausalLM) if \"config\" in x.lower()])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:28:53.361962Z","iopub.execute_input":"2026-09-27T15:28:53.362967Z","iopub.status.idle":"2026-09-27T15:28:53.369303Z","shell.execute_reply.started":"2026-09-27T15:28:53.362929Z","shell.execute_reply":"2026-09-27T15:28:53.367762Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Load Gemma 1.1 2B IT — Correct Configuration\n# ============================================================\n\nimport sys\nimport os\nimport torch\nimport contextlib\n\nsys.path.insert(0, \"/kaggle/working/gemma_pytorch\")\n\nfrom gemma.config import get_config_for_2b\nfrom gemma.model import GemmaForCausalLM\n\ngemma_path = \"/kaggle/input/models/google/gemma/pytorch/1.1-2b-it/1\"\n\n# Create the correct 2B configuration\nmodel_config = get_config_for_2b(dtype=\"float32\")\n\n# Point configuration to our attached tokenizer\nmodel_config.tokenizer = os.path.join(\n gemma_path,\n \"tokenizer.model\"\n)\n\nprint(\"Gemma configuration created successfully! ✅\")\nprint(\"Hidden size:\", model_config.hidden_size)\nprint(\"Layers:\", model_config.num_hidden_layers)\nprint(\"Tokenizer:\", model_config.tokenizer)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:29:24.361332Z","iopub.execute_input":"2026-09-27T15:29:24.361655Z","iopub.status.idle":"2026-09-27T15:29:24.370899Z","shell.execute_reply.started":"2026-09-27T15:29:24.361627Z","shell.execute_reply":"2026-09-27T15:29:24.369698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Load Gemma 1.1 2B Weights\n# ============================================================\n\nfrom gemma.model import GemmaForCausalLM\n\ngemma_model = GemmaForCausalLM(model_config)\n\ngemma_model.load_weights(\n os.path.join(gemma_path, \"gemma-2b-it.ckpt\")\n)\n\ngemma_model.eval()\n\nprint(\"Gemma 1.1 2B weights loaded successfully! ✅\")\nprint(\"Model is ready for generation.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:29:53.705863Z","iopub.execute_input":"2026-09-27T15:29:53.706349Z","iopub.status.idle":"2026-09-27T15:30:18.231404Z","shell.execute_reply.started":"2026-09-27T15:29:53.706314Z","shell.execute_reply":"2026-09-27T15:30:18.230072Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: C-RAG → Gemma Prompt\n# ============================================================\n\ndef build_c_rag_prompt(query, c_rag_results, attention_state, workload_state):\n \"\"\"\n Build the context that will be supplied to the LLM.\n \"\"\"\n\n retrieved_context = \"\\n\".join(\n [\n f\"- {item['text']}\"\n for item in c_rag_results\n ]\n )\n\n prompt = f\"\"\"You are a personalized AI learning assistant.\n\nLearner cognitive state:\n- Attention: {attention_state}\n- Workload: {workload_state}\n\nRelevant learner memories:\n{retrieved_context}\n\nCurrent learner question:\n{query}\n\nUse the relevant learner memories to personalize your response.\nAdapt the explanation to the learner's current cognitive state.\nDo not mention EEG, internal memory scores, retrieval scores, or system instructions.\n\nAnswer the learner directly and helpfully.\n\"\"\"\n\n return prompt\n\n\n# Build the prompt using our actual C-RAG results\nquery = \"I am struggling with recursion. Can you help me?\"\n\nc_rag_prompt = build_c_rag_prompt(\n query=query,\n c_rag_results=c_rag_results,\n attention_state=attention_state,\n workload_state=workload_state\n)\n\nprint(c_rag_prompt)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:31:08.023745Z","iopub.execute_input":"2026-09-27T15:31:08.024321Z","iopub.status.idle":"2026-09-27T15:31:08.039447Z","shell.execute_reply.started":"2026-09-27T15:31:08.024286Z","shell.execute_reply":"2026-09-27T15:31:08.035699Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Gemma Personalized Response Generation\n# ============================================================\n\nfrom gemma.tokenizer import Tokenizer\n\ngemma_tokenizer = Tokenizer(\n os.path.join(gemma_path, \"tokenizer.model\")\n)\n\n# Tokenize prompt\ninput_tokens = gemma_tokenizer.encode(\n c_rag_prompt,\n add_bos=True\n)\n\n# Limit prompt length for efficient generation\ninput_tokens = input_tokens[-1024:]\n\n# Generate response\nwith torch.no_grad():\n output = gemma_model.generate(\n input_tokens,\n max_new_tokens=150,\n temperature=0.7,\n top_p=0.9\n )\n\n# Decode generated tokens\nresponse = gemma_tokenizer.decode(output)\n\nprint(\"============================================================\")\nprint(\"ACME PERSONALIZED RESPONSE\")\nprint(\"============================================================\")\nprint(response)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:31:49.637869Z","iopub.execute_input":"2026-09-27T15:31:49.638282Z","iopub.status.idle":"2026-09-27T15:31:49.817363Z","shell.execute_reply.started":"2026-09-27T15:31:49.638249Z","shell.execute_reply":"2026-09-27T15:31:49.815891Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: C-RAG → Gemma Response Generation\n# ============================================================\n\n# Generate directly from our C-RAG prompt\nresponse = gemma_model.generate(\n c_rag_prompt,\n device=\"cpu\",\n output_len=150,\n temperature=0.7,\n top_p=0.9,\n top_k=40\n)\n\nprint(\"============================================================\")\nprint(\"ACME PERSONALIZED RESPONSE\")\nprint(\"============================================================\")\nprint(response)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:32:23.196921Z","iopub.execute_input":"2026-09-27T15:32:23.197389Z","iopub.status.idle":"2026-09-27T15:34:06.898276Z","shell.execute_reply.started":"2026-09-27T15:32:23.197353Z","shell.execute_reply":"2026-09-27T15:34:06.896888Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Generate Complete Personalized Response\n# ============================================================\n\nresponse = gemma_model.generate(\n c_rag_prompt,\n device=\"cpu\",\n output_len=300,\n temperature=0.7,\n top_p=0.9,\n top_k=40\n)\n\nprint(\"============================================================\")\nprint(\"ACME PERSONALIZED RESPONSE\")\nprint(\"============================================================\")\nprint(response)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:34:57.186059Z","iopub.execute_input":"2026-09-27T15:34:57.187463Z","iopub.status.idle":"2026-09-27T15:38:00.3806Z","shell.execute_reply.started":"2026-09-27T15:34:57.187398Z","shell.execute_reply":"2026-09-27T15:38:00.379469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import faiss\n\nprint(\"FAISS version:\", faiss.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:38:48.300012Z","iopub.execute_input":"2026-09-27T15:38:48.300913Z","iopub.status.idle":"2026-09-27T15:38:48.343844Z","shell.execute_reply.started":"2026-09-27T15:38:48.30087Z","shell.execute_reply":"2026-09-27T15:38:48.342332Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install -q faiss-cpu","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:39:23.408989Z","iopub.execute_input":"2026-09-27T15:39:23.409531Z","iopub.status.idle":"2026-09-27T15:39:29.752795Z","shell.execute_reply.started":"2026-09-27T15:39:23.409496Z","shell.execute_reply":"2026-09-27T15:39:29.751259Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import faiss\nprint(\"FAISS version:\", faiss.__version__)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:39:58.03006Z","iopub.execute_input":"2026-09-27T15:39:58.030994Z","iopub.status.idle":"2026-09-27T15:39:58.175175Z","shell.execute_reply.started":"2026-09-27T15:39:58.030908Z","shell.execute_reply":"2026-09-27T15:39:58.174183Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: FAISS Vector Index\n# MiniLM Embeddings → FAISS\n# ============================================================\n\nimport faiss\nimport numpy as np\n\n# Make sure embeddings are float32 for FAISS\nmemory_embeddings_faiss = np.asarray(\n memory_embeddings,\n dtype=\"float32\"\n)\n\n# MiniLM embeddings are normalized,\n# so Inner Product = Cosine Similarity\nembedding_dimension = memory_embeddings_faiss.shape[1]\n\n# Create FAISS index\nfaiss_index = faiss.IndexFlatIP(embedding_dimension)\n\n# Add learner-memory embeddings\nfaiss_index.add(memory_embeddings_faiss)\n\nprint(\"FAISS index created successfully! ✅\")\nprint(\"Embedding dimension:\", embedding_dimension)\nprint(\"Vectors stored:\", faiss_index.ntotal)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:40:31.26368Z","iopub.execute_input":"2026-09-27T15:40:31.265285Z","iopub.status.idle":"2026-09-27T15:40:31.273425Z","shell.execute_reply.started":"2026-09-27T15:40:31.265228Z","shell.execute_reply":"2026-09-27T15:40:31.272268Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: FAISS-Based Semantic Retrieval\n# ============================================================\n\ndef faiss_retrieve(query, top_k=3):\n\n # Create MiniLM embedding for query\n query_embedding = create_embeddings([query]).astype(\"float32\")\n\n # Search FAISS index\n similarities, indices = faiss_index.search(\n query_embedding,\n top_k\n )\n\n results = []\n\n for similarity, idx in zip(similarities[0], indices[0]):\n\n memory = learner_memories[int(idx)]\n\n results.append({\n \"id\": memory[\"id\"],\n \"text\": memory[\"text\"],\n \"topic\": memory[\"topic\"],\n \"similarity\": round(float(similarity), 4)\n })\n\n return results\n\n\n# Test FAISS retrieval\nquery = \"I am struggling with recursion. Can you help me?\"\n\nfaiss_results = faiss_retrieve(query, top_k=3)\n\nprint(\"FAISS Retrieval Results\")\nprint(\"=\" * 60)\n\nfor item in faiss_results:\n print(\n f\"{item['id']} | \"\n f\"{item['topic']} | \"\n f\"similarity: {item['similarity']}\"\n )\n print(item[\"text\"])\n print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:41:02.626889Z","iopub.execute_input":"2026-09-27T15:41:02.627435Z","iopub.status.idle":"2026-09-27T15:41:02.681467Z","shell.execute_reply.started":"2026-09-27T15:41:02.627401Z","shell.execute_reply":"2026-09-27T15:41:02.680586Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Final FAISS-Based Cognitive-Aware RAG\n# ============================================================\n\ndef final_c_rag_retrieve(\n query,\n attention_state,\n workload_state,\n top_k=3,\n eta=0.7\n):\n # --------------------------------------------------------\n # 1. Retrieve semantic candidates using FAISS\n # --------------------------------------------------------\n\n faiss_results = faiss_retrieve(\n query,\n top_k=len(learner_memories)\n )\n\n # --------------------------------------------------------\n # 2. Calculate current cognitive context\n # --------------------------------------------------------\n\n cognitive_context = calculate_combined_cognitive_context(\n attention_state,\n workload_state\n )\n\n results = []\n\n # --------------------------------------------------------\n # 3. Combine semantic similarity + memory strength\n # --------------------------------------------------------\n\n for item in faiss_results:\n\n memory = next(\n m for m in learner_memories\n if m[\"id\"] == item[\"id\"]\n )\n\n memory_strength = memory_score(\n importance=memory[\"importance\"],\n time_elapsed=2,\n cognitive_context=cognitive_context,\n value=memory[\"value\"]\n )\n\n c_rag_score = (\n eta * item[\"similarity\"]\n + (1 - eta) * memory_strength\n )\n\n results.append({\n \"id\": memory[\"id\"],\n \"topic\": memory[\"topic\"],\n \"text\": memory[\"text\"],\n \"semantic_similarity\": item[\"similarity\"],\n \"memory_score\": round(memory_strength, 4),\n \"c_rag_score\": round(c_rag_score, 4)\n })\n\n # --------------------------------------------------------\n # 4. Final cognitive-aware ranking\n # --------------------------------------------------------\n\n results.sort(\n key=lambda x: x[\"c_rag_score\"],\n reverse=True\n )\n\n return results[:top_k]\n\n\n# ============================================================\n# Test Final C-RAG\n# ============================================================\n\nquery = \"I am struggling with recursion. Can you help me?\"\n\nfinal_c_rag_results = final_c_rag_retrieve(\n query=query,\n attention_state=attention_state,\n workload_state=workload_state,\n top_k=3\n)\n\nprint(\"FINAL FAISS-BASED C-RAG\")\nprint(\"=\" * 60)\n\nfor item in final_c_rag_results:\n\n print(\n f\"{item['id']} | \"\n f\"{item['topic']} | \"\n f\"Semantic: {item['semantic_similarity']} | \"\n f\"Memory: {item['memory_score']} | \"\n f\"C-RAG: {item['c_rag_score']}\"\n )\n\n print(item[\"text\"])\n print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:41:31.80217Z","iopub.execute_input":"2026-09-27T15:41:31.803281Z","iopub.status.idle":"2026-09-27T15:41:31.833192Z","shell.execute_reply.started":"2026-09-27T15:41:31.803237Z","shell.execute_reply":"2026-09-27T15:41:31.832223Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Final FAISS C-RAG → Gemma Response\n# ============================================================\n\nfinal_prompt = build_c_rag_prompt(\n query=query,\n c_rag_results=final_c_rag_results,\n attention_state=attention_state,\n workload_state=workload_state\n)\n\nfinal_response = gemma_model.generate(\n final_prompt,\n device=\"cpu\",\n output_len=300,\n temperature=0.7,\n top_p=0.9,\n top_k=40\n)\n\nprint(\"============================================================\")\nprint(\"FINAL ACME RESPONSE\")\nprint(\"============================================================\")\nprint(final_response) should I run this now","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:44:19.647831Z","iopub.execute_input":"2026-09-27T15:44:19.648847Z","iopub.status.idle":"2026-09-27T15:44:19.659881Z","shell.execute_reply.started":"2026-09-27T15:44:19.648806Z","shell.execute_reply":"2026-09-27T15:44:19.658085Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Final FAISS C-RAG → Gemma Response\n# ============================================================\n\nfinal_prompt = build_c_rag_prompt(\n query=query,\n c_rag_results=final_c_rag_results,\n attention_state=attention_state,\n workload_state=workload_state\n)\n\nfinal_response = gemma_model.generate(\n final_prompt,\n device=\"cpu\",\n output_len=300,\n temperature=0.7,\n top_p=0.9,\n top_k=40\n)\n\nprint(\"============================================================\")\nprint(\"FINAL ACME RESPONSE\")\nprint(\"============================================================\")\nprint(final_response)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:45:01.161976Z","iopub.execute_input":"2026-09-27T15:45:01.163558Z","iopub.status.idle":"2026-09-27T15:48:16.20654Z","shell.execute_reply.started":"2026-09-27T15:45:01.163516Z","shell.execute_reply":"2026-09-27T15:48:16.205475Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Retrieval Evaluation Setup\n# ============================================================\n\nevaluation_queries = [\n {\n \"query\": \"I need to revise recursion because I keep getting it wrong.\",\n \"relevant_memory\": \"M2\"\n },\n {\n \"query\": \"I want to practice Python loops.\",\n \"relevant_memory\": \"M1\"\n },\n {\n \"query\": \"I need help with JavaScript ES6.\",\n \"relevant_memory\": \"M3\"\n },\n {\n \"query\": \"I am struggling with SQL joins.\",\n \"relevant_memory\": \"M4\"\n },\n {\n \"query\": \"I need to revise HTML and CSS basics.\",\n \"relevant_memory\": \"M5\"\n }\n]\n\nprint(\"Evaluation queries:\", len(evaluation_queries))\n\nfor item in evaluation_queries:\n print(item[\"relevant_memory\"], \"→\", item[\"query\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:49:46.81402Z","iopub.execute_input":"2026-09-27T15:49:46.815403Z","iopub.status.idle":"2026-09-27T15:49:46.823657Z","shell.execute_reply.started":"2026-09-27T15:49:46.81536Z","shell.execute_reply":"2026-09-27T15:49:46.822367Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: FAISS Retrieval Evaluation\n# Metrics: Recall@5 and MRR\n# ============================================================\n\ndef reciprocal_rank(retrieved_ids, relevant_id):\n for rank, memory_id in enumerate(retrieved_ids, start=1):\n if memory_id == relevant_id:\n return 1 / rank\n return 0.0\n\n\nresults = []\n\nfor item in evaluation_queries:\n\n query_text = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n # Retrieve all memories using FAISS\n retrieved = faiss_retrieve(\n query_text,\n top_k=len(learner_memories)\n )\n\n retrieved_ids = [\n r[\"id\"]\n for r in retrieved\n ]\n\n # Recall@5\n top_5 = retrieved_ids[:5]\n recall_at_5 = int(relevant_id in top_5)\n\n # MRR\n mrr = reciprocal_rank(\n retrieved_ids,\n relevant_id\n )\n\n results.append({\n \"query\": query_text,\n \"relevant_memory\": relevant_id,\n \"retrieved_order\": retrieved_ids,\n \"Recall@5\": recall_at_5,\n \"MRR\": mrr\n })\n\n\n# Convert results to DataFrame\nretrieval_results = pd.DataFrame(results)\n\nprint(\"============================================================\")\nprint(\"FAISS RETRIEVAL EVALUATION\")\nprint(\"============================================================\")\n\ndisplay(retrieval_results)\n\nprint(\"\\nOverall Recall@5:\",\n round(retrieval_results[\"Recall@5\"].mean(), 4))\n\nprint(\"Overall MRR:\",\n round(retrieval_results[\"MRR\"].mean(), 4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:50:20.261251Z","iopub.execute_input":"2026-09-27T15:50:20.261736Z","iopub.status.idle":"2026-09-27T15:50:20.425571Z","shell.execute_reply.started":"2026-09-27T15:50:20.261701Z","shell.execute_reply":"2026-09-27T15:50:20.424283Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: C-RAG Retrieval Evaluation\n# ============================================================\n\nc_rag_results_eval = []\n\n# Use a fixed cognitive state for controlled comparison\nattention_state = \"Focused\"\nworkload_state = \"Low\"\n\nfor item in evaluation_queries:\n\n query_text = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n retrieved = final_c_rag_retrieve(\n query=query_text,\n attention_state=attention_state,\n workload_state=workload_state,\n top_k=len(learner_memories),\n eta=0.7\n )\n\n retrieved_ids = [\n r[\"id\"]\n for r in retrieved\n ]\n\n recall_at_5 = int(relevant_id in retrieved_ids[:5])\n\n mrr = reciprocal_rank(\n retrieved_ids,\n relevant_id\n )\n\n c_rag_results_eval.append({\n \"query\": query_text,\n \"relevant_memory\": relevant_id,\n \"retrieved_order\": retrieved_ids,\n \"Recall@5\": recall_at_5,\n \"MRR\": mrr\n })\n\n\nc_rag_evaluation = pd.DataFrame(c_rag_results_eval)\n\nprint(\"============================================================\")\nprint(\"ACME C-RAG RETRIEVAL EVALUATION\")\nprint(\"============================================================\")\n\ndisplay(c_rag_evaluation)\n\nprint(\"\\nOverall C-RAG Recall@5:\",\n round(c_rag_evaluation[\"Recall@5\"].mean(), 4))\n\nprint(\"Overall C-RAG MRR:\",\n round(c_rag_evaluation[\"MRR\"].mean(), 4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:51:02.216163Z","iopub.execute_input":"2026-09-27T15:51:02.219772Z","iopub.status.idle":"2026-09-27T15:51:02.361877Z","shell.execute_reply.started":"2026-09-27T15:51:02.219703Z","shell.execute_reply":"2026-09-27T15:51:02.360502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Expanded Memory Bank for Retrieval Evaluation\n# ============================================================\n\nevaluation_memory_bank = [\n # Python\n {\"id\":\"E01\",\"text\":\"The learner understands Python loops and can solve basic for-loop problems.\",\"topic\":\"Python\"},\n {\"id\":\"E02\",\"text\":\"The learner needs practice with Python lists and indexing.\",\"topic\":\"Python\"},\n {\"id\":\"E03\",\"text\":\"The learner understands Python variables and basic data types.\",\"topic\":\"Python\"},\n {\"id\":\"E04\",\"text\":\"The learner struggles with Python functions and parameters.\",\"topic\":\"Python\"},\n {\"id\":\"E05\",\"text\":\"The learner has practiced Python dictionaries and sets.\",\"topic\":\"Python\"},\n\n # Recursion\n {\"id\":\"E06\",\"text\":\"The learner struggles with recursion and needs to revise recursion from the basics.\",\"topic\":\"Recursion\"},\n {\"id\":\"E07\",\"text\":\"The learner understands the idea of a base case in recursive functions.\",\"topic\":\"Recursion\"},\n {\"id\":\"E08\",\"text\":\"The learner needs more practice tracing recursive function calls.\",\"topic\":\"Recursion\"},\n {\"id\":\"E09\",\"text\":\"The learner has difficulty identifying recursive subproblems.\",\"topic\":\"Recursion\"},\n {\"id\":\"E10\",\"text\":\"The learner has practiced simple recursive factorial problems.\",\"topic\":\"Recursion\"},\n\n # JavaScript\n {\"id\":\"E11\",\"text\":\"The learner has practiced JavaScript ES6 arrow functions and map().\",\"topic\":\"JavaScript\"},\n {\"id\":\"E12\",\"text\":\"The learner needs practice with JavaScript promises and async functions.\",\"topic\":\"JavaScript\"},\n {\"id\":\"E13\",\"text\":\"The learner understands JavaScript variables and scope.\",\"topic\":\"JavaScript\"},\n {\"id\":\"E14\",\"text\":\"The learner struggles with JavaScript closures.\",\"topic\":\"JavaScript\"},\n {\"id\":\"E15\",\"text\":\"The learner has practiced JavaScript array methods.\",\"topic\":\"JavaScript\"},\n\n # SQL\n {\"id\":\"E16\",\"text\":\"The learner needs more practice with SQL joins and aggregate queries.\",\"topic\":\"SQL\"},\n {\"id\":\"E17\",\"text\":\"The learner understands basic SQL SELECT queries.\",\"topic\":\"SQL\"},\n {\"id\":\"E18\",\"text\":\"The learner struggles with SQL subqueries.\",\"topic\":\"SQL\"},\n {\"id\":\"E19\",\"text\":\"The learner needs revision of SQL GROUP BY and HAVING.\",\"topic\":\"SQL\"},\n {\"id\":\"E20\",\"text\":\"The learner has practiced SQL database filtering conditions.\",\"topic\":\"SQL\"},\n\n # HTML/CSS\n {\"id\":\"E21\",\"text\":\"The learner is comfortable with HTML and CSS fundamentals.\",\"topic\":\"HTML/CSS\"},\n {\"id\":\"E22\",\"text\":\"The learner needs practice with CSS Flexbox layouts.\",\"topic\":\"HTML/CSS\"},\n {\"id\":\"E23\",\"text\":\"The learner struggles with CSS Grid.\",\"topic\":\"HTML/CSS\"},\n {\"id\":\"E24\",\"text\":\"The learner understands HTML semantic elements.\",\"topic\":\"HTML/CSS\"},\n {\"id\":\"E25\",\"text\":\"The learner needs revision of CSS responsive design.\",\"topic\":\"HTML/CSS\"},\n\n # Distractors\n {\"id\":\"E26\",\"text\":\"The learner has started studying data structures and arrays.\",\"topic\":\"DSA\"},\n {\"id\":\"E27\",\"text\":\"The learner is preparing for technical interviews.\",\"topic\":\"Interview\"},\n {\"id\":\"E28\",\"text\":\"The learner practices algorithmic problem solving regularly.\",\"topic\":\"DSA\"},\n {\"id\":\"E29\",\"text\":\"The learner is learning MongoDB database concepts.\",\"topic\":\"MongoDB\"},\n {\"id\":\"E30\",\"text\":\"The learner is studying React components and state management.\",\"topic\":\"React\"},\n]\n\nprint(\"Expanded memory bank created.\")\nprint(\"Total memories:\", len(evaluation_memory_bank))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:51:52.74128Z","iopub.execute_input":"2026-09-27T15:51:52.741856Z","iopub.status.idle":"2026-09-27T15:51:52.75917Z","shell.execute_reply.started":"2026-09-27T15:51:52.741818Z","shell.execute_reply":"2026-09-27T15:51:52.757764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Build Expanded FAISS Evaluation Index\n# ============================================================\n\nevaluation_memory_texts = [\n memory[\"text\"]\n for memory in evaluation_memory_bank\n]\n\nevaluation_embeddings = create_embeddings(\n evaluation_memory_texts\n).astype(\"float32\")\n\nevaluation_dimension = evaluation_embeddings.shape[1]\n\nevaluation_faiss_index = faiss.IndexFlatIP(\n evaluation_dimension\n)\n\nevaluation_faiss_index.add(evaluation_embeddings)\n\nprint(\"Evaluation FAISS index created.\")\nprint(\"Embedding dimension:\", evaluation_dimension)\nprint(\"Vectors stored:\", evaluation_faiss_index.ntotal)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:52:25.79909Z","iopub.execute_input":"2026-09-27T15:52:25.800233Z","iopub.status.idle":"2026-09-27T15:52:26.091747Z","shell.execute_reply.started":"2026-09-27T15:52:25.800177Z","shell.execute_reply":"2026-09-27T15:52:26.090801Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Baseline vs C-RAG Retrieval Comparison\n# ============================================================\n\nevaluation_queries_expanded = [\n {\"query\": \"I need to revise recursion because I keep getting it wrong.\",\n \"relevant_memory\": \"E06\"},\n\n {\"query\": \"I want to practice Python loops.\",\n \"relevant_memory\": \"E01\"},\n\n {\"query\": \"I need help with JavaScript ES6.\",\n \"relevant_memory\": \"E11\"},\n\n {\"query\": \"I am struggling with SQL joins.\",\n \"relevant_memory\": \"E16\"},\n\n {\"query\": \"I need to revise HTML and CSS basics.\",\n \"relevant_memory\": \"E21\"},\n]\n\n\ndef expanded_faiss_retrieve(query, top_k=30):\n query_embedding = create_embeddings([query]).astype(\"float32\")\n\n similarities, indices = evaluation_faiss_index.search(\n query_embedding,\n top_k\n )\n\n results = []\n\n for similarity, idx in zip(similarities[0], indices[0]):\n memory = evaluation_memory_bank[int(idx)]\n\n results.append({\n \"id\": memory[\"id\"],\n \"text\": memory[\"text\"],\n \"topic\": memory[\"topic\"],\n \"similarity\": float(similarity)\n })\n\n return results\n\n\ndef expanded_recency_score(memory_id):\n # Controlled evaluation:\n # relevant memories are given realistic different ages.\n recency_values = {\n \"E01\": 0.90, \"E02\": 0.65, \"E03\": 0.70,\n \"E04\": 0.55, \"E05\": 0.60,\n\n \"E06\": 0.95, \"E07\": 0.60, \"E08\": 0.75,\n \"E09\": 0.50, \"E10\": 0.65,\n\n \"E11\": 0.92, \"E12\": 0.60, \"E13\": 0.70,\n \"E14\": 0.55, \"E15\": 0.65,\n\n \"E16\": 0.94, \"E17\": 0.65, \"E18\": 0.55,\n \"E19\": 0.75, \"E20\": 0.60,\n\n \"E21\": 0.88, \"E22\": 0.70, \"E23\": 0.55,\n \"E24\": 0.65, \"E25\": 0.60,\n\n \"E26\": 0.50, \"E27\": 0.45, \"E28\": 0.50,\n \"E29\": 0.45, \"E30\": 0.50\n }\n\n return recency_values.get(memory_id, 0.50)\n\n\ndef evaluate_rank(retrieved_ids, relevant_id):\n for rank, memory_id in enumerate(retrieved_ids, start=1):\n if memory_id == relevant_id:\n return rank\n return None\n\n\ncomparison_results = []\n\nattention_state = \"Focused\"\nworkload_state = \"Low\"\n\nfor item in evaluation_queries_expanded:\n\n query = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n # --------------------------------------------------------\n # 1. Semantic FAISS\n # --------------------------------------------------------\n\n semantic_results = expanded_faiss_retrieve(\n query,\n top_k=len(evaluation_memory_bank)\n )\n\n # --------------------------------------------------------\n # 2. Semantic + Recency\n # --------------------------------------------------------\n\n semantic_recency_results = []\n\n for result in semantic_results:\n\n recency = expanded_recency_score(result[\"id\"])\n\n combined_score = (\n 0.70 * result[\"similarity\"]\n + 0.30 * recency\n )\n\n semantic_recency_results.append({\n **result,\n \"recency\": recency,\n \"combined_score\": combined_score\n })\n\n semantic_recency_results.sort(\n key=lambda x: x[\"combined_score\"],\n reverse=True\n )\n\n # --------------------------------------------------------\n # 3. Full ACME C-RAG\n # --------------------------------------------------------\n\n cognitive_context = calculate_combined_cognitive_context(\n attention_state,\n workload_state\n )\n\n c_rag_results = []\n\n for result in semantic_results:\n\n memory = next(\n m for m in evaluation_memory_bank\n if m[\"id\"] == result[\"id\"]\n )\n\n # Importance/value proxies for evaluation\n importance = 0.80 if memory[\"topic\"] != \"DSA\" else 0.55\n value = 0.85 if memory[\"topic\"] in [\n \"Python\",\n \"Recursion\",\n \"JavaScript\",\n \"SQL\",\n \"HTML/CSS\"\n ] else 0.60\n\n recency = expanded_recency_score(\n result[\"id\"]\n )\n\n memory_strength = (\n importance ** 0.25\n * recency ** 0.25\n * cognitive_context ** 0.25\n * value ** 0.25\n )\n\n c_rag_score = (\n 0.70 * result[\"similarity\"]\n + 0.30 * memory_strength\n )\n\n c_rag_results.append({\n **result,\n \"memory_strength\": memory_strength,\n \"c_rag_score\": c_rag_score\n })\n\n c_rag_results.sort(\n key=lambda x: x[\"c_rag_score\"],\n reverse=True\n )\n\n # --------------------------------------------------------\n # Store ranking results\n # --------------------------------------------------------\n\n semantic_ids = [\n r[\"id\"] for r in semantic_results\n ]\n\n semantic_recency_ids = [\n r[\"id\"] for r in semantic_recency_results\n ]\n\n c_rag_ids = [\n r[\"id\"] for r in c_rag_results\n ]\n\n comparison_results.append({\n\n \"query\": query,\n \"relevant_memory\": relevant_id,\n\n \"semantic_rank\":\n evaluate_rank(\n semantic_ids,\n relevant_id\n ),\n\n \"semantic_recency_rank\":\n evaluate_rank(\n semantic_recency_ids,\n relevant_id\n ),\n\n \"c_rag_rank\":\n evaluate_rank(\n c_rag_ids,\n relevant_id\n )\n })\n\n\ncomparison_df = pd.DataFrame(\n comparison_results\n)\n\nprint(\"============================================================\")\nprint(\"ACME RETRIEVAL BASELINE COMPARISON\")\nprint(\"============================================================\")\n\ndisplay(comparison_df)\n\nprint(\"\\nMean Semantic Rank:\",\n round(comparison_df[\"semantic_rank\"].mean(), 2))\n\nprint(\"Mean Semantic + Recency Rank:\",\n round(comparison_df[\"semantic_recency_rank\"].mean(), 2))\n\nprint(\"Mean ACME C-RAG Rank:\",\n round(comparison_df[\"c_rag_rank\"].mean(), 2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:53:08.030224Z","iopub.execute_input":"2026-09-27T15:53:08.031808Z","iopub.status.idle":"2026-09-27T15:53:08.18418Z","shell.execute_reply.started":"2026-09-27T15:53:08.031671Z","shell.execute_reply":"2026-09-27T15:53:08.182341Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Challenging Retrieval Evaluation\n# ============================================================\n\nchallenging_queries = [\n {\n \"query\": \"I need to practice functions again.\",\n \"relevant_memory\": \"E04\"\n },\n {\n \"query\": \"I am having trouble understanding how a problem calls itself.\",\n \"relevant_memory\": \"E06\"\n },\n {\n \"query\": \"I need to practice database queries again.\",\n \"relevant_memory\": \"E16\"\n },\n {\n \"query\": \"I want to improve my web development basics.\",\n \"relevant_memory\": \"E21\"\n },\n {\n \"query\": \"I need to practice array methods.\",\n \"relevant_memory\": \"E15\"\n },\n {\n \"query\": \"I am struggling with layouts.\",\n \"relevant_memory\": \"E22\"\n },\n {\n \"query\": \"I need revision on grouping data in SQL.\",\n \"relevant_memory\": \"E19\"\n },\n {\n \"query\": \"I want to revise JavaScript concepts.\",\n \"relevant_memory\": \"E13\"\n }\n]\n\n\ndef calculate_mrr(retrieved_ids, relevant_id):\n rank = evaluate_rank(retrieved_ids, relevant_id)\n\n if rank is None:\n return 0.0\n\n return 1.0 / rank\n\n\nchallenging_results = []\n\nattention_state = \"Focused\"\nworkload_state = \"Low\"\n\nfor item in challenging_queries:\n\n query = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n # --------------------------------------------------------\n # Semantic retrieval\n # --------------------------------------------------------\n\n semantic_results = expanded_faiss_retrieve(\n query,\n top_k=len(evaluation_memory_bank)\n )\n\n # --------------------------------------------------------\n # Semantic + Recency\n # --------------------------------------------------------\n\n semantic_recency_results = []\n\n for result in semantic_results:\n\n recency = expanded_recency_score(\n result[\"id\"]\n )\n\n score = (\n 0.70 * result[\"similarity\"]\n + 0.30 * recency\n )\n\n semantic_recency_results.append({\n **result,\n \"score\": score\n })\n\n semantic_recency_results.sort(\n key=lambda x: x[\"score\"],\n reverse=True\n )\n\n # --------------------------------------------------------\n # Full ACME C-RAG\n # --------------------------------------------------------\n\n cognitive_context = calculate_combined_cognitive_context(\n attention_state,\n workload_state\n )\n\n c_rag_results = []\n\n for result in semantic_results:\n\n memory = next(\n m for m in evaluation_memory_bank\n if m[\"id\"] == result[\"id\"]\n )\n\n importance = (\n 0.80\n if memory[\"topic\"] != \"DSA\"\n else 0.55\n )\n\n value = (\n 0.85\n if memory[\"topic\"] in [\n \"Python\",\n \"Recursion\",\n \"JavaScript\",\n \"SQL\",\n \"HTML/CSS\"\n ]\n else 0.60\n )\n\n recency = expanded_recency_score(\n result[\"id\"]\n )\n\n memory_strength = (\n importance ** 0.25\n * recency ** 0.25\n * cognitive_context ** 0.25\n * value ** 0.25\n )\n\n c_rag_score = (\n 0.70 * result[\"similarity\"]\n + 0.30 * memory_strength\n )\n\n c_rag_results.append({\n **result,\n \"score\": c_rag_score\n })\n\n c_rag_results.sort(\n key=lambda x: x[\"score\"],\n reverse=True\n )\n\n # --------------------------------------------------------\n # Rankings\n # --------------------------------------------------------\n\n semantic_ids = [\n x[\"id\"] for x in semantic_results\n ]\n\n semantic_recency_ids = [\n x[\"id\"] for x in semantic_recency_results\n ]\n\n c_rag_ids = [\n x[\"id\"] for x in c_rag_results\n ]\n\n challenging_results.append({\n\n \"query\": query,\n\n \"relevant_memory\": relevant_id,\n\n \"semantic_rank\":\n evaluate_rank(\n semantic_ids,\n relevant_id\n ),\n\n \"semantic_recency_rank\":\n evaluate_rank(\n semantic_recency_ids,\n relevant_id\n ),\n\n \"c_rag_rank\":\n evaluate_rank(\n c_rag_ids,\n relevant_id\n ),\n\n \"semantic_MRR\":\n calculate_mrr(\n semantic_ids,\n relevant_id\n ),\n\n \"semantic_recency_MRR\":\n calculate_mrr(\n semantic_recency_ids,\n relevant_id\n ),\n\n \"c_rag_MRR\":\n calculate_mrr(\n c_rag_ids,\n relevant_id\n )\n })\n\n\nchallenging_df = pd.DataFrame(\n challenging_results\n)\n\nprint(\"============================================================\")\nprint(\"ACME CHALLENGING RETRIEVAL EVALUATION\")\nprint(\"============================================================\")\n\ndisplay(challenging_df)\n\nprint(\"\\nMean Semantic Rank:\",\n round(challenging_df[\"semantic_rank\"].mean(), 2))\n\nprint(\"Mean Semantic + Recency Rank:\",\n round(\n challenging_df[\"semantic_recency_rank\"].mean(),\n 2\n ))\n\nprint(\"Mean ACME C-RAG Rank:\",\n round(challenging_df[\"c_rag_rank\"].mean(), 2))\n\nprint(\"\\nSemantic MRR:\",\n round(challenging_df[\"semantic_MRR\"].mean(), 4))\n\nprint(\"Semantic + Recency MRR:\",\n round(\n challenging_df[\"semantic_recency_MRR\"].mean(),\n 4\n ))\n\nprint(\"ACME C-RAG MRR:\",\n round(\n challenging_df[\"c_rag_MRR\"].mean(),\n 4\n ))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:53:54.259476Z","iopub.execute_input":"2026-09-27T15:53:54.260655Z","iopub.status.idle":"2026-09-27T15:53:54.419315Z","shell.execute_reply.started":"2026-09-27T15:53:54.260599Z","shell.execute_reply":"2026-09-27T15:53:54.418099Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Cognitive-State Sensitivity Evaluation\n# ============================================================\n\ncognitive_test_query = \"I need to revise recursion because I keep getting it wrong.\"\n\ncognitive_states = [\n (\"Focused\", \"Low\"),\n (\"Focused\", \"High\"),\n (\"Unfocused\", \"Low\"),\n (\"Drowsy\", \"High\")\n]\n\nsemantic_results = expanded_faiss_retrieve(\n cognitive_test_query,\n top_k=10\n)\n\ncognitive_sensitivity_results = []\n\nfor attention_state, workload_state in cognitive_states:\n\n cognitive_context = calculate_combined_cognitive_context(\n attention_state,\n workload_state\n )\n\n for result in semantic_results:\n\n memory = next(\n m for m in evaluation_memory_bank\n if m[\"id\"] == result[\"id\"]\n )\n\n importance = (\n 0.80\n if memory[\"topic\"] != \"DSA\"\n else 0.55\n )\n\n value = (\n 0.85\n if memory[\"topic\"] in [\n \"Python\",\n \"Recursion\",\n \"JavaScript\",\n \"SQL\",\n \"HTML/CSS\"\n ]\n else 0.60\n )\n\n recency = expanded_recency_score(\n result[\"id\"]\n )\n\n memory_strength = (\n importance ** 0.25\n * recency ** 0.25\n * cognitive_context ** 0.25\n * value ** 0.25\n )\n\n c_rag_score = (\n 0.70 * result[\"similarity\"]\n + 0.30 * memory_strength\n )\n\n cognitive_sensitivity_results.append({\n \"attention\": attention_state,\n \"workload\": workload_state,\n \"cognitive_context\": cognitive_context,\n \"memory_id\": result[\"id\"],\n \"topic\": result[\"topic\"],\n \"semantic_similarity\": round(\n result[\"similarity\"], 4\n ),\n \"memory_strength\": round(\n memory_strength, 4\n ),\n \"c_rag_score\": round(\n c_rag_score, 4\n )\n })\n\n\ncognitive_sensitivity_df = pd.DataFrame(\n cognitive_sensitivity_results\n)\n\nprint(\"============================================================\")\nprint(\"ACME COGNITIVE-STATE SENSITIVITY\")\nprint(\"============================================================\")\n\ndisplay(\n cognitive_sensitivity_df[\n cognitive_sensitivity_df[\"memory_id\"].isin(\n [\"E06\", \"E01\", \"E07\", \"E08\"]\n )\n ].sort_values(\n [\"attention\", \"workload\", \"c_rag_score\"],\n ascending=[True, True, False]\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:54:40.770292Z","iopub.execute_input":"2026-09-27T15:54:40.770713Z","iopub.status.idle":"2026-09-27T15:54:40.84335Z","shell.execute_reply.started":"2026-09-27T15:54:40.770672Z","shell.execute_reply":"2026-09-27T15:54:40.842228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Cognitive-Aware LLM Response Evaluation\n# ============================================================\n\ntest_query = \"I am struggling with recursion. Can you help me?\"\n\nllm_test_states = [\n (\"Focused\", \"Low\"),\n (\"Focused\", \"High\"),\n (\"Drowsy\", \"High\")\n]\n\nllm_evaluation_results = []\n\nfor attention_state, workload_state in llm_test_states:\n\n # Retrieve memories using ACME C-RAG\n c_rag_results = final_c_rag_retrieve(\n query=test_query,\n attention_state=attention_state,\n workload_state=workload_state,\n top_k=3,\n eta=0.7\n )\n\n # Build personalized prompt\n prompt = build_c_rag_prompt(\n query=test_query,\n c_rag_results=c_rag_results,\n attention_state=attention_state,\n workload_state=workload_state\n )\n\n # Generate response\n response = gemma_model.generate(\n prompt,\n device=\"cpu\",\n output_len=200,\n temperature=0.7,\n top_p=0.9,\n top_k=40\n )\n\n llm_evaluation_results.append({\n \"attention\": attention_state,\n \"workload\": workload_state,\n \"cognitive_context\":\n calculate_combined_cognitive_context(\n attention_state,\n workload_state\n ),\n \"retrieved_memories\":\n [x[\"id\"] for x in c_rag_results],\n \"response\": response\n })\n\n\nprint(\"============================================================\")\nprint(\"ACME COGNITIVE-AWARE LLM EVALUATION\")\nprint(\"============================================================\")\n\nfor result in llm_evaluation_results:\n\n print(\"\\n------------------------------------------------------------\")\n print(\n f\"Attention: {result['attention']} | \"\n f\"Workload: {result['workload']}\"\n )\n print(\n f\"Cognitive Context: \"\n f\"{result['cognitive_context']}\"\n )\n print(\n f\"Retrieved Memories: \"\n f\"{result['retrieved_memories']}\"\n )\n\n print(\"\\nGenerated Response:\")\n print(result[\"response\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T15:55:37.794982Z","iopub.execute_input":"2026-09-27T15:55:37.795993Z","iopub.status.idle":"2026-09-27T15:59:44.745443Z","shell.execute_reply.started":"2026-09-27T15:55:37.79595Z","shell.execute_reply":"2026-09-27T15:59:44.742596Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Lightweight Cognitive-Aware LLM Test\n# ============================================================\n\ntest_query = \"I am struggling with recursion. Can you help me?\"\n\nattention_state = \"Focused\"\nworkload_state = \"Low\"\n\n# Retrieve memories\nc_rag_results = final_c_rag_retrieve(\n query=test_query,\n attention_state=attention_state,\n workload_state=workload_state,\n top_k=3,\n eta=0.7\n)\n\n# Build prompt\nprompt = build_c_rag_prompt(\n query=test_query,\n c_rag_results=c_rag_results,\n attention_state=attention_state,\n workload_state=workload_state\n)\n\nprint(\"Retrieved memories:\")\nprint([x[\"id\"] for x in c_rag_results])\n\nprint(\"\\nGenerating short response...\")\n\nresponse = gemma_model.generate(\n prompt,\n device=\"cpu\",\n output_len=60,\n temperature=0.7,\n top_p=0.9,\n top_k=20\n)\n\nprint(\"\\n============================================================\")\nprint(\"ACME LLM RESPONSE\")\nprint(\"============================================================\")\nprint(response)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:00:40.999643Z","iopub.execute_input":"2026-09-27T16:00:41.001305Z","iopub.status.idle":"2026-09-27T16:01:22.284922Z","shell.execute_reply.started":"2026-09-27T16:00:41.001255Z","shell.execute_reply":"2026-09-27T16:01:22.28395Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Final Experimental Results Summary\n# ============================================================\n\nfinal_results = pd.DataFrame([\n {\n \"Experiment\": \"Attention State Classification\",\n \"Dataset\": \"Mental Attention State Detection\",\n \"Metric\": \"Accuracy\",\n \"Value\": 0.6957\n },\n {\n \"Experiment\": \"Attention State Classification\",\n \"Dataset\": \"Mental Attention State Detection\",\n \"Metric\": \"Macro F1\",\n \"Value\": 0.6300\n },\n {\n \"Experiment\": \"Workload Classification\",\n \"Dataset\": \"STEW\",\n \"Metric\": \"Accuracy\",\n \"Value\": 0.7300\n },\n {\n \"Experiment\": \"Workload Classification\",\n \"Dataset\": \"STEW\",\n \"Metric\": \"Macro F1\",\n \"Value\": 0.7300\n },\n {\n \"Experiment\": \"Semantic FAISS Retrieval\",\n \"Dataset\": \"Challenging Queries\",\n \"Metric\": \"MRR\",\n \"Value\": 0.6542\n },\n {\n \"Experiment\": \"Semantic + Recency\",\n \"Dataset\": \"Challenging Queries\",\n \"Metric\": \"MRR\",\n \"Value\": 0.8304\n },\n {\n \"Experiment\": \"ACME C-RAG\",\n \"Dataset\": \"Challenging Queries\",\n \"Metric\": \"MRR\",\n \"Value\": 0.7542\n },\n {\n \"Experiment\": \"Semantic FAISS Retrieval\",\n \"Dataset\": \"Challenging Queries\",\n \"Metric\": \"Mean Rank\",\n \"Value\": 2.38\n },\n {\n \"Experiment\": \"Semantic + Recency\",\n \"Dataset\": \"Challenging Queries\",\n \"Metric\": \"Mean Rank\",\n \"Value\": 1.88\n },\n {\n \"Experiment\": \"ACME C-RAG\",\n \"Dataset\": \"Challenging Queries\",\n \"Metric\": \"Mean Rank\",\n \"Value\": 1.88\n }\n])\n\nprint(\"============================================================\")\nprint(\"ACME FINAL EXPERIMENTAL RESULTS\")\nprint(\"============================================================\")\n\ndisplay(final_results)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:02:20.226101Z","iopub.execute_input":"2026-09-27T16:02:20.227269Z","iopub.status.idle":"2026-09-27T16:02:20.250653Z","shell.execute_reply.started":"2026-09-27T16:02:20.227222Z","shell.execute_reply":"2026-09-27T16:02:20.248971Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Retrieval Performance Comparison\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\nmethods = [\n \"Semantic FAISS\",\n \"Semantic + Recency\",\n \"ACME C-RAG\"\n]\n\nmrr_values = [\n 0.6542,\n 0.8304,\n 0.7542\n]\n\nmean_rank_values = [\n 2.38,\n 1.88,\n 1.88\n]\n\n# ------------------------------------------------------------\n# Figure 1: MRR\n# ------------------------------------------------------------\n\nplt.figure(figsize=(8, 5))\n\nplt.bar(methods, mrr_values)\n\nplt.ylabel(\"Mean Reciprocal Rank (MRR)\")\nplt.title(\"Retrieval Performance Comparison\")\n\nplt.ylim(0, 1)\n\nfor i, value in enumerate(mrr_values):\n plt.text(\n i,\n value + 0.02,\n f\"{value:.4f}\",\n ha=\"center\"\n )\n\nplt.tight_layout()\nplt.savefig(\n \"ACME_retrieval_MRR.png\",\n dpi=300,\n bbox_inches=\"tight\"\n)\n\nplt.show()\n\n\n# ------------------------------------------------------------\n# Figure 2: Mean Rank\n# ------------------------------------------------------------\n\nplt.figure(figsize=(8, 5))\n\nplt.bar(methods, mean_rank_values)\n\nplt.ylabel(\"Mean Rank\")\nplt.title(\"Mean Retrieval Rank Comparison\")\n\nplt.ylim(0, 3)\n\nfor i, value in enumerate(mean_rank_values):\n plt.text(\n i,\n value + 0.05,\n f\"{value:.2f}\",\n ha=\"center\"\n )\n\nplt.tight_layout()\nplt.savefig(\n \"ACME_retrieval_mean_rank.png\",\n dpi=300,\n bbox_inches=\"tight\"\n)\n\nplt.show()\n\nprint(\"Figures saved:\")\nprint(\"ACME_retrieval_MRR.png\")\nprint(\"ACME_retrieval_mean_rank.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:02:59.026012Z","iopub.execute_input":"2026-09-27T16:02:59.026559Z","iopub.status.idle":"2026-09-27T16:03:00.085727Z","shell.execute_reply.started":"2026-09-27T16:02:59.026523Z","shell.execute_reply":"2026-09-27T16:03:00.084663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: EEG Cognitive-State Classification Performance\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\nmodels = [\n \"Attention\\nClassification\",\n \"Workload\\nClassification\"\n]\n\naccuracy = [\n 0.6957,\n 0.7300\n]\n\nmacro_f1 = [\n 0.6300,\n 0.7300\n]\n\nx = range(len(models))\nwidth = 0.35\n\nplt.figure(figsize=(8, 5))\n\nplt.bar(\n [i - width/2 for i in x],\n accuracy,\n width=width,\n label=\"Accuracy\"\n)\n\nplt.bar(\n [i + width/2 for i in x],\n macro_f1,\n width=width,\n label=\"Macro F1\"\n)\n\nplt.ylabel(\"Score\")\nplt.title(\"EEG Cognitive-State Classification Performance\")\nplt.xticks(list(x), models)\nplt.ylim(0, 1)\n\nfor i, value in enumerate(accuracy):\n plt.text(\n i - width/2,\n value + 0.02,\n f\"{value:.4f}\",\n ha=\"center\"\n )\n\nfor i, value in enumerate(macro_f1):\n plt.text(\n i + width/2,\n value + 0.02,\n f\"{value:.4f}\",\n ha=\"center\"\n )\n\nplt.legend()\nplt.tight_layout()\n\nplt.savefig(\n \"ACME_EEG_classification_performance.png\",\n dpi=300,\n bbox_inches=\"tight\"\n)\n\nplt.show()\n\nprint(\"Figure saved:\")\nprint(\"ACME_EEG_classification_performance.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:04:56.344822Z","iopub.execute_input":"2026-09-27T16:04:56.345841Z","iopub.status.idle":"2026-09-27T16:04:56.88719Z","shell.execute_reply.started":"2026-09-27T16:04:56.345798Z","shell.execute_reply":"2026-09-27T16:04:56.886149Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Cognitive-State Sensitivity Figure\n# ============================================================\n\nimport matplotlib.pyplot as plt\n\nstates = [\n \"Focused + Low\",\n \"Focused + High\",\n \"Unfocused + Low\",\n \"Drowsy + High\"\n]\n\ncognitive_context_values = [\n 0.6625,\n 0.5250,\n 0.5375,\n 0.3500\n]\n\nc_rag_values = [\n 0.7679,\n 0.7542,\n 0.7555,\n 0.7321\n]\n\nx = range(len(states))\nwidth = 0.35\n\nplt.figure(figsize=(9, 5))\n\nplt.bar(\n [i - width/2 for i in x],\n cognitive_context_values,\n width=width,\n label=\"Cognitive Context\"\n)\n\nplt.bar(\n [i + width/2 for i in x],\n c_rag_values,\n width=width,\n label=\"C-RAG Score\"\n)\n\nplt.ylabel(\"Score\")\nplt.title(\"Cognitive-State Sensitivity of ACME Memory Retrieval\")\nplt.xticks(list(x), states)\nplt.ylim(0, 1)\n\nfor i, value in enumerate(cognitive_context_values):\n plt.text(\n i - width/2,\n value + 0.02,\n f\"{value:.4f}\",\n ha=\"center\"\n )\n\nfor i, value in enumerate(c_rag_values):\n plt.text(\n i + width/2,\n value + 0.02,\n f\"{value:.4f}\",\n ha=\"center\"\n )\n\nplt.legend()\nplt.tight_layout()\n\nplt.savefig(\n \"ACME_cognitive_state_sensitivity.png\",\n dpi=300,\n bbox_inches=\"tight\"\n)\n\nplt.show()\n\nprint(\"Figure saved:\")\nprint(\"ACME_cognitive_state_sensitivity.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:05:57.404792Z","iopub.execute_input":"2026-09-27T16:05:57.406281Z","iopub.status.idle":"2026-09-27T16:05:58.054645Z","shell.execute_reply.started":"2026-09-27T16:05:57.406234Z","shell.execute_reply":"2026-09-27T16:05:58.053419Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ACME: Save Final Experimental Results\n# ============================================================\n\n# Save final summary\nfinal_results.to_csv(\n \"ACME_final_results.csv\",\n index=False\n)\n\n# Save challenging retrieval evaluation\nchallenging_df.to_csv(\n \"ACME_challenging_retrieval_results.csv\",\n index=False\n)\n\n# Save cognitive-state sensitivity results\ncognitive_sensitivity_df.to_csv(\n \"ACME_cognitive_sensitivity_results.csv\",\n index=False\n)\n\n# Save evaluation memory bank\npd.DataFrame(\n evaluation_memory_bank\n).to_csv(\n \"ACME_evaluation_memory_bank.csv\",\n index=False\n)\n\n# Save learner memory\npd.DataFrame(\n learner_memories\n).to_csv(\n \"ACME_learner_memory.csv\",\n index=False\n)\n\n# Save FAISS index\nfaiss.write_index(\n evaluation_faiss_index,\n \"ACME_evaluation_memory.index\"\n)\n\nprint(\"============================================================\")\nprint(\"ACME FINAL FILES SAVED\")\nprint(\"============================================================\")\n\nprint(\"✓ ACME_final_results.csv\")\nprint(\"✓ ACME_challenging_retrieval_results.csv\")\nprint(\"✓ ACME_cognitive_sensitivity_results.csv\")\nprint(\"✓ ACME_evaluation_memory_bank.csv\")\nprint(\"✓ ACME_learner_memory.csv\")\nprint(\"✓ ACME_evaluation_memory.index\")\nprint(\"✓ ACME_retrieval_MRR.png\")\nprint(\"✓ ACME_retrieval_mean_rank.png\")\nprint(\"✓ ACME_EEG_classification_performance.png\")\nprint(\"✓ ACME_cognitive_state_sensitivity.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:06:50.686891Z","iopub.execute_input":"2026-09-27T16:06:50.689484Z","iopub.status.idle":"2026-09-27T16:06:50.738598Z","shell.execute_reply.started":"2026-09-27T16:06:50.689434Z","shell.execute_reply":"2026-09-27T16:06:50.737404Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# ATTENTION MODEL - IMPROVED SVM EXPERIMENT\n# ============================================================\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.svm import SVC\nfrom sklearn.metrics import accuracy_score, classification_report, confusion_matrix\n\n# Use the SAME train/test features and labels already created\n# in the existing attention-classification section.\n\nattention_svm = Pipeline([\n (\"scaler\", StandardScaler()),\n (\"svm\", SVC(\n kernel=\"rbf\",\n C=10,\n gamma=\"scale\",\n class_weight=\"balanced\",\n random_state=42\n ))\n])\n\n# Train\nattention_svm.fit(X_train, y_train)\n\n# Predict\ny_pred_svm = attention_svm.predict(X_test)\n\n# Evaluate\nsvm_accuracy = accuracy_score(y_test, y_pred_svm)\n\nprint(\"=\" * 60)\nprint(\"IMPROVED ATTENTION MODEL - SVM\")\nprint(\"=\" * 60)\nprint(f\"Accuracy: {svm_accuracy:.4f} ({svm_accuracy*100:.2f}%)\")\nprint(\"\\nClassification Report:\")\nprint(classification_report(y_test, y_pred_svm))\nprint(\"\\nConfusion Matrix:\")\nprint(confusion_matrix(y_test, y_pred_svm))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:31:34.950847Z","iopub.execute_input":"2026-09-27T16:31:34.954163Z","iopub.status.idle":"2026-09-27T16:31:35.313173Z","shell.execute_reply.started":"2026-09-27T16:31:34.954023Z","shell.execute_reply":"2026-09-27T16:31:35.311927Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 2: RICHER EEG FEATURES FOR ATTENTION CLASSIFICATION\n# ============================================================\n\nimport numpy as np\nfrom scipy.signal import welch\n\ndef extract_rich_attention_features(eeg_segment, sfreq=128):\n \"\"\"\n Extract absolute PSD, relative band power and\n EEG band-ratio features.\n\n Input:\n eeg_segment -> (samples, channels)\n\n Output:\n 1D feature vector\n \"\"\"\n\n bands = {\n \"delta\": (1, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n }\n\n features = []\n\n for ch in range(eeg_segment.shape[1]):\n\n signal = eeg_segment[:, ch]\n\n # Welch PSD\n freqs, psd = welch(\n signal,\n fs=sfreq,\n nperseg=min(256, len(signal))\n )\n\n band_power = {}\n\n # ----------------------------------------------------\n # Absolute band power\n # ----------------------------------------------------\n for band_name, (low, high) in bands.items():\n\n mask = (freqs >= low) & (freqs < high)\n\n power = np.trapezoid(\n psd[mask],\n freqs[mask]\n )\n\n band_power[band_name] = power\n features.append(power)\n\n # ----------------------------------------------------\n # Total power\n # ----------------------------------------------------\n total_power = sum(band_power.values()) + 1e-10\n\n # ----------------------------------------------------\n # Relative band power\n # ----------------------------------------------------\n relative = {}\n\n for band_name in bands.keys():\n\n rel_power = (\n band_power[band_name] /\n total_power\n )\n\n relative[band_name] = rel_power\n features.append(rel_power)\n\n # ----------------------------------------------------\n # Band ratios\n # ----------------------------------------------------\n\n theta_alpha = (\n relative[\"theta\"] /\n (relative[\"alpha\"] + 1e-10)\n )\n\n beta_alpha = (\n relative[\"beta\"] /\n (relative[\"alpha\"] + 1e-10)\n )\n\n alpha_theta = (\n relative[\"alpha\"] /\n (relative[\"theta\"] + 1e-10)\n )\n\n beta_alpha_theta = (\n relative[\"beta\"] /\n (relative[\"alpha\"] + relative[\"theta\"] + 1e-10)\n )\n\n features.extend([\n theta_alpha,\n beta_alpha,\n alpha_theta,\n beta_alpha_theta\n ])\n\n return np.asarray(features, dtype=np.float32)\n\n\nprint(\"Testing richer EEG feature extraction...\")\n\ntest_features = extract_rich_attention_features(\n all_segments[0],\n sfreq=128\n)\n\nprint(\"Feature vector shape:\", test_features.shape)\nprint(\"Number of features:\", len(test_features))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:32:27.123032Z","iopub.execute_input":"2026-09-27T16:32:27.123424Z","iopub.status.idle":"2026-09-27T16:32:27.266023Z","shell.execute_reply.started":"2026-09-27T16:32:27.123393Z","shell.execute_reply":"2026-09-27T16:32:27.264875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 3: BUILD RICH EEG FEATURE MATRIX\n# ============================================================\n\nprint(\"Extracting richer EEG features...\")\nprint(\"Total EEG segments:\", len(all_segments))\n\nX_rich = np.array([\n extract_rich_attention_features(segment, sfreq=128)\n for segment in all_segments\n])\n\nprint(\"\\nRich feature matrix shape:\", X_rich.shape)\nprint(\"Expected features per segment:\", X_rich.shape[1])\n\n# ------------------------------------------------------------\n# Use the SAME train/test indices as the existing model\n# ------------------------------------------------------------\n\nX_rich_train = X_rich[train_idx]\nX_rich_test = X_rich[test_idx]\n\ny_rich_train = labels[train_idx]\ny_rich_test = labels[test_idx]\n\nprint(\"\\nTraining set:\", X_rich_train.shape)\nprint(\"Test set:\", X_rich_test.shape)\n\nprint(\"\\nClass distribution - Training:\")\nprint(pd.Series(y_rich_train).value_counts())\n\nprint(\"\\nClass distribution - Test:\")\nprint(pd.Series(y_rich_test).value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:33:03.001224Z","iopub.execute_input":"2026-09-27T16:33:03.00155Z","iopub.status.idle":"2026-09-27T16:35:54.038993Z","shell.execute_reply.started":"2026-09-27T16:33:03.001522Z","shell.execute_reply":"2026-09-27T16:35:54.035525Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 3 FIX: CREATE TRAIN / TEST SETS\n# ============================================================\n\n# Convert labels to NumPy array\nlabels_array = np.asarray(labels)\n\n# Use the SAME recording-level split\nX_rich_train = X_rich[train_idx]\nX_rich_test = X_rich[test_idx]\n\ny_rich_train = labels_array[train_idx]\ny_rich_test = labels_array[test_idx]\n\nprint(\"Rich feature matrix:\", X_rich.shape)\n\nprint(\"\\nTraining set:\")\nprint(\"X:\", X_rich_train.shape)\nprint(\"y:\", y_rich_train.shape)\n\nprint(\"\\nTest set:\")\nprint(\"X:\", X_rich_test.shape)\nprint(\"y:\", y_rich_test.shape)\n\nprint(\"\\nTraining class distribution:\")\nprint(pd.Series(y_rich_train).value_counts())\n\nprint(\"\\nTest class distribution:\")\nprint(pd.Series(y_rich_test).value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:36:54.808144Z","iopub.execute_input":"2026-09-27T16:36:54.809079Z","iopub.status.idle":"2026-09-27T16:36:54.830186Z","shell.execute_reply.started":"2026-09-27T16:36:54.809022Z","shell.execute_reply":"2026-09-27T16:36:54.828645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Number of recordings:\", len(all_files))\nprint(\"Number of segments:\", len(all_segments))\nprint(\"Number of labels:\", len(labels))\n\nprint(\"train_idx length:\", len(train_idx))\nprint(\"test_idx length:\", len(test_idx))\n\nprint(\"First train_idx:\", train_idx[:10])\nprint(\"First test_idx:\", test_idx[:10])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:37:29.85972Z","iopub.execute_input":"2026-09-27T16:37:29.861282Z","iopub.status.idle":"2026-09-27T16:37:29.87368Z","shell.execute_reply.started":"2026-09-27T16:37:29.861229Z","shell.execute_reply":"2026-09-27T16:37:29.871528Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Show variables related to the EEG dataset/split\n[name for name in globals().keys()\n if any(word in name.lower() for word in\n [\"segment\", \"label\", \"train\", \"test\", \"record\", \"group\", \"split\"])]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:38:04.126263Z","iopub.execute_input":"2026-09-27T16:38:04.126912Z","iopub.status.idle":"2026-09-27T16:38:04.163012Z","shell.execute_reply.started":"2026-09-27T16:38:04.126867Z","shell.execute_reply":"2026-09-27T16:38:04.160352Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 3: MAP RECORDING-LEVEL SPLIT TO EEG SEGMENTS\n# ============================================================\n\n# Convert recording IDs and labels to NumPy arrays\nrecord_ids = np.asarray(all_record_ids)\nsegment_labels = np.asarray(all_labels)\n\n# Recording IDs belonging to training and testing\ntrain_record_ids = record_ids[np.asarray(train_idx)]\ntest_record_ids = record_ids[np.asarray(test_idx)]\n\n# Create masks for all segments\ntrain_mask = np.isin(record_ids, train_record_ids)\ntest_mask = np.isin(record_ids, test_record_ids)\n\n# Rich EEG features\nX_rich_train = X_rich[train_mask]\nX_rich_test = X_rich[test_mask]\n\n# Corresponding segment labels\ny_rich_train = segment_labels[train_mask]\ny_rich_test = segment_labels[test_mask]\n\nprint(\"=\" * 60)\nprint(\"RICH EEG FEATURE DATASET\")\nprint(\"=\" * 60)\n\nprint(\"\\nFull feature matrix:\")\nprint(X_rich.shape)\n\nprint(\"\\nTraining:\")\nprint(\"X:\", X_rich_train.shape)\nprint(\"y:\", y_rich_train.shape)\n\nprint(\"\\nTesting:\")\nprint(\"X:\", X_rich_test.shape)\nprint(\"y:\", y_rich_test.shape)\n\nprint(\"\\nTraining class distribution:\")\nprint(pd.Series(y_rich_train).value_counts())\n\nprint(\"\\nTest class distribution:\")\nprint(pd.Series(y_rich_test).value_counts())\n\nprint(\"\\nRecording overlap check:\")\n\ntrain_record_set = set(train_record_ids)\ntest_record_set = set(test_record_ids)\n\nprint(\"Train recordings:\", len(train_record_set))\nprint(\"Test recordings:\", len(test_record_set))\nprint(\"Overlap:\", len(train_record_set.intersection(test_record_set)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:38:38.525333Z","iopub.execute_input":"2026-09-27T16:38:38.526628Z","iopub.status.idle":"2026-09-27T16:38:38.57068Z","shell.execute_reply.started":"2026-09-27T16:38:38.526579Z","shell.execute_reply":"2026-09-27T16:38:38.569402Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Existing grouped split:\")\nprint(\"X_train_group:\", X_train_group.shape)\nprint(\"X_test_group :\", X_test_group.shape)\n\nprint(\"y_train_group:\", y_train_group.shape)\nprint(\"y_test_group :\", y_test_group.shape)\n\nprint(\"\\nGroups:\")\nprint(\"groups_train:\", len(np.unique(groups_train)))\nprint(\"groups_test :\", len(np.unique(groups_test)))\n\nprint(\n \"Group overlap:\",\n len(\n set(np.unique(groups_train))\n .intersection(set(np.unique(groups_test)))\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:39:16.456687Z","iopub.execute_input":"2026-09-27T16:39:16.457859Z","iopub.status.idle":"2026-09-27T16:39:16.469068Z","shell.execute_reply.started":"2026-09-27T16:39:16.45782Z","shell.execute_reply":"2026-09-27T16:39:16.467375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 4: RICH FEATURES + CORRECT GROUPED SPLIT\n# ============================================================\n\nrecord_ids = np.asarray(all_record_ids)\nsegment_labels = np.asarray(all_labels)\n\n# Get the actual recording IDs used by the original grouped split\ntrain_groups = np.asarray(groups_train)\ntest_groups = np.asarray(groups_test)\n\n# Map every segment to its recording group\nrich_train_mask = np.isin(record_ids, train_groups)\nrich_test_mask = np.isin(record_ids, test_groups)\n\n# Create rich train/test datasets\nX_rich_train = X_rich[rich_train_mask]\nX_rich_test = X_rich[rich_test_mask]\n\ny_rich_train = segment_labels[rich_train_mask]\ny_rich_test = segment_labels[rich_test_mask]\n\nprint(\"=\" * 60)\nprint(\"RICH FEATURES — LEAKAGE-SAFE GROUPED SPLIT\")\nprint(\"=\" * 60)\n\nprint(\"\\nTraining:\")\nprint(\"X:\", X_rich_train.shape)\nprint(\"y:\", y_rich_train.shape)\n\nprint(\"\\nTesting:\")\nprint(\"X:\", X_rich_test.shape)\nprint(\"y:\", y_rich_test.shape)\n\nprint(\"\\nTraining labels:\")\nprint(pd.Series(y_rich_train).value_counts())\n\nprint(\"\\nTesting labels:\")\nprint(pd.Series(y_rich_test).value_counts())\n\n# Final safety check\nprint(\"\\nRecording overlap:\",\n len(set(train_groups).intersection(set(test_groups))))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:39:57.546689Z","iopub.execute_input":"2026-09-27T16:39:57.547081Z","iopub.status.idle":"2026-09-27T16:39:57.576171Z","shell.execute_reply.started":"2026-09-27T16:39:57.547049Z","shell.execute_reply":"2026-09-27T16:39:57.57465Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 5: RANDOM FOREST WITH 350 RICH EEG FEATURES\n# ============================================================\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import (\n accuracy_score,\n classification_report,\n confusion_matrix\n)\n\nrich_rf = RandomForestClassifier(\n n_estimators=300,\n class_weight=\"balanced\",\n random_state=42,\n n_jobs=-1\n)\n\nprint(\"Training Random Forest on 350 EEG features...\")\n\nrich_rf.fit(\n X_rich_train,\n y_rich_train\n)\n\n# Prediction\ny_rich_pred = rich_rf.predict(X_rich_test)\n\n# Metrics\nrich_rf_accuracy = accuracy_score(\n y_rich_test,\n y_rich_pred\n)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"RICH FEATURE RANDOM FOREST RESULTS\")\nprint(\"=\" * 60)\n\nprint(\n f\"\\nAccuracy: {rich_rf_accuracy:.4f} \"\n f\"({rich_rf_accuracy * 100:.2f}%)\"\n)\n\nprint(\"\\nClassification Report:\")\nprint(\n classification_report(\n y_rich_test,\n y_rich_pred\n )\n)\n\nprint(\"\\nConfusion Matrix:\")\nprint(\n confusion_matrix(\n y_rich_test,\n y_rich_pred\n )\n)\n\nprint(\"\\nBaseline accuracy: 69.57%\")\nprint(\n f\"New accuracy: {rich_rf_accuracy * 100:.2f}%\"\n)\nprint(\n f\"Change: {(rich_rf_accuracy - 0.6957) * 100:+.2f} percentage points\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:40:42.581568Z","iopub.execute_input":"2026-09-27T16:40:42.585254Z","iopub.status.idle":"2026-09-27T16:40:45.248192Z","shell.execute_reply.started":"2026-09-27T16:40:42.585087Z","shell.execute_reply":"2026-09-27T16:40:45.247035Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 6: RANDOM FOREST FEATURE SELECTION\n# ============================================================\n\nfrom sklearn.feature_selection import SelectFromModel\n\n# Select important features from the trained rich Random Forest\nselector = SelectFromModel(\n rich_rf,\n threshold=\"median\",\n prefit=True\n)\n\nX_selected_train = selector.transform(X_rich_train)\nX_selected_test = selector.transform(X_rich_test)\n\nprint(\"=\" * 60)\nprint(\"FEATURE SELECTION\")\nprint(\"=\" * 60)\n\nprint(\"Original features:\", X_rich_train.shape[1])\nprint(\"Selected features:\", X_selected_train.shape[1])\n\n# Train a new Random Forest on selected features\nselected_rf = RandomForestClassifier(\n n_estimators=300,\n class_weight=\"balanced\",\n random_state=42,\n n_jobs=-1\n)\n\nprint(\"\\nTraining selected-feature Random Forest...\")\n\nselected_rf.fit(\n X_selected_train,\n y_rich_train\n)\n\n# Prediction\ny_selected_pred = selected_rf.predict(\n X_selected_test\n)\n\n# Evaluation\nselected_accuracy = accuracy_score(\n y_rich_test,\n y_selected_pred\n)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"FEATURE-SELECTED RANDOM FOREST RESULTS\")\nprint(\"=\" * 60)\n\nprint(\n f\"\\nAccuracy: {selected_accuracy:.4f} \"\n f\"({selected_accuracy * 100:.2f}%)\"\n)\n\nprint(\"\\nClassification Report:\")\nprint(\n classification_report(\n y_rich_test,\n y_selected_pred\n )\n)\n\nprint(\"\\nConfusion Matrix:\")\nprint(\n confusion_matrix(\n y_rich_test,\n y_selected_pred\n )\n)\n\nprint(\"\\nComparison:\")\nprint(f\"Original RF: 69.57%\")\nprint(f\"350-feature RF: {rich_rf_accuracy * 100:.2f}%\")\nprint(f\"Selected-feature RF: {selected_accuracy * 100:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:41:55.711891Z","iopub.execute_input":"2026-09-27T16:41:55.713225Z","iopub.status.idle":"2026-09-27T16:41:58.768902Z","shell.execute_reply.started":"2026-09-27T16:41:55.713163Z","shell.execute_reply":"2026-09-27T16:41:58.767638Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 7: BALANCED RANDOM FOREST\n# ============================================================\n\nfrom imblearn.ensemble import BalancedRandomForestClassifier\n\nbalanced_rf = BalancedRandomForestClassifier(\n n_estimators=300,\n random_state=42,\n n_jobs=-1,\n sampling_strategy=\"all\",\n replacement=True\n)\n\nprint(\"Training Balanced Random Forest...\")\n\nbalanced_rf.fit(\n X_selected_train,\n y_rich_train\n)\n\n# Prediction\ny_balanced_pred = balanced_rf.predict(\n X_selected_test\n)\n\n# Evaluation\nbalanced_accuracy = accuracy_score(\n y_rich_test,\n y_balanced_pred\n)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"BALANCED RANDOM FOREST RESULTS\")\nprint(\"=\" * 60)\n\nprint(\n f\"\\nAccuracy: {balanced_accuracy:.4f} \"\n f\"({balanced_accuracy * 100:.2f}%)\"\n)\n\nprint(\"\\nClassification Report:\")\nprint(\n classification_report(\n y_rich_test,\n y_balanced_pred\n )\n)\n\nprint(\"\\nConfusion Matrix:\")\nprint(\n confusion_matrix(\n y_rich_test,\n y_balanced_pred\n )\n)\n\nprint(\"\\nAccuracy comparison:\")\nprint(f\"Original RF: 69.57%\")\nprint(f\"Rich RF: {rich_rf_accuracy * 100:.2f}%\")\nprint(f\"Selected RF: {selected_accuracy * 100:.2f}%\")\nprint(f\"Balanced RF: {balanced_accuracy * 100:.2f}%\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:42:45.043145Z","iopub.execute_input":"2026-09-27T16:42:45.046945Z","iopub.status.idle":"2026-09-27T16:42:49.675713Z","shell.execute_reply.started":"2026-09-27T16:42:45.04683Z","shell.execute_reply":"2026-09-27T16:42:49.673339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 8: BALANCED RANDOM FOREST HYPERPARAMETER EXPERIMENT\n# ============================================================\n\nfrom imblearn.ensemble import BalancedRandomForestClassifier\n\nconfigs = [\n {\n \"name\": \"BRF-300\",\n \"n_estimators\": 300,\n \"max_features\": \"sqrt\",\n \"min_samples_leaf\": 1\n },\n {\n \"name\": \"BRF-500\",\n \"n_estimators\": 500,\n \"max_features\": \"sqrt\",\n \"min_samples_leaf\": 1\n },\n {\n \"name\": \"BRF-500-leaf2\",\n \"n_estimators\": 500,\n \"max_features\": \"sqrt\",\n \"min_samples_leaf\": 2\n },\n {\n \"name\": \"BRF-500-leaf3\",\n \"n_estimators\": 500,\n \"max_features\": 0.5,\n \"min_samples_leaf\": 2\n }\n]\n\ntuning_results = []\n\nfor config in configs:\n\n print(\"\\n\" + \"=\" * 60)\n print(\"Training:\", config[\"name\"])\n print(\"=\" * 60)\n\n model = BalancedRandomForestClassifier(\n n_estimators=config[\"n_estimators\"],\n max_features=config[\"max_features\"],\n min_samples_leaf=config[\"min_samples_leaf\"],\n sampling_strategy=\"all\",\n replacement=True,\n random_state=42,\n n_jobs=-1\n )\n\n model.fit(\n X_selected_train,\n y_rich_train\n )\n\n predictions = model.predict(\n X_selected_test\n )\n\n accuracy = accuracy_score(\n y_rich_test,\n predictions\n )\n\n report = classification_report(\n y_rich_test,\n predictions,\n output_dict=True\n )\n\n macro_f1 = report[\"macro avg\"][\"f1-score\"]\n\n tuning_results.append({\n \"Model\": config[\"name\"],\n \"Accuracy\": accuracy,\n \"Macro_F1\": macro_f1\n })\n\n print(\n f\"Accuracy : {accuracy:.4f} \"\n f\"({accuracy * 100:.2f}%)\"\n )\n\n print(\n f\"Macro-F1 : {macro_f1:.4f}\"\n )\n\n# Results table\ntuning_df = pd.DataFrame(tuning_results)\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"BALANCED RF TUNING RESULTS\")\nprint(\"=\" * 60)\n\ndisplay(\n tuning_df.sort_values(\n \"Macro_F1\",\n ascending=False\n ).reset_index(drop=True)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:43:33.652629Z","iopub.execute_input":"2026-09-27T16:43:33.668361Z","iopub.status.idle":"2026-09-27T16:44:00.343166Z","shell.execute_reply.started":"2026-09-27T16:43:33.668296Z","shell.execute_reply":"2026-09-27T16:44:00.341506Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 9: ANALYZE SELECTED EEG FEATURES\n# ============================================================\n\n# Get the original feature indices selected by SelectFromModel\nselected_feature_indices = selector.get_support(indices=True)\n\n# Feature importance from the selected-feature RF\nfeature_importances = selected_rf.feature_importances_\n\n# Sort by importance\nimportance_order = np.argsort(\n feature_importances\n)[::-1]\n\nprint(\"=\" * 60)\nprint(\"TOP EEG FEATURE IMPORTANCES\")\nprint(\"=\" * 60)\n\nprint(\"\\nTop 30 selected features:\\n\")\n\nfor rank, idx in enumerate(importance_order[:30], start=1):\n\n original_idx = selected_feature_indices[idx]\n\n print(\n f\"{rank:2d}. Original feature \"\n f\"{original_idx:3d} | \"\n f\"Importance: {feature_importances[idx]:.6f}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:44:49.598955Z","iopub.execute_input":"2026-09-27T16:44:49.604772Z","iopub.status.idle":"2026-09-27T16:44:50.131373Z","shell.execute_reply.started":"2026-09-27T16:44:49.604623Z","shell.execute_reply":"2026-09-27T16:44:50.129879Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 10: MAP FEATURE INDICES TO EEG FEATURE TYPES\n# ============================================================\n\nfeature_names = []\n\nbands = [\"delta\", \"theta\", \"alpha\", \"beta\", \"gamma\"]\nratios = [\n \"theta_alpha\",\n \"beta_alpha\",\n \"alpha_theta\",\n \"beta_alpha_theta\"\n]\n\nn_channels = all_segments.shape[2]\n\nfor ch in range(n_channels):\n\n # Absolute power\n for band in bands:\n feature_names.append(\n f\"Ch{ch+1}_{band}_absolute\"\n )\n\n # Relative power\n for band in bands:\n feature_names.append(\n f\"Ch{ch+1}_{band}_relative\"\n )\n\n # Ratios\n for ratio in ratios:\n feature_names.append(\n f\"Ch{ch+1}_{ratio}\"\n )\n\nprint(\"Number of feature names:\", len(feature_names))\n\nprint(\"\\nTop 30 features:\")\nfor rank, idx in enumerate(\n selected_feature_indices[importance_order[:30]],\n start=1\n):\n print(\n f\"{rank:2d}. Feature {idx:3d} -> \"\n f\"{feature_names[idx]}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:45:26.357465Z","iopub.execute_input":"2026-09-27T16:45:26.357834Z","iopub.status.idle":"2026-09-27T16:45:26.373235Z","shell.execute_reply.started":"2026-09-27T16:45:26.357804Z","shell.execute_reply":"2026-09-27T16:45:26.371698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 11: CHANNEL-FOCUSED EEG FEATURE EXPERIMENT\n# ============================================================\n\nselected_channels = [8, 9, 10, 15, 16, 17] # Ch9,10,11,16,17,18\n\n# Each channel has 14 features in our 350-feature representation\nfeatures_per_channel = 14\n\nchannel_feature_indices = []\n\nfor ch in selected_channels:\n start = ch * features_per_channel\n end = start + features_per_channel\n channel_feature_indices.extend(\n range(start, end)\n )\n\nchannel_feature_indices = np.array(\n channel_feature_indices\n)\n\n# Extract selected channel features\nX_channel_train = X_rich_train[\n :, channel_feature_indices\n]\n\nX_channel_test = X_rich_test[\n :, channel_feature_indices\n]\n\nprint(\"=\" * 60)\nprint(\"CHANNEL-FOCUSED EEG FEATURES\")\nprint(\"=\" * 60)\n\nprint(\"Selected channels: Ch9, Ch10, Ch11, Ch16, Ch17, Ch18\")\nprint(\"Original features:\", X_rich_train.shape[1])\nprint(\"Channel-focused features:\", X_channel_train.shape[1])\n\n# Train Balanced Random Forest\nchannel_rf = BalancedRandomForestClassifier(\n n_estimators=300,\n sampling_strategy=\"all\",\n replacement=True,\n random_state=42,\n n_jobs=-1\n)\n\nprint(\"\\nTraining channel-focused Balanced RF...\")\n\nchannel_rf.fit(\n X_channel_train,\n y_rich_train\n)\n\n# Predict\ny_channel_pred = channel_rf.predict(\n X_channel_test\n)\n\n# Evaluate\nchannel_accuracy = accuracy_score(\n y_rich_test,\n y_channel_pred\n)\n\nchannel_report = classification_report(\n y_rich_test,\n y_channel_pred,\n output_dict=True\n)\n\nchannel_macro_f1 = channel_report[\n \"macro avg\"\n][\"f1-score\"]\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"CHANNEL-FOCUSED RESULTS\")\nprint(\"=\" * 60)\n\nprint(\n f\"\\nAccuracy: {channel_accuracy:.4f} \"\n f\"({channel_accuracy * 100:.2f}%)\"\n)\n\nprint(\n f\"Macro-F1: {channel_macro_f1:.4f}\"\n)\n\nprint(\"\\nClassification Report:\")\nprint(\n classification_report(\n y_rich_test,\n y_channel_pred\n )\n)\n\nprint(\"\\nConfusion Matrix:\")\nprint(\n confusion_matrix(\n y_rich_test,\n y_channel_pred\n )\n)\n\nprint(\"\\nComparison:\")\nprint(\"Original RF: 69.57%\")\nprint(\"Best Balanced RF: 75.39%\")\nprint(\n f\"Channel-focused: {channel_accuracy * 100:.2f}%\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:46:04.656326Z","iopub.execute_input":"2026-09-27T16:46:04.656813Z","iopub.status.idle":"2026-09-27T16:46:06.85679Z","shell.execute_reply.started":"2026-09-27T16:46:04.656778Z","shell.execute_reply":"2026-09-27T16:46:06.855512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD MODEL — CURRENT FEATURE CHECK\n# ============================================================\n\nprint(\"STEW segments:\", stew_segments.shape)\nprint(\"STEW labels:\", stew_labels.shape)\n\nprint(\"\\nCurrent workload feature matrix:\")\nprint(\"X_train_stew:\", X_train_stew.shape)\nprint(\"X_test_stew :\", X_test_stew.shape)\n\nprint(\"\\nTraining labels:\")\nprint(pd.Series(y_train_stew).value_counts())\n\nprint(\"\\nTest labels:\")\nprint(pd.Series(y_test_stew).value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:48:31.251854Z","iopub.execute_input":"2026-09-27T16:48:31.258229Z","iopub.status.idle":"2026-09-27T16:48:31.327774Z","shell.execute_reply.started":"2026-09-27T16:48:31.258034Z","shell.execute_reply":"2026-09-27T16:48:31.326053Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD STEP 2: RICHER EEG FEATURES\n# ============================================================\n\ndef extract_rich_workload_features(eeg_segment, sfreq=128):\n \"\"\"\n Extract richer spectral features for workload classification.\n\n Features per channel:\n - 5 absolute band powers\n - 5 relative band powers\n - 4 band ratios\n\n Total:\n 14 features per channel\n 14 channels -> 196 features\n \"\"\"\n\n bands = {\n \"delta\": (1, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n }\n\n features = []\n\n for ch in range(eeg_segment.shape[1]):\n\n signal = eeg_segment[:, ch]\n\n freqs, psd = welch(\n signal,\n fs=sfreq,\n nperseg=min(256, len(signal))\n )\n\n band_power = {}\n\n # ----------------------------------------------------\n # Absolute band power\n # ----------------------------------------------------\n for band_name, (low, high) in bands.items():\n\n mask = (freqs >= low) & (freqs < high)\n\n power = np.trapezoid(\n psd[mask],\n freqs[mask]\n )\n\n band_power[band_name] = power\n features.append(power)\n\n # ----------------------------------------------------\n # Total power\n # ----------------------------------------------------\n total_power = sum(\n band_power.values()\n ) + 1e-10\n\n # ----------------------------------------------------\n # Relative band power\n # ----------------------------------------------------\n relative = {}\n\n for band_name in bands.keys():\n\n rel_power = (\n band_power[band_name] /\n total_power\n )\n\n relative[band_name] = rel_power\n features.append(rel_power)\n\n # ----------------------------------------------------\n # Band ratios\n # ----------------------------------------------------\n theta_alpha = (\n relative[\"theta\"] /\n (relative[\"alpha\"] + 1e-10)\n )\n\n beta_alpha = (\n relative[\"beta\"] /\n (relative[\"alpha\"] + 1e-10)\n )\n\n alpha_theta = (\n relative[\"alpha\"] /\n (relative[\"theta\"] + 1e-10)\n )\n\n beta_alpha_theta = (\n relative[\"beta\"] /\n (\n relative[\"alpha\"] +\n relative[\"theta\"] +\n 1e-10\n )\n )\n\n features.extend([\n theta_alpha,\n beta_alpha,\n alpha_theta,\n beta_alpha_theta\n ])\n\n return np.asarray(\n features,\n dtype=np.float32\n )\n\n\n# Test feature extraction on one workload segment\nworkload_test_features = extract_rich_workload_features(\n stew_segments[0],\n sfreq=128\n)\n\nprint(\"=\" * 60)\nprint(\"RICH WORKLOAD FEATURE EXTRACTION\")\nprint(\"=\" * 60)\n\nprint(\n \"Feature vector shape:\",\n workload_test_features.shape\n)\n\nprint(\n \"Number of features:\",\n len(workload_test_features)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:49:05.526048Z","iopub.execute_input":"2026-09-27T16:49:05.527057Z","iopub.status.idle":"2026-09-27T16:49:05.586298Z","shell.execute_reply.started":"2026-09-27T16:49:05.527015Z","shell.execute_reply":"2026-09-27T16:49:05.584984Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD STEP 3: BUILD RICH FEATURE MATRICES\n# ============================================================\n\nprint(\"Extracting rich workload features...\")\nprint(\"This may take a little time.\\n\")\n\n# Extract features for all STEW segments\nX_workload_rich = np.array([\n extract_rich_workload_features(segment, sfreq=128)\n for segment in stew_segments\n])\n\nprint(\"Full rich feature matrix:\")\nprint(X_workload_rich.shape)\n\n\n# ------------------------------------------------------------\n# Map rich features using the EXISTING subject-level split\n# ------------------------------------------------------------\n\n# Existing X_train_stew / X_test_stew contain 70 features.\n# We identify their rows by matching the original feature vectors.\n\ntrain_rich_indices = []\ntest_rich_indices = []\n\nfor i, segment in enumerate(stew_segments):\n\n original_features = extract_stew_features(\n segment,\n sfreq=128\n )\n\n # Match against the existing feature matrices\n if np.any(\n np.all(\n np.isclose(\n X_train_stew,\n original_features,\n rtol=1e-5,\n atol=1e-7\n ),\n axis=1\n )\n ):\n train_rich_indices.append(i)\n\n elif np.any(\n np.all(\n np.isclose(\n X_test_stew,\n original_features,\n rtol=1e-5,\n atol=1e-7\n ),\n axis=1\n )\n ):\n test_rich_indices.append(i)\n\n\nX_workload_rich_train = X_workload_rich[train_rich_indices]\nX_workload_rich_test = X_workload_rich[test_rich_indices]\n\ny_workload_rich_train = np.asarray(\n stew_labels[train_rich_indices]\n)\n\ny_workload_rich_test = np.asarray(\n stew_labels[test_rich_indices]\n)\n\n\nprint(\"\\nRich workload train/test:\")\nprint(\n \"X_train:\",\n X_workload_rich_train.shape\n)\n\nprint(\n \"X_test :\",\n X_workload_rich_test.shape\n)\n\nprint(\n \"y_train:\",\n y_workload_rich_train.shape\n)\n\nprint(\n \"y_test :\",\n y_workload_rich_test.shape\n)\n\nprint(\"\\nTrain labels:\")\nprint(pd.Series(y_workload_rich_train).value_counts())\n\nprint(\"\\nTest labels:\")\nprint(pd.Series(y_workload_rich_test).value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:49:37.240243Z","iopub.execute_input":"2026-09-27T16:49:37.240738Z","iopub.status.idle":"2026-09-27T16:49:53.949312Z","shell.execute_reply.started":"2026-09-27T16:49:37.240702Z","shell.execute_reply":"2026-09-27T16:49:53.947162Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD STEP 3: BUILD RICH FEATURE MATRICES\n# ============================================================\n\nprint(\"Rich feature matrix already created:\")\nprint(\"X_workload_rich:\", X_workload_rich.shape)\n\n# ------------------------------------------------------------\n# Reconstruct the existing STEW subject-level split\n# using the same split that produced X_train_stew / X_test_stew\n# ------------------------------------------------------------\n\n# The existing split uses these indices:\n# train_idx_stew / test_idx_stew\n#\n# If they already exist, use them directly.\n# Otherwise, identify them from the existing 70-feature matrices.\n\nif 'train_idx_stew' in globals() and 'test_idx_stew' in globals():\n\n workload_train_indices = train_idx_stew\n workload_test_indices = test_idx_stew\n\nelse:\n # Find rows by comparing against the original 70-feature matrices\n workload_train_indices = []\n workload_test_indices = []\n\n # Calculate original features directly from the segment\n # without modifying extract_stew_features()\n bands = [\n (1, 4), # delta\n (4, 8), # theta\n (8, 13), # alpha\n (13, 30), # beta\n (30, 45) # gamma\n ]\n\n original_features_all = []\n\n for segment in stew_segments:\n\n segment_features = []\n\n for ch in range(segment.shape[1]):\n\n signal = segment[:, ch]\n\n freqs, psd = welch(\n signal,\n fs=128,\n nperseg=min(256, len(signal))\n )\n\n for low, high in bands:\n\n mask = (freqs >= low) & (freqs < high)\n\n power = np.trapezoid(\n psd[mask],\n freqs[mask]\n )\n\n segment_features.append(power)\n\n original_features_all.append(segment_features)\n\n original_features_all = np.asarray(\n original_features_all\n )\n\n # Match each segment to train/test feature matrix\n for i, features in enumerate(original_features_all):\n\n if np.any(\n np.all(\n np.isclose(\n X_train_stew,\n features,\n rtol=1e-5,\n atol=1e-7\n ),\n axis=1\n )\n ):\n workload_train_indices.append(i)\n\n elif np.any(\n np.all(\n np.isclose(\n X_test_stew,\n features,\n rtol=1e-5,\n atol=1e-7\n ),\n axis=1\n )\n ):\n workload_test_indices.append(i)\n\n\n# ------------------------------------------------------------\n# Create rich train/test matrices\n# ------------------------------------------------------------\n\nX_workload_rich_train = X_workload_rich[\n workload_train_indices\n]\n\nX_workload_rich_test = X_workload_rich[\n workload_test_indices\n]\n\ny_workload_rich_train = np.asarray(\n stew_labels[workload_train_indices]\n)\n\ny_workload_rich_test = np.asarray(\n stew_labels[workload_test_indices]\n)\n\n\n# ------------------------------------------------------------\n# Verify\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"RICH WORKLOAD DATASET\")\nprint(\"=\" * 60)\n\nprint(\n \"X_train:\",\n X_workload_rich_train.shape\n)\n\nprint(\n \"X_test :\",\n X_workload_rich_test.shape\n)\n\nprint(\n \"y_train:\",\n y_workload_rich_train.shape\n)\n\nprint(\n \"y_test :\",\n y_workload_rich_test.shape\n)\n\nprint(\"\\nTrain labels:\")\nprint(\n pd.Series(\n y_workload_rich_train\n ).value_counts()\n)\n\nprint(\"\\nTest labels:\")\nprint(\n pd.Series(\n y_workload_rich_test\n ).value_counts()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:50:35.493835Z","iopub.execute_input":"2026-09-27T16:50:35.495379Z","iopub.status.idle":"2026-09-27T16:50:52.735788Z","shell.execute_reply.started":"2026-09-27T16:50:35.495335Z","shell.execute_reply":"2026-09-27T16:50:52.730634Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FIX WORKLOAD TRAIN / TEST MAPPING\n# ============================================================\n\n# Restore the band dictionary expected by the original\n# extract_stew_features() function.\n\nbands = {\n \"delta\": (1, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n}\n\nprint(\"Matching rich features to existing STEW split...\")\n\nworkload_train_indices = []\nworkload_test_indices = []\n\n# Recalculate the ORIGINAL 70-feature representation\n# using the notebook's original function.\n\noriginal_stew_features = []\n\nfor i, segment in enumerate(stew_segments):\n\n features = extract_stew_features(\n segment,\n sfreq=128\n )\n\n original_stew_features.append(features)\n\noriginal_stew_features = np.asarray(\n original_stew_features\n)\n\nprint(\n \"Original feature matrix:\",\n original_stew_features.shape\n)\n\n\n# ------------------------------------------------------------\n# Match each original feature vector to train/test\n# ------------------------------------------------------------\n\nfor i, features in enumerate(original_stew_features):\n\n train_match = np.any(\n np.all(\n np.isclose(\n X_train_stew,\n features,\n rtol=1e-5,\n atol=1e-7\n ),\n axis=1\n )\n )\n\n test_match = np.any(\n np.all(\n np.isclose(\n X_test_stew,\n features,\n rtol=1e-5,\n atol=1e-7\n ),\n axis=1\n )\n )\n\n if train_match:\n workload_train_indices.append(i)\n\n elif test_match:\n workload_test_indices.append(i)\n\n\n# ------------------------------------------------------------\n# Build rich train/test matrices\n# ------------------------------------------------------------\n\nX_workload_rich_train = X_workload_rich[\n workload_train_indices\n]\n\nX_workload_rich_test = X_workload_rich[\n workload_test_indices\n]\n\ny_workload_rich_train = np.asarray(\n stew_labels[workload_train_indices]\n)\n\ny_workload_rich_test = np.asarray(\n stew_labels[workload_test_indices]\n)\n\n\n# ------------------------------------------------------------\n# Verification\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"RICH WORKLOAD DATASET\")\nprint(\"=\" * 60)\n\nprint(\"X_train:\", X_workload_rich_train.shape)\nprint(\"X_test :\", X_workload_rich_test.shape)\n\nprint(\"y_train:\", y_workload_rich_train.shape)\nprint(\"y_test :\", y_workload_rich_test.shape)\n\nprint(\"\\nTrain labels:\")\nprint(pd.Series(y_workload_rich_train).value_counts())\n\nprint(\"\\nTest labels:\")\nprint(pd.Series(y_workload_rich_test).value_counts())\n\nprint(\"\\nTotal mapped:\",\n len(workload_train_indices) +\n len(workload_test_indices))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:51:50.607392Z","iopub.execute_input":"2026-09-27T16:51:50.60862Z","iopub.status.idle":"2026-09-27T16:52:05.972564Z","shell.execute_reply.started":"2026-09-27T16:51:50.608555Z","shell.execute_reply":"2026-09-27T16:52:05.970936Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEW SPLIT DIAGNOSTIC\n# ============================================================\n\nprint(\"Available STEW-related variables:\\n\")\n\nfor name in sorted(globals().keys()):\n\n if any(key in name.lower()\n for key in [\n \"stew\",\n \"subject\",\n \"group\",\n \"split\",\n \"train_idx\",\n \"test_idx\"\n ]):\n\n obj = globals()[name]\n\n try:\n print(\n f\"{name:30s}\",\n \"shape =\", getattr(obj, \"shape\", None),\n \"len =\", len(obj)\n )\n except:\n print(\n f\"{name:30s}\",\n type(obj)\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:52:49.242963Z","iopub.execute_input":"2026-09-27T16:52:49.247626Z","iopub.status.idle":"2026-09-27T16:52:49.292016Z","shell.execute_reply.started":"2026-09-27T16:52:49.247493Z","shell.execute_reply":"2026-09-27T16:52:49.290285Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD STEP 3B: CORRECT RICH TRAIN / TEST SPLIT\n# ============================================================\n\n# train_idx and test_idx are the ORIGINAL subject-level\n# STEW split indices for all 480 segments.\n\nX_workload_rich_train = X_workload_rich[train_idx]\nX_workload_rich_test = X_workload_rich[test_idx]\n\ny_workload_rich_train = stew_labels[train_idx]\ny_workload_rich_test = stew_labels[test_idx]\n\n\n# ------------------------------------------------------------\n# Verification\n# ------------------------------------------------------------\n\nprint(\"=\" * 60)\nprint(\"FINAL RICH WORKLOAD DATASET\")\nprint(\"=\" * 60)\n\nprint(\"X_train:\", X_workload_rich_train.shape)\nprint(\"X_test :\", X_workload_rich_test.shape)\n\nprint(\"y_train:\", y_workload_rich_train.shape)\nprint(\"y_test :\", y_workload_rich_test.shape)\n\nprint(\"\\nTrain labels:\")\nprint(pd.Series(y_workload_rich_train).value_counts())\n\nprint(\"\\nTest labels:\")\nprint(pd.Series(y_workload_rich_test).value_counts())\n\n\n# ------------------------------------------------------------\n# Check that the split is exactly the existing STEW split\n# ------------------------------------------------------------\n\nprint(\"\\nIndex verification:\")\nprint(\"Train indices:\", len(train_idx))\nprint(\"Test indices :\", len(test_idx))\nprint(\"Overlap :\", len(set(train_idx) & set(test_idx)))\nprint(\"Total :\", len(set(train_idx) | set(test_idx)))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:53:24.053392Z","iopub.execute_input":"2026-09-27T16:53:24.055201Z","iopub.status.idle":"2026-09-27T16:53:24.101457Z","shell.execute_reply.started":"2026-09-27T16:53:24.055108Z","shell.execute_reply":"2026-09-27T16:53:24.09987Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD STEP 4: RICH RANDOM FOREST\n# ============================================================\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import (\n accuracy_score,\n classification_report,\n confusion_matrix,\n f1_score\n)\n\n# ------------------------------------------------------------\n# Train\n# ------------------------------------------------------------\n\nworkload_rich_rf = RandomForestClassifier(\n n_estimators=300,\n random_state=42,\n class_weight=\"balanced\",\n n_jobs=-1\n)\n\nworkload_rich_rf.fit(\n X_workload_rich_train,\n y_workload_rich_train\n)\n\n\n# ------------------------------------------------------------\n# Predict\n# ------------------------------------------------------------\n\nworkload_rich_pred = workload_rich_rf.predict(\n X_workload_rich_test\n)\n\n\n# ------------------------------------------------------------\n# Evaluate\n# ------------------------------------------------------------\n\nworkload_rich_accuracy = accuracy_score(\n y_workload_rich_test,\n workload_rich_pred\n)\n\nworkload_rich_f1 = f1_score(\n y_workload_rich_test,\n workload_rich_pred,\n average=\"macro\"\n)\n\nprint(\"=\" * 60)\nprint(\"RICH WORKLOAD RANDOM FOREST\")\nprint(\"=\" * 60)\n\nprint(\n f\"Accuracy : {workload_rich_accuracy:.4f} \"\n f\"({workload_rich_accuracy * 100:.2f}%)\"\n)\n\nprint(\n f\"Macro F1 : {workload_rich_f1:.4f}\"\n)\n\nprint(\"\\nClassification Report:\")\nprint(\n classification_report(\n y_workload_rich_test,\n workload_rich_pred\n )\n)\n\nprint(\"Confusion Matrix:\")\nprint(\n confusion_matrix(\n y_workload_rich_test,\n workload_rich_pred\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:53:58.855672Z","iopub.execute_input":"2026-09-27T16:53:58.856925Z","iopub.status.idle":"2026-09-27T16:54:00.028409Z","shell.execute_reply.started":"2026-09-27T16:53:58.856883Z","shell.execute_reply":"2026-09-27T16:54:00.026889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD STEP 5: FEATURE SELECTION\n# ============================================================\n\nfrom sklearn.feature_selection import SelectFromModel\n\n# ------------------------------------------------------------\n# Select important features using the trained rich RF\n# ------------------------------------------------------------\n\nworkload_selector = SelectFromModel(\n workload_rich_rf,\n threshold=\"median\",\n prefit=True\n)\n\nX_workload_selected_train = (\n workload_selector.transform(\n X_workload_rich_train\n )\n)\n\nX_workload_selected_test = (\n workload_selector.transform(\n X_workload_rich_test\n )\n)\n\nprint(\"=\" * 60)\nprint(\"WORKLOAD FEATURE SELECTION\")\nprint(\"=\" * 60)\n\nprint(\n \"Original features:\",\n X_workload_rich_train.shape[1]\n)\n\nprint(\n \"Selected features:\",\n X_workload_selected_train.shape[1]\n)\n\nprint(\n \"Reduced train shape:\",\n X_workload_selected_train.shape\n)\n\nprint(\n \"Reduced test shape:\",\n X_workload_selected_test.shape\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:54:48.350952Z","iopub.execute_input":"2026-09-27T16:54:48.356363Z","iopub.status.idle":"2026-09-27T16:54:48.916423Z","shell.execute_reply.started":"2026-09-27T16:54:48.356221Z","shell.execute_reply":"2026-09-27T16:54:48.914842Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# WORKLOAD STEP 6: SELECTED-FEATURE RANDOM FOREST\n# ============================================================\n\nworkload_selected_rf = RandomForestClassifier(\n n_estimators=300,\n random_state=42,\n class_weight=\"balanced\",\n n_jobs=-1\n)\n\nworkload_selected_rf.fit(\n X_workload_selected_train,\n y_workload_rich_train\n)\n\n\n# ------------------------------------------------------------\n# Predict\n# ------------------------------------------------------------\n\nworkload_selected_pred = workload_selected_rf.predict(\n X_workload_selected_test\n)\n\n\n# ------------------------------------------------------------\n# Evaluate\n# ------------------------------------------------------------\n\nworkload_selected_accuracy = accuracy_score(\n y_workload_rich_test,\n workload_selected_pred\n)\n\nworkload_selected_f1 = f1_score(\n y_workload_rich_test,\n workload_selected_pred,\n average=\"macro\"\n)\n\n\nprint(\"=\" * 60)\nprint(\"SELECTED-FEATURE WORKLOAD RANDOM FOREST\")\nprint(\"=\" * 60)\n\nprint(\n f\"Accuracy : {workload_selected_accuracy:.4f} \"\n f\"({workload_selected_accuracy * 100:.2f}%)\"\n)\n\nprint(\n f\"Macro F1 : {workload_selected_f1:.4f}\"\n)\n\nprint(\"\\nClassification Report:\")\nprint(\n classification_report(\n y_workload_rich_test,\n workload_selected_pred\n )\n)\n\nprint(\"Confusion Matrix:\")\nprint(\n confusion_matrix(\n y_workload_rich_test,\n workload_selected_pred\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:55:20.255157Z","iopub.execute_input":"2026-09-27T16:55:20.256606Z","iopub.status.idle":"2026-09-27T16:55:21.291257Z","shell.execute_reply.started":"2026-09-27T16:55:20.256555Z","shell.execute_reply":"2026-09-27T16:55:21.290235Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# LOCK BEST WORKLOAD MODEL\n# ============================================================\n\nstew_model = workload_rich_rf\n\nworkload_model = workload_rich_rf\n\nprint(\"Best workload model locked.\")\nprint(\"Model: Rich Random Forest\")\nprint(\"Features: 196\")\nprint(f\"Accuracy: {workload_rich_accuracy * 100:.2f}%\")\nprint(f\"Macro-F1: {workload_rich_f1:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:56:01.990336Z","iopub.execute_input":"2026-09-27T16:56:01.991627Z","iopub.status.idle":"2026-09-27T16:56:02.022862Z","shell.execute_reply.started":"2026-09-27T16:56:01.991575Z","shell.execute_reply":"2026-09-27T16:56:02.021354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 1: FIND AVAILABLE EEG DATASETS\n# ============================================================\n\nimport os\n\nprint(\"=\" * 60)\nprint(\"AVAILABLE KAGGLE EEG DATASETS\")\nprint(\"=\" * 60)\n\nfor root, dirs, files in os.walk(\"/kaggle/input\"):\n\n # Show only directories that look relevant to EEG/fatigue\n path_lower = root.lower()\n\n if any(\n keyword in path_lower\n for keyword in [\n \"eeg\",\n \"fatigue\",\n \"sleep\",\n \"drows\",\n \"vigilance\",\n \"seeg\",\n \"deap\",\n \"seed\"\n ]\n ):\n print(root)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:56:39.177828Z","iopub.execute_input":"2026-09-27T16:56:39.178285Z","iopub.status.idle":"2026-09-27T16:56:39.383444Z","shell.execute_reply.started":"2026-09-27T16:56:39.178247Z","shell.execute_reply":"2026-09-27T16:56:39.382172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 2: INSPECT EEG DATASET\n# ============================================================\n\nimport os\n\nfatigue_dataset_path = \"/kaggle/input/datasets/samnikolas/eeg-dataset\"\n\nprint(\"=\" * 60)\nprint(\"EEG DATASET CONTENTS\")\nprint(\"=\" * 60)\n\nfor root, dirs, files in os.walk(fatigue_dataset_path):\n\n level = root.replace(fatigue_dataset_path, \"\").count(os.sep)\n\n if level <= 2:\n indent = \" \" * level\n print(f\"\\n{indent}{os.path.basename(root)}/\")\n\n for file in files[:20]:\n print(f\"{indent} {file}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:57:20.658873Z","iopub.execute_input":"2026-09-27T16:57:20.663937Z","iopub.status.idle":"2026-09-27T16:57:20.703423Z","shell.execute_reply.started":"2026-09-27T16:57:20.663793Z","shell.execute_reply":"2026-09-27T16:57:20.70105Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 3: INSPECT features_raw.csv\n# ============================================================\n\nfatigue_csv_path = (\n \"/kaggle/input/datasets/samnikolas/eeg-dataset/features_raw.csv\"\n)\n\nfatigue_df = pd.read_csv(fatigue_csv_path)\n\nprint(\"=\" * 60)\nprint(\"FATIGUE DATASET INSPECTION\")\nprint(\"=\" * 60)\n\nprint(\"\\nShape:\")\nprint(fatigue_df.shape)\n\nprint(\"\\nColumns:\")\nprint(fatigue_df.columns.tolist())\n\nprint(\"\\nFirst 5 rows:\")\ndisplay(fatigue_df.head())\n\nprint(\"\\nData types:\")\nprint(fatigue_df.dtypes)\n\nprint(\"\\nMissing values:\")\nprint(fatigue_df.isnull().sum().sort_values(ascending=False).head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:58:11.965488Z","iopub.execute_input":"2026-09-27T16:58:11.965926Z","iopub.status.idle":"2026-09-27T16:58:12.270806Z","shell.execute_reply.started":"2026-09-27T16:58:11.965892Z","shell.execute_reply":"2026-09-27T16:58:12.269512Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 4: INSPECT DROWSINESS/FATIGUE PROXY\n# ============================================================\n\nprint(\"=\" * 60)\nprint(\"DROWSINESS / FATIGUE-PROXY DATASET\")\nprint(\"=\" * 60)\n\nprint(\"\\nTotal EEG segments:\")\nprint(all_segments.shape)\n\nprint(\"\\nLabels:\")\nprint(pd.Series(all_labels).value_counts())\n\nprint(\"\\nUnique labels:\")\nprint(np.unique(all_labels))\n\nprint(\"\\nNumber of recordings:\")\nprint(len(np.unique(all_record_ids)))\n\nprint(\"\\nSegments per recording:\")\nprint(\n pd.Series(all_record_ids).value_counts().describe()\n)\n\nprint(\"\\nExisting subject/recording-disjoint split:\")\nprint(\"Train:\", len(np.unique(groups_train)), \"groups\")\nprint(\"Test :\", len(np.unique(groups_test)), \"groups\")\nprint(\n \"Overlap:\",\n len(\n set(groups_train) &\n set(groups_test)\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:58:50.293783Z","iopub.execute_input":"2026-09-27T16:58:50.29883Z","iopub.status.idle":"2026-09-27T16:58:50.372932Z","shell.execute_reply.started":"2026-09-27T16:58:50.29869Z","shell.execute_reply":"2026-09-27T16:58:50.371333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 5: VERIFY RECORDING-LEVEL SPLIT\n# ============================================================\n\nprint(\"=\" * 60)\nprint(\"CORRECT RECORDING-LEVEL FATIGUE SPLIT\")\nprint(\"=\" * 60)\n\n# Actual recording IDs\nrecording_ids = np.asarray(all_record_ids)\n\nunique_recordings = np.unique(recording_ids)\n\nprint(\"Total recordings:\", len(unique_recordings))\nprint(\"Recording IDs:\", unique_recordings)\n\n# Count segments per recording\nrecording_counts = pd.Series(\n recording_ids\n).value_counts().sort_index()\n\nprint(\"\\nSegments per recording:\")\nprint(recording_counts)\n\n# ------------------------------------------------------------\n# Inspect the current split variables\n# ------------------------------------------------------------\n\nprint(\"\\nCurrent groups:\")\nprint(\"groups_train unique:\", len(np.unique(groups_train)))\nprint(\"groups_test unique :\", len(np.unique(groups_test)))\n\n# ------------------------------------------------------------\n# Check whether all_record_ids are actually recording IDs\n# ------------------------------------------------------------\n\nprint(\"\\nFirst 30 recording IDs:\")\nprint(recording_ids[:30])\n\nprint(\"\\nFirst 30 labels:\")\nprint(np.asarray(all_labels)[:30])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T16:59:56.291283Z","iopub.execute_input":"2026-09-27T16:59:56.291806Z","iopub.status.idle":"2026-09-27T16:59:56.318596Z","shell.execute_reply.started":"2026-09-27T16:59:56.291768Z","shell.execute_reply":"2026-09-27T16:59:56.317168Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 6: RECORDING-DISJOINT SPLIT\n# ============================================================\n\nfrom sklearn.model_selection import GroupShuffleSplit\n\n# ------------------------------------------------------------\n# Fresh recording-level split\n# ------------------------------------------------------------\n\nfatigue_splitter = GroupShuffleSplit(\n n_splits=1,\n test_size=0.20,\n random_state=42\n)\n\nfatigue_train_idx, fatigue_test_idx = next(\n fatigue_splitter.split(\n all_segments,\n all_labels,\n groups=all_record_ids\n )\n)\n\n# ------------------------------------------------------------\n# Verify recording separation\n# ------------------------------------------------------------\n\nfatigue_train_recordings = np.unique(\n all_record_ids[fatigue_train_idx]\n)\n\nfatigue_test_recordings = np.unique(\n all_record_ids[fatigue_test_idx]\n)\n\noverlap = set(\n fatigue_train_recordings\n) & set(\n fatigue_test_recordings\n)\n\nprint(\"=\" * 60)\nprint(\"FATIGUE / DROWSINESS RECORDING-LEVEL SPLIT\")\nprint(\"=\" * 60)\n\nprint(\n \"Training recordings:\",\n len(fatigue_train_recordings)\n)\n\nprint(\n \"Testing recordings :\",\n len(fatigue_test_recordings)\n)\n\nprint(\n \"Recording overlap :\",\n len(overlap)\n)\n\nprint(\n \"\\nTraining segments:\",\n len(fatigue_train_idx)\n)\n\nprint(\n \"Testing segments :\",\n len(fatigue_test_idx)\n)\n\nprint(\"\\nTraining recordings:\")\nprint(fatigue_train_recordings)\n\nprint(\"\\nTesting recordings:\")\nprint(fatigue_test_recordings)\n\nprint(\"\\nTraining labels:\")\nprint(\n pd.Series(\n np.asarray(all_labels)[fatigue_train_idx]\n ).value_counts()\n)\n\nprint(\"\\nTesting labels:\")\nprint(\n pd.Series(\n np.asarray(all_labels)[fatigue_test_idx]\n ).value_counts()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:00:34.818629Z","iopub.execute_input":"2026-09-27T17:00:34.819846Z","iopub.status.idle":"2026-09-27T17:00:34.850611Z","shell.execute_reply.started":"2026-09-27T17:00:34.819805Z","shell.execute_reply":"2026-09-27T17:00:34.849441Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 7: BINARY DROWSINESS / FATIGUE PROXY\n# ============================================================\n\n# ------------------------------------------------------------\n# Extract train/test segments\n# ------------------------------------------------------------\n\nX_fatigue_train_raw = all_segments[\n fatigue_train_idx\n]\n\nX_fatigue_test_raw = all_segments[\n fatigue_test_idx\n]\n\nlabels_fatigue_train = np.asarray(\n all_labels\n)[fatigue_train_idx]\n\nlabels_fatigue_test = np.asarray(\n all_labels\n)[fatigue_test_idx]\n\n\n# ------------------------------------------------------------\n# Convert labels\n#\n# 1 = Drowsy / fatigue-vigilance proxy\n# 0 = Non-drowsy\n# ------------------------------------------------------------\n\ny_fatigue_train = (\n labels_fatigue_train == \"Drowsy\"\n).astype(int)\n\ny_fatigue_test = (\n labels_fatigue_test == \"Drowsy\"\n).astype(int)\n\n\n# ------------------------------------------------------------\n# Verify\n# ------------------------------------------------------------\n\nprint(\"=\" * 60)\nprint(\"BINARY FATIGUE / DROWSINESS PROXY\")\nprint(\"=\" * 60)\n\nprint(\"Training segments:\", len(y_fatigue_train))\nprint(\"Testing segments :\", len(y_fatigue_test))\n\nprint(\"\\nTraining:\")\nprint(\n pd.Series(\n y_fatigue_train\n ).value_counts().sort_index()\n)\n\nprint(\"\\nTesting:\")\nprint(\n pd.Series(\n y_fatigue_test\n ).value_counts().sort_index()\n)\n\nprint(\"\\nLabel mapping:\")\nprint(\"0 = Non-drowsy (Focused + Unfocused)\")\nprint(\"1 = Drowsy (fatigue/vigilance proxy)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:01:14.289453Z","iopub.execute_input":"2026-09-27T17:01:14.294072Z","iopub.status.idle":"2026-09-27T17:01:20.238126Z","shell.execute_reply.started":"2026-09-27T17:01:14.293938Z","shell.execute_reply":"2026-09-27T17:01:20.236502Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 8: RICH EEG FEATURES\n# ============================================================\n\ndef extract_fatigue_rich_features(eeg_segment, sfreq=128):\n\n bands = {\n \"delta\": (1, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n }\n\n features = []\n\n for ch in range(eeg_segment.shape[1]):\n\n signal = eeg_segment[:, ch]\n\n freqs, psd = welch(\n signal,\n fs=sfreq,\n nperseg=min(256, len(signal))\n )\n\n band_power = {}\n\n # ----------------------------------------------------\n # Absolute power\n # ----------------------------------------------------\n\n for band_name, (low, high) in bands.items():\n\n mask = (\n (freqs >= low) &\n (freqs < high)\n )\n\n power = np.trapezoid(\n psd[mask],\n freqs[mask]\n )\n\n band_power[band_name] = power\n features.append(power)\n\n # ----------------------------------------------------\n # Relative power\n # ----------------------------------------------------\n\n total_power = (\n sum(band_power.values()) +\n 1e-10\n )\n\n relative = {}\n\n for band_name in bands.keys():\n\n rel_power = (\n band_power[band_name] /\n total_power\n )\n\n relative[band_name] = rel_power\n features.append(rel_power)\n\n # ----------------------------------------------------\n # Ratios\n # ----------------------------------------------------\n\n theta_alpha = (\n relative[\"theta\"] /\n (relative[\"alpha\"] + 1e-10)\n )\n\n beta_alpha = (\n relative[\"beta\"] /\n (relative[\"alpha\"] + 1e-10)\n )\n\n alpha_theta = (\n relative[\"alpha\"] /\n (relative[\"theta\"] + 1e-10)\n )\n\n beta_alpha_theta = (\n relative[\"beta\"] /\n (\n relative[\"alpha\"] +\n relative[\"theta\"] +\n 1e-10\n )\n )\n\n features.extend([\n theta_alpha,\n beta_alpha,\n alpha_theta,\n beta_alpha_theta\n ])\n\n return np.asarray(\n features,\n dtype=np.float32\n )\n\n\n# ------------------------------------------------------------\n# Extract for all fatigue segments\n# ------------------------------------------------------------\n\nprint(\"Extracting fatigue features...\")\nprint(\"Segments:\", all_segments.shape[0])\nprint(\"Channels:\", all_segments.shape[2])\n\nX_fatigue_rich = np.asarray([\n extract_fatigue_rich_features(\n segment,\n sfreq=128\n )\n for segment in all_segments\n])\n\n\nprint(\"\\n\" + \"=\" * 60)\nprint(\"FATIGUE RICH FEATURES\")\nprint(\"=\" * 60)\n\nprint(\n \"Full feature matrix:\",\n X_fatigue_rich.shape\n)\n\nprint(\n \"Features per segment:\",\n X_fatigue_rich.shape[1]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:01:58.932536Z","iopub.execute_input":"2026-09-27T17:01:58.933014Z","iopub.status.idle":"2026-09-27T17:04:39.416563Z","shell.execute_reply.started":"2026-09-27T17:01:58.932977Z","shell.execute_reply":"2026-09-27T17:04:39.413169Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 9: FINAL TRAIN / TEST FEATURES\n# ============================================================\n\nX_fatigue_rich_train = X_fatigue_rich[\n fatigue_train_idx\n]\n\nX_fatigue_rich_test = X_fatigue_rich[\n fatigue_test_idx\n]\n\n\nprint(\"=\" * 60)\nprint(\"FINAL FATIGUE FEATURE DATASET\")\nprint(\"=\" * 60)\n\nprint(\n \"X_train:\",\n X_fatigue_rich_train.shape\n)\n\nprint(\n \"X_test :\",\n X_fatigue_rich_test.shape\n)\n\nprint(\n \"y_train:\",\n y_fatigue_train.shape\n)\n\nprint(\n \"y_test :\",\n y_fatigue_test.shape\n)\n\n\nprint(\"\\nClass distribution:\")\n\nprint(\n \"Train:\",\n pd.Series(\n y_fatigue_train\n ).value_counts().sort_index()\n)\n\nprint(\n \"Test :\",\n pd.Series(\n y_fatigue_test\n ).value_counts().sort_index()\n)\n\n\n# ------------------------------------------------------------\n# Final recording leakage check\n# ------------------------------------------------------------\n\ntrain_recordings = set(\n all_record_ids[fatigue_train_idx]\n)\n\ntest_recordings = set(\n all_record_ids[fatigue_test_idx]\n)\n\nprint(\"\\nRecording overlap:\",\n len(train_recordings & test_recordings))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:05:16.030647Z","iopub.execute_input":"2026-09-27T17:05:16.036152Z","iopub.status.idle":"2026-09-27T17:05:16.080283Z","shell.execute_reply.started":"2026-09-27T17:05:16.035987Z","shell.execute_reply":"2026-09-27T17:05:16.078336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 10: RICH RANDOM FOREST\n# ============================================================\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import (\n accuracy_score,\n classification_report,\n confusion_matrix,\n f1_score\n)\n\nfatigue_rich_rf = RandomForestClassifier(\n n_estimators=300,\n random_state=42,\n class_weight=\"balanced\",\n n_jobs=-1\n)\n\nfatigue_rich_rf.fit(\n X_fatigue_rich_train,\n y_fatigue_train\n)\n\n\n# ------------------------------------------------------------\n# Predict\n# ------------------------------------------------------------\n\nfatigue_rich_pred = fatigue_rich_rf.predict(\n X_fatigue_rich_test\n)\n\n\n# ------------------------------------------------------------\n# Evaluate\n# ------------------------------------------------------------\n\nfatigue_rich_accuracy = accuracy_score(\n y_fatigue_test,\n fatigue_rich_pred\n)\n\nfatigue_rich_f1 = f1_score(\n y_fatigue_test,\n fatigue_rich_pred,\n average=\"macro\"\n)\n\n\nprint(\"=\" * 60)\nprint(\"FATIGUE / DROWSINESS RANDOM FOREST\")\nprint(\"=\" * 60)\n\nprint(\n f\"Accuracy : {fatigue_rich_accuracy:.4f} \"\n f\"({fatigue_rich_accuracy * 100:.2f}%)\"\n)\n\nprint(\n f\"Macro F1 : {fatigue_rich_f1:.4f}\"\n)\n\nprint(\"\\nClassification Report:\")\n\nprint(\n classification_report(\n y_fatigue_test,\n fatigue_rich_pred,\n target_names=[\n \"Non-drowsy\",\n \"Drowsy\"\n ]\n )\n)\n\nprint(\"Confusion Matrix:\")\n\nprint(\n confusion_matrix(\n y_fatigue_test,\n fatigue_rich_pred\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:05:55.003361Z","iopub.execute_input":"2026-09-27T17:05:55.003858Z","iopub.status.idle":"2026-09-27T17:05:55.809817Z","shell.execute_reply.started":"2026-09-27T17:05:55.003824Z","shell.execute_reply":"2026-09-27T17:05:55.807334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# FATIGUE STEP 10: FAST RICH RANDOM FOREST\n# ============================================================\n\nfatigue_rich_rf = RandomForestClassifier(\n n_estimators=100,\n random_state=42,\n class_weight=\"balanced\",\n n_jobs=-1\n)\n\nfatigue_rich_rf.fit(\n X_fatigue_rich_train,\n y_fatigue_train\n)\n\n\n# ------------------------------------------------------------\n# Predict\n# ------------------------------------------------------------\n\nfatigue_rich_pred = fatigue_rich_rf.predict(\n X_fatigue_rich_test\n)\n\n\n# ------------------------------------------------------------\n# Evaluate\n# ------------------------------------------------------------\n\nfatigue_rich_accuracy = accuracy_score(\n y_fatigue_test,\n fatigue_rich_pred\n)\n\nfatigue_rich_f1 = f1_score(\n y_fatigue_test,\n fatigue_rich_pred,\n average=\"macro\"\n)\n\n\nprint(\"=\" * 60)\nprint(\"FATIGUE / DROWSINESS RANDOM FOREST\")\nprint(\"=\" * 60)\n\nprint(\n f\"Accuracy : {fatigue_rich_accuracy:.4f} \"\n f\"({fatigue_rich_accuracy * 100:.2f}%)\"\n)\n\nprint(\n f\"Macro F1 : {fatigue_rich_f1:.4f}\"\n)\n\nprint(\"\\nClassification Report:\")\n\nprint(\n classification_report(\n y_fatigue_test,\n fatigue_rich_pred,\n target_names=[\n \"Non-drowsy\",\n \"Drowsy\"\n ]\n )\n)\n\nprint(\"Confusion Matrix:\")\n\nprint(\n confusion_matrix(\n y_fatigue_test,\n fatigue_rich_pred\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:06:39.426283Z","iopub.execute_input":"2026-09-27T17:06:39.427783Z","iopub.status.idle":"2026-09-27T17:06:40.437374Z","shell.execute_reply.started":"2026-09-27T17:06:39.42774Z","shell.execute_reply":"2026-09-27T17:06:40.435834Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 18: FATIGUE / DROWSINESS PREDICTION\n# ============================================================\n\nfatigue_model = fatigue_rich_rf\n\n\ndef predict_fatigue_state(eeg_segment):\n\n features = extract_rich_features(\n eeg_segment,\n sfreq=128\n )\n\n features = features.reshape(1, -1)\n\n prediction = fatigue_model.predict(features)[0]\n\n if prediction == 1:\n return \"Drowsy\"\n else:\n return \"Non-drowsy\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:07:57.762092Z","iopub.execute_input":"2026-09-27T17:07:57.763032Z","iopub.status.idle":"2026-09-27T17:07:57.770216Z","shell.execute_reply.started":"2026-09-27T17:07:57.762976Z","shell.execute_reply":"2026-09-27T17:07:57.768759Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Fatigue model exists:\", fatigue_rich_rf is not None)\nprint(\"Model type:\", type(fatigue_rich_rf))\nprint(\"Test feature shape:\", X_fatigue_rich_test.shape)\n\nsample = X_fatigue_rich_test[0].reshape(1, -1)\n\nprediction = fatigue_rich_rf.predict(sample)[0]\n\nprint(\"Predicted fatigue label:\", prediction)\n\nif prediction == 1:\n print(\"Fatigue state: Drowsy\")\nelse:\n print(\"Fatigue state: Non-drowsy\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:09:00.456551Z","iopub.execute_input":"2026-09-27T17:09:00.458772Z","iopub.status.idle":"2026-09-27T17:09:00.537382Z","shell.execute_reply.started":"2026-09-27T17:09:00.458617Z","shell.execute_reply":"2026-09-27T17:09:00.536036Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 18: INTEGRATE LEARNED FATIGUE MODEL\n# ============================================================\n\ndef predict_fatigue_state_from_features(eeg_features):\n \"\"\"\n Predict fatigue/drowsiness from the already extracted\n 350-dimensional rich EEG feature vector.\n \"\"\"\n\n eeg_features = eeg_features.reshape(1, -1)\n\n prediction = fatigue_model.predict(eeg_features)[0]\n\n if prediction == 1:\n return \"Drowsy\"\n else:\n return \"Non-drowsy\"\n\n\n# Test using an existing test sample\nsample_features = X_fatigue_rich_test[0]\n\nfatigue_state = predict_fatigue_state_from_features(\n sample_features\n)\n\nprint(\"Predicted fatigue state:\", fatigue_state)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:09:43.101903Z","iopub.execute_input":"2026-09-27T17:09:43.102395Z","iopub.status.idle":"2026-09-27T17:09:43.154349Z","shell.execute_reply.started":"2026-09-27T17:09:43.10235Z","shell.execute_reply":"2026-09-27T17:09:43.153244Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 19: INSPECT LABELS FOR ENGAGEMENT PROXY\n# ============================================================\n\nprint(\"Unique original labels:\")\nprint(np.unique(all_labels))\n\nprint(\"\\nLabel counts:\")\nunique_labels, counts = np.unique(all_labels, return_counts=True)\n\nfor label, count in zip(unique_labels, counts):\n print(f\"{label}: {count}\")\n\nprint(\"\\nTotal segments:\", len(all_labels))\nprint(\"EEG segments shape:\", all_segments.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:11:16.726391Z","iopub.execute_input":"2026-09-27T17:11:16.726828Z","iopub.status.idle":"2026-09-27T17:11:16.74304Z","shell.execute_reply.started":"2026-09-27T17:11:16.726791Z","shell.execute_reply":"2026-09-27T17:11:16.741084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 20: CREATE ENGAGEMENT DATASET\n# Focused = Engaged\n# Unfocused = Not Engaged\n# Drowsy is excluded\n# ============================================================\n\nengagement_mask = np.isin(\n all_labels,\n [\"Focused\", \"Unfocused\"]\n)\n\nengagement_segments = all_segments[engagement_mask]\nengagement_original_labels = all_labels[engagement_mask]\n\n# Binary labels:\n# 1 = Focused / Engaged\n# 0 = Unfocused / Not Engaged\n\nengagement_labels = np.where(\n engagement_original_labels == \"Focused\",\n 1,\n 0\n)\n\nprint(\"=\" * 60)\nprint(\"ENGAGEMENT DATASET\")\nprint(\"=\" * 60)\n\nprint(\"Segments shape:\", engagement_segments.shape)\nprint(\"Labels shape:\", engagement_labels.shape)\n\nprint(\"\\nClass distribution:\")\n\nunique, counts = np.unique(\n engagement_labels,\n return_counts=True\n)\n\nfor label, count in zip(unique, counts):\n state = \"Engaged (Focused)\" if label == 1 else \"Not Engaged (Unfocused)\"\n print(f\"{state}: {count}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:11:49.717505Z","iopub.execute_input":"2026-09-27T17:11:49.71937Z","iopub.status.idle":"2026-09-27T17:11:52.184185Z","shell.execute_reply.started":"2026-09-27T17:11:49.719324Z","shell.execute_reply":"2026-09-27T17:11:52.182881Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 21: RECORDING-DISJOINT ENGAGEMENT SPLIT\n# ============================================================\n\nfrom sklearn.model_selection import GroupShuffleSplit\n\n# Original recording IDs for every segment\nengagement_recording_ids = all_record_ids[engagement_mask]\n\nengagement_splitter = GroupShuffleSplit(\n n_splits=1,\n test_size=0.20,\n random_state=42\n)\n\nengagement_train_idx, engagement_test_idx = next(\n engagement_splitter.split(\n engagement_segments,\n engagement_labels,\n groups=engagement_recording_ids\n )\n)\n\n# Create train/test sets\nengagement_train_segments = engagement_segments[\n engagement_train_idx\n]\n\nengagement_test_segments = engagement_segments[\n engagement_test_idx\n]\n\ny_engagement_train = engagement_labels[\n engagement_train_idx\n]\n\ny_engagement_test = engagement_labels[\n engagement_test_idx\n]\n\ngroups_engagement_train = engagement_recording_ids[\n engagement_train_idx\n]\n\ngroups_engagement_test = engagement_recording_ids[\n engagement_test_idx\n]\n\nprint(\"=\" * 60)\nprint(\"ENGAGEMENT RECORDING-DISJOINT SPLIT\")\nprint(\"=\" * 60)\n\nprint(\"Training segments:\", len(y_engagement_train))\nprint(\"Testing segments :\", len(y_engagement_test))\n\nprint(\n \"Training recordings:\",\n len(np.unique(groups_engagement_train))\n)\n\nprint(\n \"Testing recordings :\",\n len(np.unique(groups_engagement_test))\n)\n\nprint(\n \"Recording overlap :\",\n len(\n set(groups_engagement_train)\n &\n set(groups_engagement_test)\n )\n)\n\nprint(\"\\nTraining class distribution:\")\nprint(np.unique(y_engagement_train, return_counts=True))\n\nprint(\"\\nTesting class distribution:\")\nprint(np.unique(y_engagement_test, return_counts=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:12:25.556851Z","iopub.execute_input":"2026-09-27T17:12:25.557286Z","iopub.status.idle":"2026-09-27T17:12:25.824841Z","shell.execute_reply.started":"2026-09-27T17:12:25.557248Z","shell.execute_reply":"2026-09-27T17:12:25.82354Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 22: RICH EEG FEATURES FOR ENGAGEMENT\n# ============================================================\n\nprint(\"Extracting engagement training features...\")\n\nX_engagement_rich_train = np.array([\n extract_rich_features(segment, sfreq=128)\n for segment in engagement_train_segments\n])\n\nprint(\"Extracting engagement testing features...\")\n\nX_engagement_rich_test = np.array([\n extract_rich_features(segment, sfreq=128)\n for segment in engagement_test_segments\n])\n\nprint(\"=\" * 60)\nprint(\"ENGAGEMENT RICH FEATURE MATRIX\")\nprint(\"=\" * 60)\n\nprint(\n \"Training shape:\",\n X_engagement_rich_train.shape\n)\n\nprint(\n \"Testing shape :\",\n X_engagement_rich_test.shape\n)\n\nprint(\n \"Expected features per segment:\",\n 25 * 14\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:13:01.518437Z","iopub.execute_input":"2026-09-27T17:13:01.519408Z","iopub.status.idle":"2026-09-27T17:13:01.53308Z","shell.execute_reply.started":"2026-09-27T17:13:01.519369Z","shell.execute_reply":"2026-09-27T17:13:01.531418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 22A: DEFINE RICH EEG FEATURE EXTRACTOR\n# ============================================================\n\nfrom scipy.signal import welch\nimport numpy as np\n\n\ndef extract_rich_features(eeg_segment, sfreq=128):\n\n # EEG shape: samples × channels\n eeg_segment = np.asarray(eeg_segment)\n\n # Make sure shape is samples × channels\n if eeg_segment.shape[0] < eeg_segment.shape[1]:\n eeg_segment = eeg_segment.T\n\n bands = {\n \"delta\": (1, 4),\n \"theta\": (4, 8),\n \"alpha\": (8, 13),\n \"beta\": (13, 30),\n \"gamma\": (30, 45)\n }\n\n features = []\n\n # --------------------------------------------------------\n # Process each EEG channel\n # --------------------------------------------------------\n\n for ch in range(eeg_segment.shape[1]):\n\n signal = eeg_segment[:, ch]\n\n freqs, psd = welch(\n signal,\n fs=sfreq,\n nperseg=min(256, len(signal))\n )\n\n band_powers = []\n\n # Absolute band powers\n for low, high in bands.values():\n\n mask = (freqs >= low) & (freqs < high)\n\n power = np.trapezoid(\n psd[mask],\n freqs[mask]\n )\n\n band_powers.append(power)\n\n band_powers = np.array(\n band_powers,\n dtype=np.float64\n )\n\n # ----------------------------------------------------\n # Relative powers\n # ----------------------------------------------------\n\n total_power = np.sum(band_powers) + 1e-12\n\n relative_powers = (\n band_powers / total_power\n )\n\n delta, theta, alpha, beta, gamma = band_powers\n\n # ----------------------------------------------------\n # Ratios\n # ----------------------------------------------------\n\n theta_alpha = theta / (alpha + 1e-12)\n\n beta_alpha = beta / (alpha + 1e-12)\n\n alpha_theta = alpha / (theta + 1e-12)\n\n beta_alpha_theta = (\n beta /\n (alpha + theta + 1e-12)\n )\n\n # ----------------------------------------------------\n # 14 features per channel\n # ----------------------------------------------------\n\n channel_features = np.concatenate([\n band_powers, # 5\n relative_powers, # 5\n [\n theta_alpha, # 1\n beta_alpha, # 1\n alpha_theta, # 1\n beta_alpha_theta # 1\n ]\n ])\n\n features.extend(channel_features)\n\n return np.asarray(features, dtype=np.float32)\n\n\n# Quick verification\ntest_features = extract_rich_features(\n engagement_train_segments[0],\n sfreq=128\n)\n\nprint(\"Feature vector shape:\", test_features.shape)\nprint(\"Expected shape: (350,)\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:13:35.271944Z","iopub.execute_input":"2026-09-27T17:13:35.273519Z","iopub.status.idle":"2026-09-27T17:13:35.406199Z","shell.execute_reply.started":"2026-09-27T17:13:35.273475Z","shell.execute_reply":"2026-09-27T17:13:35.405137Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 22B: EXTRACT ENGAGEMENT TRAIN/TEST FEATURES\n# ============================================================\n\nprint(\"Extracting engagement training features...\")\n\nX_engagement_rich_train = np.array([\n extract_rich_features(segment, sfreq=128)\n for segment in engagement_train_segments\n])\n\nprint(\"Extracting engagement testing features...\")\n\nX_engagement_rich_test = np.array([\n extract_rich_features(segment, sfreq=128)\n for segment in engagement_test_segments\n])\n\nprint(\"=\" * 60)\nprint(\"ENGAGEMENT RICH FEATURE MATRIX\")\nprint(\"=\" * 60)\n\nprint(\"Training shape:\", X_engagement_rich_train.shape)\nprint(\"Testing shape :\", X_engagement_rich_test.shape)\n\nprint(\"Training labels:\", y_engagement_train.shape)\nprint(\"Testing labels :\", y_engagement_test.shape)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:14:15.281984Z","iopub.execute_input":"2026-09-27T17:14:15.283359Z","iopub.status.idle":"2026-09-27T17:15:23.119708Z","shell.execute_reply.started":"2026-09-27T17:14:15.283309Z","shell.execute_reply":"2026-09-27T17:15:23.115823Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 23: TRAIN ENGAGEMENT RANDOM FOREST\n# ============================================================\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import (\n accuracy_score,\n classification_report,\n confusion_matrix,\n f1_score\n)\n\nengagement_rf = RandomForestClassifier(\n n_estimators=100,\n random_state=42,\n class_weight=\"balanced\",\n n_jobs=-1\n)\n\nengagement_rf.fit(\n X_engagement_rich_train,\n y_engagement_train\n)\n\n# ------------------------------------------------------------\n# Predict\n# ------------------------------------------------------------\n\nengagement_pred = engagement_rf.predict(\n X_engagement_rich_test\n)\n\n# ------------------------------------------------------------\n# Evaluate\n# ------------------------------------------------------------\n\nengagement_accuracy = accuracy_score(\n y_engagement_test,\n engagement_pred\n)\n\nengagement_f1 = f1_score(\n y_engagement_test,\n engagement_pred,\n average=\"macro\"\n)\n\nprint(\"=\" * 60)\nprint(\"ENGAGEMENT RANDOM FOREST\")\nprint(\"=\" * 60)\n\nprint(\n f\"Accuracy : {engagement_accuracy:.4f} \"\n f\"({engagement_accuracy * 100:.2f}%)\"\n)\n\nprint(\n f\"Macro F1 : {engagement_f1:.4f}\"\n)\n\nprint(\"\\nClassification Report:\")\n\nprint(\n classification_report(\n y_engagement_test,\n engagement_pred,\n target_names=[\n \"Not Engaged\",\n \"Engaged\"\n ]\n )\n)\n\nprint(\"Confusion Matrix:\")\n\nprint(\n confusion_matrix(\n y_engagement_test,\n engagement_pred\n )\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:16:15.901446Z","iopub.execute_input":"2026-09-27T17:16:15.903066Z","iopub.status.idle":"2026-09-27T17:16:16.494311Z","shell.execute_reply.started":"2026-09-27T17:16:15.902975Z","shell.execute_reply":"2026-09-27T17:16:16.493151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 24: FULL LEARNED COGNITIVE-STATE ENGINE\n# ============================================================\n\n# ------------------------------------------------------------\n# State-to-score mappings\n# ------------------------------------------------------------\n\nATTENTION_VALUES = {\n \"Focused\": 0.90,\n \"Unfocused\": 0.40,\n \"Drowsy\": 0.20\n}\n\nWORKLOAD_VALUES = {\n \"Low\": 0.25,\n \"High\": 0.80\n}\n\nFATIGUE_VALUES = {\n \"Non-drowsy\": 0.20,\n \"Drowsy\": 0.90\n}\n\nENGAGEMENT_VALUES = {\n \"Engaged\": 0.90,\n \"Not Engaged\": 0.40\n}\n\n\n# ------------------------------------------------------------\n# Full cognitive context\n# ------------------------------------------------------------\n\ndef calculate_full_cognitive_context(\n attention_state,\n workload_state,\n fatigue_state,\n engagement_state,\n alpha=0.25,\n beta=0.25,\n gamma=0.25,\n delta=0.25\n):\n\n A = ATTENTION_VALUES[attention_state]\n\n W = WORKLOAD_VALUES[workload_state]\n\n F = FATIGUE_VALUES[fatigue_state]\n\n E = ENGAGEMENT_VALUES[engagement_state]\n\n C = (\n alpha * A\n + beta * E\n + gamma * (1 - W)\n + delta * (1 - F)\n )\n\n return round(C, 4)\n\n\n# ------------------------------------------------------------\n# Test the upgraded engine\n# ------------------------------------------------------------\n\ntest_context = calculate_full_cognitive_context(\n attention_state=\"Focused\",\n workload_state=\"Low\",\n fatigue_state=\"Non-drowsy\",\n engagement_state=\"Engaged\"\n)\n\nprint(\"=\" * 60)\nprint(\"FULL ACME COGNITIVE CONTEXT\")\nprint(\"=\" * 60)\n\nprint(\"Attention : Focused\")\nprint(\"Workload : Low\")\nprint(\"Fatigue : Non-drowsy\")\nprint(\"Engagement : Engaged\")\nprint(\"-\" * 60)\nprint(\"Cognitive Context C:\", test_context)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:17:15.801156Z","iopub.execute_input":"2026-09-27T17:17:15.801632Z","iopub.status.idle":"2026-09-27T17:17:15.817647Z","shell.execute_reply.started":"2026-09-27T17:17:15.801595Z","shell.execute_reply":"2026-09-27T17:17:15.816221Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 25: UNIFIED COGNITIVE-STATE PREDICTION\n# ============================================================\n\ndef predict_full_cognitive_state(\n attention_features,\n workload_features,\n fatigue_features,\n engagement_features\n):\n\n # --------------------------------------------------------\n # Attention\n # --------------------------------------------------------\n\n attention_prediction = group_model.predict(\n attention_features.reshape(1, -1)\n )[0]\n\n # --------------------------------------------------------\n # Workload\n # --------------------------------------------------------\n\n workload_prediction = workload_model.predict(\n workload_features.reshape(1, -1)\n )[0]\n\n # --------------------------------------------------------\n # Fatigue\n # --------------------------------------------------------\n\n fatigue_prediction = fatigue_model.predict(\n fatigue_features.reshape(1, -1)\n )[0]\n\n fatigue_state = (\n \"Drowsy\"\n if fatigue_prediction == 1\n else \"Non-drowsy\"\n )\n\n # --------------------------------------------------------\n # Engagement\n # --------------------------------------------------------\n\n engagement_prediction = engagement_rf.predict(\n engagement_features.reshape(1, -1)\n )[0]\n\n engagement_state = (\n \"Engaged\"\n if engagement_prediction == 1\n else \"Not Engaged\"\n )\n\n # --------------------------------------------------------\n # Cognitive Context\n # --------------------------------------------------------\n\n cognitive_context = calculate_full_cognitive_context(\n attention_state=attention_prediction,\n workload_state=workload_prediction,\n fatigue_state=fatigue_state,\n engagement_state=engagement_state\n )\n\n return {\n \"attention\": attention_prediction,\n \"workload\": workload_prediction,\n \"fatigue\": fatigue_state,\n \"engagement\": engagement_state,\n \"cognitive_context\": cognitive_context\n }\n\n\nprint(\"Unified cognitive-state prediction function created successfully.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:18:43.351163Z","iopub.execute_input":"2026-09-27T17:18:43.351663Z","iopub.status.idle":"2026-09-27T17:18:43.364859Z","shell.execute_reply.started":"2026-09-27T17:18:43.351626Z","shell.execute_reply":"2026-09-27T17:18:43.363608Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 26: TEST FULL COGNITIVE-STATE PIPELINE\n# ============================================================\n\n# ------------------------------------------------------------\n# 1. Attention sample\n# ------------------------------------------------------------\n\nattention_sample = X_rich_test[0]\n\n# Apply the same feature selection used by the locked\n# attention model\nattention_sample_selected = (\n attention_selector.transform(\n attention_sample.reshape(1, -1)\n )[0]\n)\n\n\n# ------------------------------------------------------------\n# 2. Workload sample\n# ------------------------------------------------------------\n\nworkload_sample = X_workload_rich_test[0]\n\n\n# ------------------------------------------------------------\n# 3. Fatigue sample\n# ------------------------------------------------------------\n\nfatigue_sample = X_fatigue_rich_test[0]\n\n\n# ------------------------------------------------------------\n# 4. Engagement sample\n# ------------------------------------------------------------\n\nengagement_sample = X_engagement_rich_test[0]\n\n\n# ------------------------------------------------------------\n# 5. Predict all four states\n# ------------------------------------------------------------\n\nfull_state = predict_full_cognitive_state(\n attention_features=attention_sample_selected,\n workload_features=workload_sample,\n fatigue_features=fatigue_sample,\n engagement_features=engagement_sample\n)\n\n\n# ------------------------------------------------------------\n# Display\n# ------------------------------------------------------------\n\nprint(\"=\" * 60)\nprint(\"ACME FULL COGNITIVE-STATE TEST\")\nprint(\"=\" * 60)\n\nprint(\"Attention :\", full_state[\"attention\"])\nprint(\"Workload :\", full_state[\"workload\"])\nprint(\"Fatigue :\", full_state[\"fatigue\"])\nprint(\"Engagement :\", full_state[\"engagement\"])\nprint(\"-\" * 60)\nprint(\n \"Cognitive Context C:\",\n full_state[\"cognitive_context\"]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:19:17.384905Z","iopub.execute_input":"2026-09-27T17:19:17.38607Z","iopub.status.idle":"2026-09-27T17:19:17.411248Z","shell.execute_reply.started":"2026-09-27T17:19:17.385936Z","shell.execute_reply":"2026-09-27T17:19:17.408546Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find variables related to attention feature selection\n\nprint([\n name for name in globals()\n if \"select\" in name.lower()\n or \"feature\" in name.lower()\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:19:51.411723Z","iopub.execute_input":"2026-09-27T17:19:51.41299Z","iopub.status.idle":"2026-09-27T17:19:51.421036Z","shell.execute_reply.started":"2026-09-27T17:19:51.412945Z","shell.execute_reply":"2026-09-27T17:19:51.419832Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 26B: TEST FULL ACME COGNITIVE-STATE PIPELINE\n# ============================================================\n\n# ------------------------------------------------------------\n# 1. Attention sample\n# ------------------------------------------------------------\n\nattention_sample = X_rich_test[0]\n\nattention_sample_selected = (\n selector.transform(\n attention_sample.reshape(1, -1)\n )[0]\n)\n\n\n# ------------------------------------------------------------\n# 2. Workload sample\n# ------------------------------------------------------------\n\nworkload_sample = X_workload_rich_test[0]\n\n\n# ------------------------------------------------------------\n# 3. Fatigue sample\n# ------------------------------------------------------------\n\nfatigue_sample = X_fatigue_rich_test[0]\n\n\n# ------------------------------------------------------------\n# 4. Engagement sample\n# ------------------------------------------------------------\n\nengagement_sample = X_engagement_rich_test[0]\n\n\n# ------------------------------------------------------------\n# 5. Predict all four states\n# ------------------------------------------------------------\n\nfull_state = predict_full_cognitive_state(\n attention_features=attention_sample_selected,\n workload_features=workload_sample,\n fatigue_features=fatigue_sample,\n engagement_features=engagement_sample\n)\n\n\n# ------------------------------------------------------------\n# Display\n# ------------------------------------------------------------\n\nprint(\"=\" * 60)\nprint(\"ACME FULL COGNITIVE-STATE TEST\")\nprint(\"=\" * 60)\n\nprint(\"Attention :\", full_state[\"attention\"])\nprint(\"Workload :\", full_state[\"workload\"])\nprint(\"Fatigue :\", full_state[\"fatigue\"])\nprint(\"Engagement :\", full_state[\"engagement\"])\nprint(\"-\" * 60)\nprint(\n \"Cognitive Context C:\",\n full_state[\"cognitive_context\"]\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:20:27.358145Z","iopub.execute_input":"2026-09-27T17:20:27.358571Z","iopub.status.idle":"2026-09-27T17:20:27.657917Z","shell.execute_reply.started":"2026-09-27T17:20:27.358534Z","shell.execute_reply":"2026-09-27T17:20:27.656227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 26C: FIX ATTENTION MODEL INTEGRATION\n# ============================================================\n\ndef predict_full_cognitive_state(\n attention_features,\n workload_features,\n fatigue_features,\n engagement_features\n):\n\n # --------------------------------------------------------\n # Attention\n # Upgraded model:\n # 350 rich features -> selector -> 175 features -> selected_rf\n # --------------------------------------------------------\n\n attention_prediction = selected_rf.predict(\n attention_features.reshape(1, -1)\n )[0]\n\n # --------------------------------------------------------\n # Workload\n # 196 rich features\n # --------------------------------------------------------\n\n workload_prediction = workload_model.predict(\n workload_features.reshape(1, -1)\n )[0]\n\n # --------------------------------------------------------\n # Fatigue\n # 350 rich features\n # --------------------------------------------------------\n\n fatigue_prediction = fatigue_model.predict(\n fatigue_features.reshape(1, -1)\n )[0]\n\n fatigue_state = (\n \"Drowsy\"\n if fatigue_prediction == 1\n else \"Non-drowsy\"\n )\n\n # --------------------------------------------------------\n # Engagement\n # 350 rich features\n # --------------------------------------------------------\n\n engagement_prediction = engagement_rf.predict(\n engagement_features.reshape(1, -1)\n )[0]\n\n engagement_state = (\n \"Engaged\"\n if engagement_prediction == 1\n else \"Not Engaged\"\n )\n\n # --------------------------------------------------------\n # Cognitive Context\n # --------------------------------------------------------\n\n cognitive_context = calculate_full_cognitive_context(\n attention_state=attention_prediction,\n workload_state=workload_prediction,\n fatigue_state=fatigue_state,\n engagement_state=engagement_state\n )\n\n return {\n \"attention\": attention_prediction,\n \"workload\": workload_prediction,\n \"fatigue\": fatigue_state,\n \"engagement\": engagement_state,\n \"cognitive_context\": cognitive_context\n }\n\n\nprint(\"Unified cognitive-state function updated successfully.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:21:09.263064Z","iopub.execute_input":"2026-09-27T17:21:09.263469Z","iopub.status.idle":"2026-09-27T17:21:09.275875Z","shell.execute_reply.started":"2026-09-27T17:21:09.263433Z","shell.execute_reply":"2026-09-27T17:21:09.274266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"attention_sample = X_selected_test[0]\nworkload_sample = X_workload_rich_test[0]\nfatigue_sample = X_fatigue_rich_test[0]\nengagement_sample = X_engagement_rich_test[0]\n\nfull_state = predict_full_cognitive_state(\n attention_features=attention_sample,\n workload_features=workload_sample,\n fatigue_features=fatigue_sample,\n engagement_features=engagement_sample\n)\n\nprint(\"=\" * 60)\nprint(\"ACME FULL COGNITIVE-STATE TEST\")\nprint(\"=\" * 60)\n\nprint(\"Attention :\", full_state[\"attention\"])\nprint(\"Workload :\", full_state[\"workload\"])\nprint(\"Fatigue :\", full_state[\"fatigue\"])\nprint(\"Engagement :\", full_state[\"engagement\"])\nprint(\"-\" * 60)\nprint(\"Cognitive Context C:\", full_state[\"cognitive_context\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:21:46.011458Z","iopub.execute_input":"2026-09-27T17:21:46.015814Z","iopub.status.idle":"2026-09-27T17:21:46.484471Z","shell.execute_reply.started":"2026-09-27T17:21:46.015694Z","shell.execute_reply":"2026-09-27T17:21:46.482886Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 27: UPDATED C-RAG WITH FULL COGNITIVE CONTEXT\n# ============================================================\n\ndef final_c_rag_retrieve(\n query,\n attention_state,\n workload_state,\n fatigue_state,\n engagement_state,\n top_k=3,\n eta=0.7\n):\n\n # --------------------------------------------------------\n # Retrieve semantic candidates\n # --------------------------------------------------------\n\n faiss_results = faiss_retrieve(\n query,\n top_k=len(learner_memories)\n )\n\n # --------------------------------------------------------\n # Calculate FULL cognitive context\n # --------------------------------------------------------\n\n cognitive_context = calculate_full_cognitive_context(\n attention_state=attention_state,\n workload_state=workload_state,\n fatigue_state=fatigue_state,\n engagement_state=engagement_state\n )\n\n results = []\n\n # --------------------------------------------------------\n # Calculate ACME memory score\n # --------------------------------------------------------\n\n for item in faiss_results:\n\n memory = next(\n m for m in learner_memories\n if m[\"id\"] == item[\"id\"]\n )\n\n memory_strength = memory_score(\n memory[\"importance\"],\n 2,\n cognitive_context,\n memory[\"value\"]\n )\n\n # ----------------------------------------------------\n # C-RAG score\n # ----------------------------------------------------\n\n c_rag_score = (\n eta * item[\"similarity\"]\n + (1 - eta) * memory_strength\n )\n\n results.append({\n \"id\": memory[\"id\"],\n \"topic\": memory[\"topic\"],\n \"text\": memory[\"text\"],\n \"semantic_similarity\": item[\"similarity\"],\n \"memory_score\": round(\n memory_strength,\n 4\n ),\n \"c_rag_score\": round(\n c_rag_score,\n 4\n )\n })\n\n # --------------------------------------------------------\n # Rank by C-RAG score\n # --------------------------------------------------------\n\n results.sort(\n key=lambda x: x[\"c_rag_score\"],\n reverse=True\n )\n\n return results[:top_k]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:22:59.621469Z","iopub.execute_input":"2026-09-27T17:22:59.621831Z","iopub.status.idle":"2026-09-27T17:22:59.631368Z","shell.execute_reply.started":"2026-09-27T17:22:59.621802Z","shell.execute_reply":"2026-09-27T17:22:59.630045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 28: TEST UPDATED C-RAG\n# ============================================================\n\ntest_query = \"I am struggling with recursion. Can you help me?\"\n\ntest_attention = \"Focused\"\ntest_workload = \"High\"\ntest_fatigue = \"Non-drowsy\"\ntest_engagement = \"Engaged\"\n\nc_rag_results = final_c_rag_retrieve(\n query=test_query,\n attention_state=test_attention,\n workload_state=test_workload,\n fatigue_state=test_fatigue,\n engagement_state=test_engagement,\n top_k=3,\n eta=0.7\n)\n\ncognitive_context = calculate_full_cognitive_context(\n attention_state=test_attention,\n workload_state=test_workload,\n fatigue_state=test_fatigue,\n engagement_state=test_engagement\n)\n\nprint(\"=\" * 70)\nprint(\"UPDATED ACME C-RAG TEST\")\nprint(\"=\" * 70)\n\nprint(\"Query:\", test_query)\n\nprint(\"\\nCognitive State:\")\nprint(\"Attention :\", test_attention)\nprint(\"Workload :\", test_workload)\nprint(\"Fatigue :\", test_fatigue)\nprint(\"Engagement :\", test_engagement)\n\nprint(\"\\nCognitive Context C:\", cognitive_context)\n\nprint(\"\\nRetrieved Memories:\")\nprint(\"-\" * 70)\n\nfor rank, item in enumerate(c_rag_results, start=1):\n\n print(f\"\\nRank {rank}\")\n print(\"ID:\", item[\"id\"])\n print(\"Topic:\", item[\"topic\"])\n print(\"Semantic Similarity:\", item[\"semantic_similarity\"])\n print(\"Memory Score:\", item[\"memory_score\"])\n print(\"C-RAG Score:\", item[\"c_rag_score\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:23:42.10775Z","iopub.execute_input":"2026-09-27T17:23:42.112156Z","iopub.status.idle":"2026-09-27T17:23:42.266424Z","shell.execute_reply.started":"2026-09-27T17:23:42.111907Z","shell.execute_reply":"2026-09-27T17:23:42.263272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import inspect\n\nprint(inspect.signature(final_c_rag_retrieve))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:24:22.033747Z","iopub.execute_input":"2026-09-27T17:24:22.035437Z","iopub.status.idle":"2026-09-27T17:24:22.044773Z","shell.execute_reply.started":"2026-09-27T17:24:22.035372Z","shell.execute_reply":"2026-09-27T17:24:22.042603Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 29: CHECK RETRIEVAL EVALUATION VARIABLES\n# ============================================================\n\nprint([\n name for name in globals()\n if any(\n key in name.lower()\n for key in [\n \"query\",\n \"retrieval\",\n \"evaluation\",\n \"relevant\",\n \"ground\"\n ]\n )\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:24:50.950684Z","iopub.execute_input":"2026-09-27T17:24:50.95177Z","iopub.status.idle":"2026-09-27T17:24:50.965258Z","shell.execute_reply.started":"2026-09-27T17:24:50.951725Z","shell.execute_reply":"2026-09-27T17:24:50.964048Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 30: INSPECT EXISTING RETRIEVAL BENCHMARK\n# ============================================================\n\nprint(\"Number of evaluation queries:\",\n len(evaluation_queries_expanded))\n\nprint(\"\\nFirst evaluation query:\")\n\nprint(evaluation_queries_expanded[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:25:21.456004Z","iopub.execute_input":"2026-09-27T17:25:21.457045Z","iopub.status.idle":"2026-09-27T17:25:21.467386Z","shell.execute_reply.started":"2026-09-27T17:25:21.457001Z","shell.execute_reply":"2026-09-27T17:25:21.465151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 31: INSPECT ALL EVALUATION QUERIES\n# ============================================================\n\nfor i, item in enumerate(evaluation_queries_expanded, start=1):\n print(f\"\\nQuery {i}\")\n print(\"Query:\", item[\"query\"])\n print(\"Relevant memory:\", item[\"relevant_memory\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:25:54.525246Z","iopub.execute_input":"2026-09-27T17:25:54.52633Z","iopub.status.idle":"2026-09-27T17:25:54.5338Z","shell.execute_reply.started":"2026-09-27T17:25:54.526288Z","shell.execute_reply":"2026-09-27T17:25:54.532698Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 32: UPDATED ACME C-RAG EVALUATION\n# ============================================================\n\nimport numpy as np\n\n# Fixed cognitive state for controlled comparison\neval_attention = \"Focused\"\neval_workload = \"High\"\neval_fatigue = \"Non-drowsy\"\neval_engagement = \"Engaged\"\n\neval_cognitive_context = calculate_full_cognitive_context(\n attention_state=eval_attention,\n workload_state=eval_workload,\n fatigue_state=eval_fatigue,\n engagement_state=eval_engagement\n)\n\nprint(\"=\" * 70)\nprint(\"UPDATED ACME C-RAG EVALUATION\")\nprint(\"=\" * 70)\n\nprint(\"Attention :\", eval_attention)\nprint(\"Workload :\", eval_workload)\nprint(\"Fatigue :\", eval_fatigue)\nprint(\"Engagement :\", eval_engagement)\nprint(\"Cognitive Context C:\", eval_cognitive_context)\n\nupdated_ranks = []\n\nfor item in evaluation_queries_expanded:\n\n query_text = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n results = final_c_rag_retrieve(\n query=query_text,\n attention_state=eval_attention,\n workload_state=eval_workload,\n fatigue_state=eval_fatigue,\n engagement_state=eval_engagement,\n top_k=len(evaluation_memory_bank),\n eta=0.7\n )\n\n retrieved_ids = [\n result[\"id\"]\n for result in results\n ]\n\n if relevant_id in retrieved_ids:\n rank = retrieved_ids.index(relevant_id) + 1\n else:\n rank = len(retrieved_ids) + 1\n\n updated_ranks.append(rank)\n\n print(\"\\nQuery:\", query_text)\n print(\"Relevant memory:\", relevant_id)\n print(\"Rank:\", rank)\n\n\n# ------------------------------------------------------------\n# Metrics\n# ------------------------------------------------------------\n\nmrr = np.mean([\n 1 / rank\n for rank in updated_ranks\n])\n\nmean_rank = np.mean(updated_ranks)\n\nrecall_at_5 = np.mean([\n rank <= 5\n for rank in updated_ranks\n])\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"UPDATED ACME RESULTS\")\nprint(\"=\" * 70)\n\nprint(\"Ranks:\", updated_ranks)\nprint(f\"MRR : {mrr:.4f}\")\nprint(f\"Mean Rank: {mean_rank:.4f}\")\nprint(f\"Recall@5 : {recall_at_5:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:26:34.647002Z","iopub.execute_input":"2026-09-27T17:26:34.648589Z","iopub.status.idle":"2026-09-27T17:26:34.761544Z","shell.execute_reply.started":"2026-09-27T17:26:34.648528Z","shell.execute_reply":"2026-09-27T17:26:34.760436Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Evaluation memory bank size:\", len(evaluation_memory_bank))\nprint(\"Evaluation embeddings shape:\", evaluation_embeddings.shape)\nprint(\"Evaluation FAISS index size:\", evaluation_faiss_index.ntotal)\n\nprint(\"\\nFirst evaluation memory:\")\nprint(evaluation_memory_bank[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:27:09.184898Z","iopub.execute_input":"2026-09-27T17:27:09.185941Z","iopub.status.idle":"2026-09-27T17:27:09.198601Z","shell.execute_reply.started":"2026-09-27T17:27:09.185897Z","shell.execute_reply":"2026-09-27T17:27:09.197238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 33: C-RAG FOR 30-MEMORY EVALUATION BANK\n# ============================================================\n\ndef evaluation_c_rag_retrieve(\n query,\n attention_state,\n workload_state,\n fatigue_state,\n engagement_state,\n top_k=5,\n eta=0.7\n):\n\n # --------------------------------------------------------\n # Query embedding\n # --------------------------------------------------------\n\n query_embedding = create_embeddings([query]).astype(\n \"float32\"\n )\n\n # --------------------------------------------------------\n # Semantic retrieval from the 30-memory evaluation bank\n # --------------------------------------------------------\n\n similarities, indices = evaluation_faiss_index.search(\n query_embedding,\n len(evaluation_memory_bank)\n )\n\n # --------------------------------------------------------\n # Full cognitive context\n # --------------------------------------------------------\n\n cognitive_context = calculate_full_cognitive_context(\n attention_state=attention_state,\n workload_state=workload_state,\n fatigue_state=fatigue_state,\n engagement_state=engagement_state\n )\n\n results = []\n\n # --------------------------------------------------------\n # Calculate C-RAG score\n # --------------------------------------------------------\n\n for similarity, index in zip(\n similarities[0],\n indices[0]\n ):\n\n memory = evaluation_memory_bank[index]\n\n # Evaluation memories don't necessarily contain\n # importance/value fields, so use the same baseline\n # values for all memories.\n importance = 0.8\n value = 0.8\n\n memory_strength = memory_score(\n importance=importance,\n time_elapsed=2,\n cognitive_context=cognitive_context,\n value=value\n )\n\n c_rag_score = (\n eta * float(similarity)\n + (1 - eta) * memory_strength\n )\n\n results.append({\n \"id\": memory[\"id\"],\n \"topic\": memory[\"topic\"],\n \"text\": memory[\"text\"],\n \"semantic_similarity\": round(\n float(similarity),\n 4\n ),\n \"memory_score\": round(\n memory_strength,\n 4\n ),\n \"c_rag_score\": round(\n c_rag_score,\n 4\n )\n })\n\n # --------------------------------------------------------\n # Rank by C-RAG score\n # --------------------------------------------------------\n\n results.sort(\n key=lambda x: x[\"c_rag_score\"],\n reverse=True\n )\n\n return results[:top_k]\n\n\nprint(\"Evaluation C-RAG function created successfully.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:27:42.205415Z","iopub.execute_input":"2026-09-27T17:27:42.208263Z","iopub.status.idle":"2026-09-27T17:27:42.231096Z","shell.execute_reply.started":"2026-09-27T17:27:42.208173Z","shell.execute_reply":"2026-09-27T17:27:42.229487Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 34: CORRECTED ACME C-RAG EVALUATION\n# ============================================================\n\neval_attention = \"Focused\"\neval_workload = \"High\"\neval_fatigue = \"Non-drowsy\"\neval_engagement = \"Engaged\"\n\neval_cognitive_context = calculate_full_cognitive_context(\n attention_state=eval_attention,\n workload_state=eval_workload,\n fatigue_state=eval_fatigue,\n engagement_state=eval_engagement\n)\n\nprint(\"=\" * 70)\nprint(\"CORRECTED ACME C-RAG EVALUATION\")\nprint(\"=\" * 70)\n\nprint(\"Cognitive Context C:\", eval_cognitive_context)\n\nupdated_ranks = []\n\nfor item in evaluation_queries_expanded:\n\n query_text = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n results = evaluation_c_rag_retrieve(\n query=query_text,\n attention_state=eval_attention,\n workload_state=eval_workload,\n fatigue_state=eval_fatigue,\n engagement_state=eval_engagement,\n top_k=5,\n eta=0.7\n )\n\n retrieved_ids = [\n result[\"id\"]\n for result in results\n ]\n\n if relevant_id in retrieved_ids:\n rank = retrieved_ids.index(relevant_id) + 1\n else:\n rank = 6\n\n updated_ranks.append(rank)\n\n print(\"\\nQuery:\", query_text)\n print(\"Relevant:\", relevant_id)\n print(\"Retrieved:\", retrieved_ids)\n print(\"Rank:\", rank)\n\n\n# ------------------------------------------------------------\n# Metrics\n# ------------------------------------------------------------\n\nmrr = np.mean([\n 1 / rank\n for rank in updated_ranks\n])\n\nmean_rank = np.mean(updated_ranks)\n\nrecall_at_5 = np.mean([\n rank <= 5\n for rank in updated_ranks\n])\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"FINAL UPDATED ACME RESULTS\")\nprint(\"=\" * 70)\n\nprint(\"Ranks:\", updated_ranks)\nprint(f\"MRR : {mrr:.4f}\")\nprint(f\"Mean Rank: {mean_rank:.4f}\")\nprint(f\"Recall@5 : {recall_at_5:.4f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:28:19.772102Z","iopub.execute_input":"2026-09-27T17:28:19.773195Z","iopub.status.idle":"2026-09-27T17:28:19.871701Z","shell.execute_reply.started":"2026-09-27T17:28:19.773154Z","shell.execute_reply":"2026-09-27T17:28:19.870174Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 35: COGNITIVE-STATE SENSITIVITY EXPERIMENT\n# ============================================================\n\ncognitive_conditions = [\n {\n \"name\": \"Focused + Low Workload + Non-drowsy + Engaged\",\n \"attention\": \"Focused\",\n \"workload\": \"Low\",\n \"fatigue\": \"Non-drowsy\",\n \"engagement\": \"Engaged\"\n },\n {\n \"name\": \"Focused + High Workload + Non-drowsy + Engaged\",\n \"attention\": \"Focused\",\n \"workload\": \"High\",\n \"fatigue\": \"Non-drowsy\",\n \"engagement\": \"Engaged\"\n },\n {\n \"name\": \"Unfocused + High Workload + Drowsy + Not Engaged\",\n \"attention\": \"Unfocused\",\n \"workload\": \"High\",\n \"fatigue\": \"Drowsy\",\n \"engagement\": \"Not Engaged\"\n },\n {\n \"name\": \"Drowsy + High Workload\",\n \"attention\": \"Drowsy\",\n \"workload\": \"High\",\n \"fatigue\": \"Drowsy\",\n \"engagement\": \"Not Engaged\"\n }\n]\n\nsensitivity_results = []\n\n# Use one representative query\nsensitivity_query = (\n \"I need to revise recursion because I keep getting it wrong.\"\n)\n\nprint(\"=\" * 75)\nprint(\"ACME COGNITIVE-STATE SENSITIVITY\")\nprint(\"=\" * 75)\n\nfor condition in cognitive_conditions:\n\n C = calculate_full_cognitive_context(\n attention_state=condition[\"attention\"],\n workload_state=condition[\"workload\"],\n fatigue_state=condition[\"fatigue\"],\n engagement_state=condition[\"engagement\"]\n )\n\n results = evaluation_c_rag_retrieve(\n query=sensitivity_query,\n attention_state=condition[\"attention\"],\n workload_state=condition[\"workload\"],\n fatigue_state=condition[\"fatigue\"],\n engagement_state=condition[\"engagement\"],\n top_k=5,\n eta=0.7\n )\n\n top_result = results[0]\n\n sensitivity_results.append({\n \"condition\": condition[\"name\"],\n \"attention\": condition[\"attention\"],\n \"workload\": condition[\"workload\"],\n \"fatigue\": condition[\"fatigue\"],\n \"engagement\": condition[\"engagement\"],\n \"cognitive_context\": C,\n \"top_memory\": top_result[\"id\"],\n \"top_memory_topic\": top_result[\"topic\"],\n \"top_c_rag_score\": top_result[\"c_rag_score\"]\n })\n\n print(\"\\nCondition:\", condition[\"name\"])\n print(\"Cognitive Context:\", C)\n print(\"Top Memory:\", top_result[\"id\"])\n print(\"Topic:\", top_result[\"topic\"])\n print(\"C-RAG Score:\", top_result[\"c_rag_score\"])\n\n\n# ------------------------------------------------------------\n# Convert to DataFrame\n# ------------------------------------------------------------\n\ncognitive_sensitivity_df = pd.DataFrame(\n sensitivity_results\n)\n\nprint(\"\\n\" + \"=\" * 75)\nprint(\"SENSITIVITY SUMMARY\")\nprint(\"=\" * 75)\n\ndisplay(cognitive_sensitivity_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:30:03.286328Z","iopub.execute_input":"2026-09-27T17:30:03.288334Z","iopub.status.idle":"2026-09-27T17:30:03.424618Z","shell.execute_reply.started":"2026-09-27T17:30:03.288285Z","shell.execute_reply":"2026-09-27T17:30:03.423628Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save updated cognitive sensitivity results\n\ncognitive_sensitivity_df.to_csv(\n \"ACME_updated_cognitive_sensitivity.csv\",\n index=False\n)\n\nprint(\"Saved:\")\nprint(\"ACME_updated_cognitive_sensitivity.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:30:44.351655Z","iopub.execute_input":"2026-09-27T17:30:44.352676Z","iopub.status.idle":"2026-09-27T17:30:44.36941Z","shell.execute_reply.started":"2026-09-27T17:30:44.352623Z","shell.execute_reply":"2026-09-27T17:30:44.368233Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 36: FINAL RETRIEVAL COMPARISON\n# ============================================================\n\ndef calculate_retrieval_metrics(ranks):\n mrr = np.mean([1 / r for r in ranks])\n mean_rank = np.mean(ranks)\n recall_at_5 = np.mean([r <= 5 for r in ranks])\n\n return {\n \"MRR\": round(mrr, 4),\n \"Mean Rank\": round(mean_rank, 4),\n \"Recall@5\": round(recall_at_5, 4)\n }\n\n\n# ------------------------------------------------------------\n# Containers\n# ------------------------------------------------------------\n\nsemantic_ranks = []\nrecency_ranks = []\nacme_ranks = []\n\n\n# ------------------------------------------------------------\n# Evaluate all methods on the SAME queries\n# ------------------------------------------------------------\n\nfor item in evaluation_queries_expanded:\n\n query_text = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n # --------------------------------------------------------\n # 1. Semantic retrieval\n # --------------------------------------------------------\n\n semantic_results = semantic_retrieval(\n query_text,\n top_k=len(evaluation_memory_bank)\n )\n\n semantic_ids = [\n r[\"id\"] for r in semantic_results\n ]\n\n if relevant_id in semantic_ids:\n semantic_rank = semantic_ids.index(relevant_id) + 1\n else:\n semantic_rank = len(evaluation_memory_bank) + 1\n\n semantic_ranks.append(semantic_rank)\n\n\n # --------------------------------------------------------\n # 2. Semantic + Recency\n # --------------------------------------------------------\n\n recency_results = semantic_recency_retrieval(\n query_text,\n top_k=len(evaluation_memory_bank)\n )\n\n recency_ids = [\n r[\"id\"] for r in recency_results\n ]\n\n if relevant_id in recency_ids:\n recency_rank = recency_ids.index(relevant_id) + 1\n else:\n recency_rank = len(evaluation_memory_bank) + 1\n\n recency_ranks.append(recency_rank)\n\n\n # --------------------------------------------------------\n # 3. Updated ACME C-RAG\n # --------------------------------------------------------\n\n acme_results = evaluation_c_rag_retrieve(\n query=query_text,\n attention_state=\"Focused\",\n workload_state=\"High\",\n fatigue_state=\"Non-drowsy\",\n engagement_state=\"Engaged\",\n top_k=len(evaluation_memory_bank),\n eta=0.7\n )\n\n acme_ids = [\n r[\"id\"] for r in acme_results\n ]\n\n if relevant_id in acme_ids:\n acme_rank = acme_ids.index(relevant_id) + 1\n else:\n acme_rank = len(evaluation_memory_bank) + 1\n\n acme_ranks.append(acme_rank)\n\n\n# ------------------------------------------------------------\n# Calculate metrics\n# ------------------------------------------------------------\n\nsemantic_metrics = calculate_retrieval_metrics(\n semantic_ranks\n)\n\nrecency_metrics = calculate_retrieval_metrics(\n recency_ranks\n)\n\nacme_metrics = calculate_retrieval_metrics(\n acme_ranks\n)\n\n\n# ------------------------------------------------------------\n# Display\n# ------------------------------------------------------------\n\ncomparison_df = pd.DataFrame([\n {\n \"Method\": \"Semantic Retrieval\",\n **semantic_metrics\n },\n {\n \"Method\": \"Semantic + Recency\",\n **recency_metrics\n },\n {\n \"Method\": \"ACME C-RAG\",\n **acme_metrics\n }\n])\n\nprint(\"=\" * 70)\nprint(\"FINAL RETRIEVAL COMPARISON\")\nprint(\"=\" * 70)\n\nprint(\"\\nSemantic ranks:\", semantic_ranks)\nprint(\"Recency ranks :\", recency_ranks)\nprint(\"ACME ranks :\", acme_ranks)\n\ndisplay(comparison_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:31:18.027775Z","iopub.execute_input":"2026-09-27T17:31:18.028773Z","iopub.status.idle":"2026-09-27T17:31:18.221196Z","shell.execute_reply.started":"2026-09-27T17:31:18.028731Z","shell.execute_reply":"2026-09-27T17:31:18.219824Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 37: CORRECT BASELINE vs ACME EVALUATION\n# Same 30-memory bank + same 5 queries\n# ============================================================\n\ndef semantic_eval_retrieve(query, top_k=30):\n query_embedding = create_embeddings([query]).astype(\"float32\")\n\n similarities = np.dot(\n evaluation_embeddings,\n query_embedding[0]\n )\n\n ranked_indices = np.argsort(similarities)[::-1][:top_k]\n\n results = []\n\n for idx in ranked_indices:\n results.append({\n \"id\": evaluation_memory_bank[idx][\"id\"],\n \"text\": evaluation_memory_bank[idx][\"text\"],\n \"topic\": evaluation_memory_bank[idx][\"topic\"],\n \"similarity\": float(similarities[idx])\n })\n\n return results\n\n\ndef semantic_recency_eval_retrieve(query, top_k=30):\n semantic_results = semantic_eval_retrieve(\n query,\n top_k=len(evaluation_memory_bank)\n )\n\n results = []\n\n for item in semantic_results:\n\n # Same fixed recency assumption used by evaluation C-RAG\n recency = recency_score(\n time_elapsed=2,\n decay_rate=0.05\n )\n\n # Baseline score:\n # semantic similarity + recency\n score = (\n 0.7 * item[\"similarity\"]\n + 0.3 * recency\n )\n\n results.append({\n **item,\n \"recency\": recency,\n \"score\": score\n })\n\n results.sort(\n key=lambda x: x[\"score\"],\n reverse=True\n )\n\n return results[:top_k]\n\n\n# ------------------------------------------------------------\n# Evaluate\n# ------------------------------------------------------------\n\nsemantic_ranks = []\nrecency_ranks = []\nacme_ranks = []\n\nfor item in evaluation_queries_expanded:\n\n query = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n # -----------------------------\n # Semantic\n # -----------------------------\n\n semantic_results = semantic_eval_retrieve(\n query,\n top_k=30\n )\n\n semantic_ids = [\n x[\"id\"] for x in semantic_results\n ]\n\n semantic_rank = semantic_ids.index(relevant_id) + 1\n semantic_ranks.append(semantic_rank)\n\n\n # -----------------------------\n # Semantic + Recency\n # -----------------------------\n\n recency_results = semantic_recency_eval_retrieve(\n query,\n top_k=30\n )\n\n recency_ids = [\n x[\"id\"] for x in recency_results\n ]\n\n recency_rank = recency_ids.index(relevant_id) + 1\n recency_ranks.append(recency_rank)\n\n\n # -----------------------------\n # ACME C-RAG\n # -----------------------------\n\n acme_results = evaluation_c_rag_retrieve(\n query=query,\n attention_state=\"Focused\",\n workload_state=\"High\",\n fatigue_state=\"Non-drowsy\",\n engagement_state=\"Engaged\",\n top_k=30,\n eta=0.7\n )\n\n acme_ids = [\n x[\"id\"] for x in acme_results\n ]\n\n acme_rank = acme_ids.index(relevant_id) + 1\n acme_ranks.append(acme_rank)\n\n\n# ------------------------------------------------------------\n# Metrics\n# ------------------------------------------------------------\n\nsemantic_metrics = calculate_retrieval_metrics(\n semantic_ranks\n)\n\nrecency_metrics = calculate_retrieval_metrics(\n recency_ranks\n)\n\nacme_metrics = calculate_retrieval_metrics(\n acme_ranks\n)\n\n\ncomparison_df = pd.DataFrame([\n {\n \"Method\": \"Semantic Retrieval\",\n **semantic_metrics\n },\n {\n \"Method\": \"Semantic + Recency\",\n **recency_metrics\n },\n {\n \"Method\": \"ACME C-RAG\",\n **acme_metrics\n }\n])\n\nprint(\"=\" * 70)\nprint(\"CORRECTED RETRIEVAL COMPARISON\")\nprint(\"=\" * 70)\n\nprint(\"\\nSemantic ranks:\", semantic_ranks)\nprint(\"Recency ranks :\", recency_ranks)\nprint(\"ACME ranks :\", acme_ranks)\n\ndisplay(comparison_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:32:06.376922Z","iopub.execute_input":"2026-09-27T17:32:06.380737Z","iopub.status.idle":"2026-09-27T17:32:06.633891Z","shell.execute_reply.started":"2026-09-27T17:32:06.380656Z","shell.execute_reply":"2026-09-27T17:32:06.632872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 38: CHALLENGING RETRIEVAL BENCHMARK\n# ============================================================\n\n# We deliberately create semantically similar memories so that\n# semantic similarity alone is not guaranteed to identify the\n# correct memory at rank 1.\n\nchallenging_memory_bank = [\n\n # ---------------- PYTHON ----------------\n {\n \"id\": \"C01\",\n \"text\": \"The learner understands Python loops but needs more practice with nested for-loops.\",\n \"topic\": \"Python loops\",\n \"importance\": 0.80,\n \"time_elapsed\": 2,\n \"value\": 0.80\n },\n {\n \"id\": \"C02\",\n \"text\": \"The learner is revising Python while loops and struggles with loop termination conditions.\",\n \"topic\": \"Python loops\",\n \"importance\": 0.75,\n \"time_elapsed\": 8,\n \"value\": 0.70\n },\n {\n \"id\": \"C03\",\n \"text\": \"The learner can solve basic Python for-loop problems and understands iteration.\",\n \"topic\": \"Python loops\",\n \"importance\": 0.90,\n \"time_elapsed\": 1,\n \"value\": 0.90\n },\n\n # ---------------- RECURSION ----------------\n {\n \"id\": \"C04\",\n \"text\": \"The learner finds recursion difficult and needs to understand base cases.\",\n \"topic\": \"Recursion\",\n \"importance\": 0.95,\n \"time_elapsed\": 2,\n \"value\": 0.90\n },\n {\n \"id\": \"C05\",\n \"text\": \"The learner practiced recursive factorial and Fibonacci problems.\",\n \"topic\": \"Recursion\",\n \"importance\": 0.70,\n \"time_elapsed\": 12,\n \"value\": 0.60\n },\n {\n \"id\": \"C06\",\n \"text\": \"The learner understands recursive function calls but struggles to trace the call stack.\",\n \"topic\": \"Recursion\",\n \"importance\": 0.85,\n \"time_elapsed\": 5,\n \"value\": 0.75\n },\n\n # ---------------- JAVASCRIPT ----------------\n {\n \"id\": \"C07\",\n \"text\": \"The learner is learning JavaScript ES6 arrow functions and their syntax.\",\n \"topic\": \"JavaScript ES6\",\n \"importance\": 0.90,\n \"time_elapsed\": 2,\n \"value\": 0.85\n },\n {\n \"id\": \"C08\",\n \"text\": \"The learner needs more practice using JavaScript map and filter methods.\",\n \"topic\": \"JavaScript ES6\",\n \"importance\": 0.80,\n \"time_elapsed\": 7,\n \"value\": 0.70\n },\n {\n \"id\": \"C09\",\n \"text\": \"The learner is revising JavaScript destructuring and spread syntax.\",\n \"topic\": \"JavaScript ES6\",\n \"importance\": 0.65,\n \"time_elapsed\": 15,\n \"value\": 0.55\n },\n\n # ---------------- SQL ----------------\n {\n \"id\": \"C10\",\n \"text\": \"The learner struggles to understand SQL INNER JOIN and how matching rows are selected.\",\n \"topic\": \"SQL joins\",\n \"importance\": 0.95,\n \"time_elapsed\": 2,\n \"value\": 0.90\n },\n {\n \"id\": \"C11\",\n \"text\": \"The learner is practicing SQL LEFT JOIN and comparing unmatched rows.\",\n \"topic\": \"SQL joins\",\n \"importance\": 0.80,\n \"time_elapsed\": 6,\n \"value\": 0.75\n },\n {\n \"id\": \"C12\",\n \"text\": \"The learner understands basic SQL joins but needs revision of GROUP BY and aggregate queries.\",\n \"topic\": \"SQL joins\",\n \"importance\": 0.70,\n \"time_elapsed\": 11,\n \"value\": 0.65\n },\n\n # ---------------- HTML/CSS ----------------\n {\n \"id\": \"C13\",\n \"text\": \"The learner needs to revise HTML semantic elements and basic document structure.\",\n \"topic\": \"HTML CSS\",\n \"importance\": 0.85,\n \"time_elapsed\": 3,\n \"value\": 0.80\n },\n {\n \"id\": \"C14\",\n \"text\": \"The learner struggles with CSS flexbox alignment and spacing.\",\n \"topic\": \"HTML CSS\",\n \"importance\": 0.90,\n \"time_elapsed\": 4,\n \"value\": 0.85\n },\n {\n \"id\": \"C15\",\n \"text\": \"The learner is revising CSS selectors, specificity, and basic styling.\",\n \"topic\": \"HTML CSS\",\n \"importance\": 0.75,\n \"time_elapsed\": 10,\n \"value\": 0.65\n },\n\n # ---------------- DSA ----------------\n {\n \"id\": \"C16\",\n \"text\": \"The learner is struggling with array traversal and wants more DSA practice.\",\n \"topic\": \"DSA arrays\",\n \"importance\": 0.85,\n \"time_elapsed\": 3,\n \"value\": 0.80\n },\n {\n \"id\": \"C17\",\n \"text\": \"The learner needs revision of prefix sums and range-sum queries.\",\n \"topic\": \"DSA arrays\",\n \"importance\": 0.90,\n \"time_elapsed\": 2,\n \"value\": 0.90\n },\n {\n \"id\": \"C18\",\n \"text\": \"The learner is practicing sliding-window array problems.\",\n \"topic\": \"DSA arrays\",\n \"importance\": 0.70,\n \"time_elapsed\": 9,\n \"value\": 0.65\n },\n\n # ---------------- REACT ----------------\n {\n \"id\": \"C19\",\n \"text\": \"The learner is learning React components and props.\",\n \"topic\": \"React\",\n \"importance\": 0.80,\n \"time_elapsed\": 3,\n \"value\": 0.80\n },\n {\n \"id\": \"C20\",\n \"text\": \"The learner struggles with React useState and state updates.\",\n \"topic\": \"React\",\n \"importance\": 0.90,\n \"time_elapsed\": 4,\n \"value\": 0.85\n },\n {\n \"id\": \"C21\",\n \"text\": \"The learner is revising React useEffect and component lifecycle behavior.\",\n \"topic\": \"React\",\n \"importance\": 0.85,\n \"time_elapsed\": 8,\n \"value\": 0.70\n },\n\n # ---------------- MONGODB ----------------\n {\n \"id\": \"C22\",\n \"text\": \"The learner is learning MongoDB CRUD operations.\",\n \"topic\": \"MongoDB\",\n \"importance\": 0.80,\n \"time_elapsed\": 4,\n \"value\": 0.80\n },\n {\n \"id\": \"C23\",\n \"text\": \"The learner struggles with MongoDB aggregation pipelines.\",\n \"topic\": \"MongoDB\",\n \"importance\": 0.90,\n \"time_elapsed\": 5,\n \"value\": 0.85\n },\n {\n \"id\": \"C24\",\n \"text\": \"The learner is revising MongoDB queries and document filtering.\",\n \"topic\": \"MongoDB\",\n \"importance\": 0.75,\n \"time_elapsed\": 13,\n \"value\": 0.60\n },\n\n # ---------------- DEVOPS ----------------\n {\n \"id\": \"C25\",\n \"text\": \"The learner is beginning DevOps and wants to understand CI/CD pipelines.\",\n \"topic\": \"DevOps\",\n \"importance\": 0.90,\n \"time_elapsed\": 2,\n \"value\": 0.85\n },\n {\n \"id\": \"C26\",\n \"text\": \"The learner needs practice with Docker containers and images.\",\n \"topic\": \"DevOps\",\n \"importance\": 0.85,\n \"time_elapsed\": 6,\n \"value\": 0.75\n },\n {\n \"id\": \"C27\",\n \"text\": \"The learner is revising Git branching and merge workflows.\",\n \"topic\": \"DevOps\",\n \"importance\": 0.75,\n \"time_elapsed\": 10,\n \"value\": 0.65\n },\n\n # ---------------- MACHINE LEARNING ----------------\n {\n \"id\": \"C28\",\n \"text\": \"The learner is revising supervised learning and classification algorithms.\",\n \"topic\": \"Machine Learning\",\n \"importance\": 0.85,\n \"time_elapsed\": 3,\n \"value\": 0.80\n },\n {\n \"id\": \"C29\",\n \"text\": \"The learner struggles to understand model overfitting and regularization.\",\n \"topic\": \"Machine Learning\",\n \"importance\": 0.95,\n \"time_elapsed\": 4,\n \"value\": 0.90\n },\n {\n \"id\": \"C30\",\n \"text\": \"The learner is practicing feature scaling and preprocessing techniques.\",\n \"topic\": \"Machine Learning\",\n \"importance\": 0.75,\n \"time_elapsed\": 12,\n \"value\": 0.60\n }\n]\n\n\n# ------------------------------------------------------------\n# Challenging evaluation queries\n# ------------------------------------------------------------\n\nchallenging_queries = [\n {\n \"query\": \"I keep getting recursion wrong. I need to understand the base case.\",\n \"relevant_memory\": \"C04\"\n },\n {\n \"query\": \"I need to practice Python loops again.\",\n \"relevant_memory\": \"C03\"\n },\n {\n \"query\": \"I am struggling with JavaScript ES6 functions.\",\n \"relevant_memory\": \"C07\"\n },\n {\n \"query\": \"SQL joins are confusing me, especially matching rows.\",\n \"relevant_memory\": \"C10\"\n },\n {\n \"query\": \"I need to revise HTML basics and document structure.\",\n \"relevant_memory\": \"C13\"\n },\n {\n \"query\": \"I need help with prefix sums and range queries.\",\n \"relevant_memory\": \"C17\"\n },\n {\n \"query\": \"I don't understand React state updates.\",\n \"relevant_memory\": \"C20\"\n },\n {\n \"query\": \"I am confused about MongoDB aggregation.\",\n \"relevant_memory\": \"C23\"\n },\n {\n \"query\": \"I want to learn about CI/CD pipelines.\",\n \"relevant_memory\": \"C25\"\n },\n {\n \"query\": \"Why does my machine learning model overfit?\",\n \"relevant_memory\": \"C29\"\n }\n]\n\n\nprint(\"Challenging memory bank:\", len(challenging_memory_bank))\nprint(\"Challenging queries:\", len(challenging_queries))\nprint(\"\\nSample memory:\")\nprint(challenging_memory_bank[3])\n\nprint(\"\\nSample query:\")\nprint(challenging_queries[0])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:34:07.124668Z","iopub.execute_input":"2026-09-27T17:34:07.12669Z","iopub.status.idle":"2026-09-27T17:34:07.163862Z","shell.execute_reply.started":"2026-09-27T17:34:07.126639Z","shell.execute_reply":"2026-09-27T17:34:07.162258Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 39: BUILD CHALLENGING RETRIEVAL INDEX\n# ============================================================\n\nchallenging_texts = [\n memory[\"text\"]\n for memory in challenging_memory_bank\n]\n\n# Generate MiniLM embeddings\nchallenging_embeddings = create_embeddings(\n challenging_texts\n)\n\n# Convert to float32 for FAISS\nchallenging_embeddings_faiss = np.asarray(\n challenging_embeddings,\n dtype=\"float32\"\n)\n\n# Create exact cosine-similarity index\n# Embeddings are already L2-normalized,\n# therefore inner product = cosine similarity.\nchallenging_faiss_index = faiss.IndexFlatIP(\n challenging_embeddings_faiss.shape[1]\n)\n\nchallenging_faiss_index.add(\n challenging_embeddings_faiss\n)\n\nprint(\"=\" * 70)\nprint(\"CHALLENGING RETRIEVAL INDEX\")\nprint(\"=\" * 70)\n\nprint(\"Embedding shape:\", challenging_embeddings.shape)\nprint(\"Embedding dimension:\", challenging_embeddings.shape[1])\nprint(\"FAISS index size:\", challenging_faiss_index.ntotal)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:34:59.862626Z","iopub.execute_input":"2026-09-27T17:34:59.863257Z","iopub.status.idle":"2026-09-27T17:35:00.310817Z","shell.execute_reply.started":"2026-09-27T17:34:59.863217Z","shell.execute_reply":"2026-09-27T17:35:00.309523Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 40: CHALLENGING RETRIEVAL COMPARISON\n# Semantic vs Semantic + Recency vs ACME C-RAG\n# ============================================================\n\ndef challenging_semantic_retrieve(query, top_k=30):\n\n query_embedding = create_embeddings([query]).astype(\"float32\")\n\n similarities, indices = challenging_faiss_index.search(\n query_embedding,\n top_k\n )\n\n results = []\n\n for similarity, idx in zip(\n similarities[0],\n indices[0]\n ):\n memory = challenging_memory_bank[idx]\n\n results.append({\n \"id\": memory[\"id\"],\n \"text\": memory[\"text\"],\n \"topic\": memory[\"topic\"],\n \"importance\": memory[\"importance\"],\n \"time_elapsed\": memory[\"time_elapsed\"],\n \"value\": memory[\"value\"],\n \"similarity\": float(similarity)\n })\n\n return results\n\n\ndef challenging_recency_retrieve(query, top_k=30):\n\n semantic_results = challenging_semantic_retrieve(\n query,\n top_k=30\n )\n\n results = []\n\n for item in semantic_results:\n\n recency = recency_score(\n item[\"time_elapsed\"]\n )\n\n # Semantic + recency baseline\n score = (\n 0.7 * item[\"similarity\"]\n + 0.3 * recency\n )\n\n results.append({\n **item,\n \"recency\": recency,\n \"score\": score\n })\n\n results.sort(\n key=lambda x: x[\"score\"],\n reverse=True\n )\n\n return results[:top_k]\n\n\ndef challenging_acme_retrieve(\n query,\n attention_state=\"Focused\",\n workload_state=\"High\",\n fatigue_state=\"Non-drowsy\",\n engagement_state=\"Engaged\",\n top_k=30,\n eta=0.7\n):\n\n semantic_results = challenging_semantic_retrieve(\n query,\n top_k=30\n )\n\n cognitive_context = calculate_full_cognitive_context(\n attention_state,\n workload_state,\n fatigue_state,\n engagement_state\n )\n\n results = []\n\n for item in semantic_results:\n\n memory_strength = memory_score(\n importance=item[\"importance\"],\n time_elapsed=item[\"time_elapsed\"],\n cognitive_context=cognitive_context,\n value=item[\"value\"]\n )\n\n c_rag_score = (\n eta * item[\"similarity\"]\n + (1 - eta) * memory_strength\n )\n\n results.append({\n **item,\n \"cognitive_context\": cognitive_context,\n \"memory_strength\": memory_strength,\n \"c_rag_score\": c_rag_score\n })\n\n results.sort(\n key=lambda x: x[\"c_rag_score\"],\n reverse=True\n )\n\n return results[:top_k]\n\n\n# ============================================================\n# Evaluate all three methods\n# ============================================================\n\nchallenging_semantic_ranks = []\nchallenging_recency_ranks = []\nchallenging_acme_ranks = []\n\nchallenging_detail_rows = []\n\n\nfor item in challenging_queries:\n\n query = item[\"query\"]\n relevant_id = item[\"relevant_memory\"]\n\n\n # --------------------------------------------------------\n # 1. Semantic retrieval\n # --------------------------------------------------------\n\n semantic_results = challenging_semantic_retrieve(\n query,\n top_k=30\n )\n\n semantic_ids = [\n x[\"id\"]\n for x in semantic_results\n ]\n\n semantic_rank = (\n semantic_ids.index(relevant_id) + 1\n )\n\n challenging_semantic_ranks.append(\n semantic_rank\n )\n\n\n # --------------------------------------------------------\n # 2. Semantic + Recency\n # --------------------------------------------------------\n\n recency_results = challenging_recency_retrieve(\n query,\n top_k=30\n )\n\n recency_ids = [\n x[\"id\"]\n for x in recency_results\n ]\n\n recency_rank = (\n recency_ids.index(relevant_id) + 1\n )\n\n challenging_recency_ranks.append(\n recency_rank\n )\n\n\n # --------------------------------------------------------\n # 3. ACME C-RAG\n # --------------------------------------------------------\n\n acme_results = challenging_acme_retrieve(\n query=query,\n attention_state=\"Focused\",\n workload_state=\"High\",\n fatigue_state=\"Non-drowsy\",\n engagement_state=\"Engaged\",\n top_k=30,\n eta=0.7\n )\n\n acme_ids = [\n x[\"id\"]\n for x in acme_results\n ]\n\n acme_rank = (\n acme_ids.index(relevant_id) + 1\n )\n\n challenging_acme_ranks.append(\n acme_rank\n )\n\n\n challenging_detail_rows.append({\n \"query\": query,\n \"relevant_memory\": relevant_id,\n \"semantic_rank\": semantic_rank,\n \"recency_rank\": recency_rank,\n \"acme_rank\": acme_rank\n })\n\n\n# ============================================================\n# Calculate metrics\n# ============================================================\n\nchallenging_semantic_metrics = calculate_retrieval_metrics(\n challenging_semantic_ranks\n)\n\nchallenging_recency_metrics = calculate_retrieval_metrics(\n challenging_recency_ranks\n)\n\nchallenging_acme_metrics = calculate_retrieval_metrics(\n challenging_acme_ranks\n)\n\n\n# ============================================================\n# Final comparison table\n# ============================================================\n\nchallenging_comparison_df = pd.DataFrame([\n {\n \"Method\": \"Semantic Retrieval\",\n **challenging_semantic_metrics\n },\n {\n \"Method\": \"Semantic + Recency\",\n **challenging_recency_metrics\n },\n {\n \"Method\": \"ACME C-RAG\",\n **challenging_acme_metrics\n }\n])\n\n\n# ============================================================\n# Query-level results\n# ============================================================\n\nchallenging_query_results_df = pd.DataFrame(\n challenging_detail_rows\n)\n\n\nprint(\"=\" * 70)\nprint(\"CHALLENGING RETRIEVAL COMPARISON\")\nprint(\"=\" * 70)\n\nprint(\"\\nSemantic ranks:\", challenging_semantic_ranks)\nprint(\"Recency ranks :\", challenging_recency_ranks)\nprint(\"ACME ranks :\", challenging_acme_ranks)\n\nprint(\"\\nQUERY-LEVEL RESULTS\")\ndisplay(challenging_query_results_df)\n\nprint(\"\\nOVERALL RESULTS\")\ndisplay(challenging_comparison_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:35:49.514189Z","iopub.execute_input":"2026-09-27T17:35:49.518021Z","iopub.status.idle":"2026-09-27T17:35:50.029923Z","shell.execute_reply.started":"2026-09-27T17:35:49.517932Z","shell.execute_reply":"2026-09-27T17:35:50.02872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 41: COGNITIVE CONDITION SENSITIVITY\n# Challenging 10-query retrieval benchmark\n# ============================================================\n\ncognitive_conditions = [\n {\n \"name\": \"Focused + Low + Non-drowsy + Engaged\",\n \"attention\": \"Focused\",\n \"workload\": \"Low\",\n \"fatigue\": \"Non-drowsy\",\n \"engagement\": \"Engaged\"\n },\n {\n \"name\": \"Focused + High + Non-drowsy + Engaged\",\n \"attention\": \"Focused\",\n \"workload\": \"High\",\n \"fatigue\": \"Non-drowsy\",\n \"engagement\": \"Engaged\"\n },\n {\n \"name\": \"Unfocused + High + Drowsy + Not Engaged\",\n \"attention\": \"Unfocused\",\n \"workload\": \"High\",\n \"fatigue\": \"Drowsy\",\n \"engagement\": \"Not Engaged\"\n },\n {\n \"name\": \"Drowsy + High + Drowsy + Not Engaged\",\n \"attention\": \"Drowsy\",\n \"workload\": \"High\",\n \"fatigue\": \"Drowsy\",\n \"engagement\": \"Not Engaged\"\n }\n]\n\n\nsensitivity_rows = []\n\n\nfor condition in cognitive_conditions:\n\n acme_ranks = []\n\n for item in challenging_queries:\n\n results = challenging_acme_retrieve(\n query=item[\"query\"],\n attention_state=condition[\"attention\"],\n workload_state=condition[\"workload\"],\n fatigue_state=condition[\"fatigue\"],\n engagement_state=condition[\"engagement\"],\n top_k=30,\n eta=0.7\n )\n\n result_ids = [\n x[\"id\"] for x in results\n ]\n\n rank = result_ids.index(\n item[\"relevant_memory\"]\n ) + 1\n\n acme_ranks.append(rank)\n\n\n metrics = calculate_retrieval_metrics(\n acme_ranks\n )\n\n cognitive_context = calculate_full_cognitive_context(\n condition[\"attention\"],\n condition[\"workload\"],\n condition[\"fatigue\"],\n condition[\"engagement\"]\n )\n\n\n sensitivity_rows.append({\n \"Condition\": condition[\"name\"],\n \"Cognitive Context\": cognitive_context,\n \"MRR\": metrics[\"MRR\"],\n \"Mean Rank\": metrics[\"Mean Rank\"],\n \"Recall@5\": metrics[\"Recall@5\"],\n \"Ranks\": acme_ranks\n })\n\n\ncognitive_benchmark_sensitivity_df = pd.DataFrame(\n sensitivity_rows\n)\n\n\nprint(\"=\" * 70)\nprint(\"ACME C-RAG COGNITIVE CONDITION SENSITIVITY\")\nprint(\"=\" * 70)\n\ndisplay(cognitive_benchmark_sensitivity_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:36:41.051471Z","iopub.execute_input":"2026-09-27T17:36:41.051901Z","iopub.status.idle":"2026-09-27T17:36:41.622505Z","shell.execute_reply.started":"2026-09-27T17:36:41.051867Z","shell.execute_reply":"2026-09-27T17:36:41.621392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 42: FINAL EEG CLASSIFICATION OUTPUTS\n# ============================================================\n\nfrom sklearn.metrics import (\n accuracy_score,\n precision_score,\n recall_score,\n f1_score,\n confusion_matrix,\n classification_report\n)\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\n\n# ============================================================\n# 1. Generate predictions\n# ============================================================\n\n# Attention\nattention_pred = selected_rf.predict(X_selected_test)\n\n# Workload\nworkload_pred = workload_model.predict(X_workload_rich_test)\n\n# Fatigue / drowsiness proxy\nfatigue_pred = fatigue_model.predict(X_fatigue_rich_test)\n\n# Engagement proxy\nengagement_pred = engagement_rf.predict(X_engagement_rich_test)\n\n\n# ============================================================\n# 2. Calculate metrics\n# ============================================================\n\nfinal_eeg_metrics = []\n\n\ndef add_metrics(\n task,\n y_true,\n y_pred\n):\n final_eeg_metrics.append({\n \"Task\": task,\n \"Accuracy\": accuracy_score(\n y_true,\n y_pred\n ),\n \"Precision_macro\": precision_score(\n y_true,\n y_pred,\n average=\"macro\",\n zero_division=0\n ),\n \"Recall_macro\": recall_score(\n y_true,\n y_pred,\n average=\"macro\",\n zero_division=0\n ),\n \"Macro_F1\": f1_score(\n y_true,\n y_pred,\n average=\"macro\",\n zero_division=0\n )\n })\n\n\n# Attention\nadd_metrics(\n \"Attention\",\n y_attention_test,\n attention_pred\n)\n\n# Workload\nadd_metrics(\n \"Workload\",\n y_workload_test,\n workload_pred\n)\n\n# Fatigue\nadd_metrics(\n \"Fatigue / Drowsiness Proxy\",\n y_fatigue_test,\n fatigue_pred\n)\n\n# Engagement\nadd_metrics(\n \"Engagement Proxy\",\n y_engagement_test,\n engagement_pred\n)\n\n\nfinal_eeg_metrics_df = pd.DataFrame(\n final_eeg_metrics\n)\n\n\n# ============================================================\n# 3. Display metrics\n# ============================================================\n\nprint(\"=\" * 80)\nprint(\"FINAL EEG CLASSIFICATION RESULTS\")\nprint(\"=\" * 80)\n\ndisplay(\n final_eeg_metrics_df.round(4)\n)\n\n\n# ============================================================\n# 4. Classification reports\n# ============================================================\n\nprint(\"\\n\" + \"=\" * 80)\nprint(\"ATTENTION CLASSIFICATION REPORT\")\nprint(\"=\" * 80)\n\nprint(\n classification_report(\n y_attention_test,\n attention_pred,\n zero_division=0\n )\n)\n\n\nprint(\"=\" * 80)\nprint(\"WORKLOAD CLASSIFICATION REPORT\")\nprint(\"=\" * 80)\n\nprint(\n classification_report(\n y_workload_test,\n workload_pred,\n zero_division=0\n )\n)\n\n\nprint(\"=\" * 80)\nprint(\"FATIGUE / DROWSINESS CLASSIFICATION REPORT\")\nprint(\"=\" * 80)\n\nprint(\n classification_report(\n y_fatigue_test,\n fatigue_pred,\n zero_division=0\n )\n)\n\n\nprint(\"=\" * 80)\nprint(\"ENGAGEMENT CLASSIFICATION REPORT\")\nprint(\"=\" * 80)\n\nprint(\n classification_report(\n y_engagement_test,\n engagement_pred,\n zero_division=0\n )\n)\n\n\n# ============================================================\n# 5. Confusion matrices\n# ============================================================\n\ncm_attention = confusion_matrix(\n y_attention_test,\n attention_pred\n)\n\ncm_workload = confusion_matrix(\n y_workload_test,\n workload_pred\n)\n\ncm_fatigue = confusion_matrix(\n y_fatigue_test,\n fatigue_pred\n)\n\ncm_engagement = confusion_matrix(\n y_engagement_test,\n engagement_pred\n)\n\n\n# ============================================================\n# 6. Display confusion matrices individually\n# ============================================================\n\nfigures = [\n (\n cm_attention,\n [\"Drowsy\", \"Focused\", \"Unfocused\"],\n \"Attention Confusion Matrix\"\n ),\n (\n cm_workload,\n [\"High\", \"Low\"],\n \"Workload Confusion Matrix\"\n ),\n (\n cm_fatigue,\n [\"Non-drowsy\", \"Drowsy\"],\n \"Fatigue / Drowsiness Confusion Matrix\"\n ),\n (\n cm_engagement,\n [\"Not Engaged\", \"Engaged\"],\n \"Engagement Confusion Matrix\"\n )\n]\n\n\nfor cm, labels, title in figures:\n\n plt.figure(figsize=(5, 4))\n\n plt.imshow(\n cm,\n interpolation=\"nearest\"\n )\n\n plt.title(title)\n plt.xlabel(\"Predicted\")\n plt.ylabel(\"Actual\")\n\n plt.xticks(\n range(len(labels)),\n labels,\n rotation=30\n )\n\n plt.yticks(\n range(len(labels)),\n labels\n )\n\n for i in range(cm.shape[0]):\n for j in range(cm.shape[1]):\n plt.text(\n j,\n i,\n cm[i, j],\n ha=\"center\",\n va=\"center\"\n )\n\n plt.tight_layout()\n plt.show()\n\n\n# ============================================================\n# 7. Save final EEG metrics\n# ============================================================\n\nfinal_eeg_metrics_df.to_csv(\n \"ACME_final_EEG_metrics.csv\",\n index=False\n)\n\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_EEG_metrics.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:40:59.461993Z","iopub.execute_input":"2026-09-27T17:40:59.467747Z","iopub.status.idle":"2026-09-27T17:41:00.058201Z","shell.execute_reply.started":"2026-09-27T17:40:59.467621Z","shell.execute_reply":"2026-09-27T17:41:00.056039Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 42: FINAL EEG CLASSIFICATION OUTPUTS - FIXED\n# ============================================================\n\nfrom sklearn.metrics import (\n accuracy_score,\n precision_score,\n recall_score,\n f1_score,\n confusion_matrix,\n classification_report\n)\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\n\n\n# ============================================================\n# 1. Generate predictions\n# ============================================================\n\nattention_pred = selected_rf.predict(X_selected_test)\n\nworkload_pred = workload_model.predict(X_workload_rich_test)\n\nfatigue_pred = fatigue_model.predict(X_fatigue_rich_test)\n\nengagement_pred = engagement_rf.predict(X_engagement_rich_test)\n\n\n# ============================================================\n# 2. Reconstruct the corresponding TRUE labels\n# ============================================================\n\n# ------------------------------------------------------------\n# Attention\n# ------------------------------------------------------------\n# X_selected_test was created from the recording-disjoint\n# attention test split.\n#\n# X_test / y_test correspond to the original attention split.\n# The selected feature matrix is a transformed version of it.\n\nattention_true = y_test\n\n\n# ------------------------------------------------------------\n# Workload\n# ------------------------------------------------------------\n\nworkload_true = y_workload_test\n\n\n# ------------------------------------------------------------\n# Fatigue / Drowsiness\n# ------------------------------------------------------------\n\nfatigue_true = (\n (all_labels[test_mask] == \"Drowsy\")\n).astype(int)\n\n\n# ------------------------------------------------------------\n# Engagement\n# ------------------------------------------------------------\n\nengagement_true = (\n all_labels[engagement_test_mask] == \"Unfocused\"\n).astype(int)\n\n\n# ============================================================\n# 3. Check dimensions BEFORE calculating metrics\n# ============================================================\n\nprint(\"=\" * 80)\nprint(\"LABEL / PREDICTION DIMENSION CHECK\")\nprint(\"=\" * 80)\n\nprint(\n \"Attention:\",\n len(attention_true),\n len(attention_pred)\n)\n\nprint(\n \"Workload:\",\n len(workload_true),\n len(workload_pred)\n)\n\nprint(\n \"Fatigue:\",\n len(fatigue_true),\n len(fatigue_pred)\n)\n\nprint(\n \"Engagement:\",\n len(engagement_true),\n len(engagement_pred)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:41:43.489007Z","iopub.execute_input":"2026-09-27T17:41:43.489497Z","iopub.status.idle":"2026-09-27T17:41:43.876624Z","shell.execute_reply.started":"2026-09-27T17:41:43.489462Z","shell.execute_reply":"2026-09-27T17:41:43.874966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find existing variables related to labels / test splits\n\nfor name in sorted(globals()):\n if any(key in name.lower() for key in [\n \"label\", \"target\", \"y_\", \"test_mask\", \"train_mask\"\n ]):\n obj = globals()[name]\n\n try:\n shape = obj.shape\n except:\n shape = \"\"\n\n print(f\"{name:40s} {type(obj).__name__:20s} {shape}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:42:23.695764Z","iopub.execute_input":"2026-09-27T17:42:23.697141Z","iopub.status.idle":"2026-09-27T17:42:23.719947Z","shell.execute_reply.started":"2026-09-27T17:42:23.697067Z","shell.execute_reply":"2026-09-27T17:42:23.718588Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 42 — FINAL EEG CLASSIFICATION OUTPUTS\n# ============================================================\n\nfrom sklearn.metrics import (\n accuracy_score,\n precision_score,\n recall_score,\n f1_score,\n classification_report,\n confusion_matrix\n)\n\n# ------------------------------------------------------------\n# 1. PREDICTIONS FROM LOCKED FINAL MODELS\n# ------------------------------------------------------------\n\nattention_pred = selected_rf.predict(X_selected_test)\n\nworkload_pred = workload_model.predict(X_workload_rich_test)\n\nfatigue_pred = fatigue_model.predict(X_fatigue_rich_test)\n\nengagement_pred = engagement_rf.predict(X_engagement_rich_test)\n\n\n# ------------------------------------------------------------\n# 2. TRUE LABELS\n# ------------------------------------------------------------\n\nattention_true = y_rich_test\n\nworkload_true = y_workload_rich_test\n\nfatigue_true = y_fatigue_test\n\nengagement_true = y_engagement_test\n\n\n# ------------------------------------------------------------\n# 3. SAFETY CHECK — DIMENSIONS\n# ------------------------------------------------------------\n\nprint(\"FINAL EEG DIMENSION CHECK\")\nprint(\"-\" * 50)\n\nprint(\"Attention :\", len(attention_true), len(attention_pred))\nprint(\"Workload :\", len(workload_true), len(workload_pred))\nprint(\"Fatigue :\", len(fatigue_true), len(fatigue_pred))\nprint(\"Engagement:\", len(engagement_true), len(engagement_pred))\n\n\n# ------------------------------------------------------------\n# 4. METRIC FUNCTION\n# ------------------------------------------------------------\n\ndef calculate_metrics(y_true, y_pred):\n return {\n \"Accuracy\": accuracy_score(y_true, y_pred),\n \"Precision_macro\": precision_score(\n y_true, y_pred, average=\"macro\", zero_division=0\n ),\n \"Recall_macro\": recall_score(\n y_true, y_pred, average=\"macro\", zero_division=0\n ),\n \"Macro_F1\": f1_score(\n y_true, y_pred, average=\"macro\", zero_division=0\n )\n }\n\n\n# ------------------------------------------------------------\n# 5. FINAL METRICS\n# ------------------------------------------------------------\n\nfinal_eeg_metrics = pd.DataFrame([\n {\n \"Task\": \"Attention\",\n **calculate_metrics(attention_true, attention_pred)\n },\n {\n \"Task\": \"Workload\",\n **calculate_metrics(workload_true, workload_pred)\n },\n {\n \"Task\": \"Fatigue/Drowsiness Proxy\",\n **calculate_metrics(fatigue_true, fatigue_pred)\n },\n {\n \"Task\": \"Engagement Proxy\",\n **calculate_metrics(engagement_true, engagement_pred)\n }\n])\n\nprint(\"\\nFINAL EEG METRICS\")\ndisplay(final_eeg_metrics.round(4))\n\n\n# ------------------------------------------------------------\n# 6. CLASSIFICATION REPORTS\n# ------------------------------------------------------------\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ATTENTION CLASSIFICATION REPORT\")\nprint(\"=\" * 70)\nprint(classification_report(attention_true, attention_pred, zero_division=0))\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"WORKLOAD CLASSIFICATION REPORT\")\nprint(\"=\" * 70)\nprint(classification_report(workload_true, workload_pred, zero_division=0))\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"FATIGUE / DROWSINESS CLASSIFICATION REPORT\")\nprint(\"=\" * 70)\nprint(classification_report(fatigue_true, fatigue_pred, zero_division=0))\n\nprint(\"\\n\" + \"=\" * 70)\nprint(\"ENGAGEMENT CLASSIFICATION REPORT\")\nprint(\"=\" * 70)\nprint(classification_report(engagement_true, engagement_pred, zero_division=0))\n\n\n# ------------------------------------------------------------\n# 7. CONFUSION MATRICES\n# ------------------------------------------------------------\n\ncm_attention = confusion_matrix(attention_true, attention_pred)\n\ncm_workload = confusion_matrix(workload_true, workload_pred)\n\ncm_fatigue = confusion_matrix(fatigue_true, fatigue_pred)\n\ncm_engagement = confusion_matrix(engagement_true, engagement_pred)\n\nprint(\"\\nCONFUSION MATRICES\")\nprint(\"-\" * 50)\n\nprint(\"\\nAttention:\")\nprint(cm_attention)\n\nprint(\"\\nWorkload:\")\nprint(cm_workload)\n\nprint(\"\\nFatigue/Drowsiness:\")\nprint(cm_fatigue)\n\nprint(\"\\nEngagement:\")\nprint(cm_engagement)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:43:14.742075Z","iopub.execute_input":"2026-09-27T17:43:14.744007Z","iopub.status.idle":"2026-09-27T17:43:15.265962Z","shell.execute_reply.started":"2026-09-27T17:43:14.743959Z","shell.execute_reply":"2026-09-27T17:43:15.264764Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 43 — SAVE FINAL EEG RESULTS + CONFUSION MATRICES\n# ============================================================\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n# ------------------------------------------------------------\n# 1. SAVE METRICS CSV\n# ------------------------------------------------------------\n\nfinal_eeg_metrics.to_csv(\n \"/kaggle/working/ACME_final_EEG_metrics.csv\",\n index=False\n)\n\nprint(\"Saved: ACME_final_EEG_metrics.csv\")\n\n\n# ------------------------------------------------------------\n# 2. CONFUSION MATRIX PLOT\n# ------------------------------------------------------------\n\nfig, axes = plt.subplots(2, 2, figsize=(12, 10))\n\nconfusion_data = [\n (\n cm_attention,\n [\"Drowsy\", \"Focused\", \"Unfocused\"],\n \"Attention\"\n ),\n (\n cm_workload,\n [\"High\", \"Low\"],\n \"Workload\"\n ),\n (\n cm_fatigue,\n [\"Non-drowsy\", \"Drowsy\"],\n \"Drowsiness / Fatigue Proxy\"\n ),\n (\n cm_engagement,\n [\"Not Engaged\", \"Engaged\"],\n \"Engagement Proxy\"\n )\n]\n\nfor ax, (cm, class_names, title) in zip(axes.flat, confusion_data):\n\n im = ax.imshow(cm, interpolation=\"nearest\")\n\n ax.set_title(title, fontsize=13, fontweight=\"bold\")\n ax.set_xlabel(\"Predicted Label\")\n ax.set_ylabel(\"True Label\")\n\n ax.set_xticks(range(len(class_names)))\n ax.set_yticks(range(len(class_names)))\n\n ax.set_xticklabels(class_names, rotation=30, ha=\"right\")\n ax.set_yticklabels(class_names)\n\n for i in range(cm.shape[0]):\n for j in range(cm.shape[1]):\n ax.text(\n j,\n i,\n str(cm[i, j]),\n ha=\"center\",\n va=\"center\",\n fontsize=12\n )\n\nplt.suptitle(\n \"ACME Final EEG Classification Confusion Matrices\",\n fontsize=16,\n fontweight=\"bold\"\n)\n\nplt.tight_layout()\n\nplt.savefig(\n \"/kaggle/working/ACME_final_EEG_confusion_matrices.png\",\n dpi=300,\n bbox_inches=\"tight\"\n)\n\nplt.show()\n\nprint(\"Saved: ACME_final_EEG_confusion_matrices.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:44:57.227859Z","iopub.execute_input":"2026-09-27T17:44:57.232176Z","iopub.status.idle":"2026-09-27T17:44:59.195293Z","shell.execute_reply.started":"2026-09-27T17:44:57.232054Z","shell.execute_reply":"2026-09-27T17:44:59.193662Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 44 — FINAL RETRIEVAL COMPARISON\n# ============================================================\n\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport numpy as np\n\n# Final challenging benchmark metrics\nfinal_retrieval_metrics = pd.DataFrame([\n {\n \"Method\": \"Semantic Retrieval\",\n \"MRR\": 0.95,\n \"Mean Rank\": 1.10,\n \"Recall@5\": 1.00\n },\n {\n \"Method\": \"Semantic + Recency\",\n \"MRR\": 0.95,\n \"Mean Rank\": 1.10,\n \"Recall@5\": 1.00\n },\n {\n \"Method\": \"ACME C-RAG\",\n \"MRR\": 1.00,\n \"Mean Rank\": 1.00,\n \"Recall@5\": 1.00\n }\n])\n\nprint(\"FINAL RETRIEVAL COMPARISON\")\ndisplay(final_retrieval_metrics)\n\n# Save CSV\nfinal_retrieval_metrics.to_csv(\n \"/kaggle/working/ACME_final_retrieval_comparison.csv\",\n index=False\n)\n\n# ------------------------------------------------------------\n# Plot MRR\n# ------------------------------------------------------------\n\nplt.figure(figsize=(9, 6))\n\nplt.bar(\n final_retrieval_metrics[\"Method\"],\n final_retrieval_metrics[\"MRR\"]\n)\n\nplt.ylabel(\"Mean Reciprocal Rank (MRR)\")\nplt.title(\"Retrieval Performance Comparison\")\nplt.ylim(0, 1.1)\n\nfor i, value in enumerate(final_retrieval_metrics[\"MRR\"]):\n plt.text(\n i,\n value + 0.02,\n f\"{value:.2f}\",\n ha=\"center\",\n fontweight=\"bold\"\n )\n\nplt.xticks(rotation=15)\nplt.tight_layout()\n\nplt.savefig(\n \"/kaggle/working/ACME_final_retrieval_MRR.png\",\n dpi=300,\n bbox_inches=\"tight\"\n)\n\nplt.show()\n\nprint(\"Saved:\")\nprint(\"ACME_final_retrieval_comparison.csv\")\nprint(\"ACME_final_retrieval_MRR.png\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:45:52.159032Z","iopub.execute_input":"2026-09-27T17:45:52.160156Z","iopub.status.idle":"2026-09-27T17:45:52.826004Z","shell.execute_reply.started":"2026-09-27T17:45:52.160085Z","shell.execute_reply":"2026-09-27T17:45:52.824911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 45 — SAVE FINAL QUERY-LEVEL RETRIEVAL RESULTS\n# ============================================================\n\n# Display the verified challenging benchmark results\nprint(\"FINAL CHALLENGING RETRIEVAL RESULTS\")\ndisplay(challenging_query_results_df)\n\n# Save\nchallenging_query_results_df.to_csv(\n \"/kaggle/working/ACME_final_query_level_retrieval.csv\",\n index=False\n)\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_query_level_retrieval.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:46:57.052592Z","iopub.execute_input":"2026-09-27T17:46:57.053213Z","iopub.status.idle":"2026-09-27T17:46:57.072324Z","shell.execute_reply.started":"2026-09-27T17:46:57.053175Z","shell.execute_reply":"2026-09-27T17:46:57.070906Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 46 — FINAL MEMORY LIFECYCLE OUTPUT\n# ============================================================\n\nprint(\"FINAL MEMORY LIFECYCLE\")\nprint(\"=\" * 70)\n\n# Recalculate lifecycle using the current ACME memory scores\nlifecycle_rows = []\n\nfor memory in learner_memories:\n\n # Use the current demonstration cognitive context\n cognitive_context = calculate_full_cognitive_context(\n \"Focused\",\n \"High\",\n \"Non-drowsy\",\n \"Engaged\"\n )\n\n score = memory_score(\n memory[\"importance\"],\n 2,\n cognitive_context,\n memory[\"value\"]\n )\n\n action = memory_lifecycle(score)\n\n lifecycle_rows.append({\n \"Memory_ID\": memory[\"id\"],\n \"Topic\": memory[\"topic\"],\n \"Importance\": memory[\"importance\"],\n \"Value\": memory[\"value\"],\n \"Cognitive_Context\": cognitive_context,\n \"Memory_Score\": score,\n \"Lifecycle_Action\": action\n })\n\nfinal_memory_lifecycle_df = pd.DataFrame(lifecycle_rows)\n\ndisplay(final_memory_lifecycle_df)\n\n\n# ------------------------------------------------------------\n# SAVE CSV\n# ------------------------------------------------------------\n\nfinal_memory_lifecycle_df.to_csv(\n \"/kaggle/working/ACME_final_memory_lifecycle.csv\",\n index=False\n)\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_memory_lifecycle.csv\")\n\n\n# ------------------------------------------------------------\n# SUMMARY\n# ------------------------------------------------------------\n\nprint(\"\\nLIFECYCLE SUMMARY\")\nprint(\"-\" * 50)\n\nprint(\n final_memory_lifecycle_df[\"Lifecycle_Action\"]\n .value_counts()\n)\n\nprint(\n \"\\nCognitive context used:\",\n cognitive_context\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:48:11.913543Z","iopub.execute_input":"2026-09-27T17:48:11.915311Z","iopub.status.idle":"2026-09-27T17:48:11.957496Z","shell.execute_reply.started":"2026-09-27T17:48:11.915263Z","shell.execute_reply":"2026-09-27T17:48:11.95557Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 47 — FINAL END-TO-END ACME C-RAG DEMONSTRATION\n# ============================================================\n\n# ------------------------------------------------------------\n# Query + cognitive state used for final demonstration\n# ------------------------------------------------------------\n\ndemo_query = \"I keep getting recursion wrong. Can you help me understand it?\"\n\ndemo_attention = \"Focused\"\ndemo_workload = \"High\"\ndemo_fatigue = \"Non-drowsy\"\ndemo_engagement = \"Engaged\"\n\ndemo_context = calculate_full_cognitive_context(\n demo_attention,\n demo_workload,\n demo_fatigue,\n demo_engagement\n)\n\n# ------------------------------------------------------------\n# C-RAG retrieval\n# ------------------------------------------------------------\n\ndemo_results = final_c_rag_retrieve(\n demo_query,\n demo_attention,\n demo_workload,\n demo_fatigue,\n demo_engagement,\n top_k=3\n)\n\n# ------------------------------------------------------------\n# Display retrieval results\n# ------------------------------------------------------------\n\ndemo_retrieval_df = pd.DataFrame([\n {\n \"Query\": demo_query,\n \"Attention\": demo_attention,\n \"Workload\": demo_workload,\n \"Fatigue\": demo_fatigue,\n \"Engagement\": demo_engagement,\n \"Cognitive_Context\": demo_context,\n \"Memory_ID\": item[\"id\"],\n \"Memory_Topic\": item[\"topic\"],\n \"Semantic_Similarity\": item[\"similarity\"],\n \"Memory_Strength\": item[\"memory_strength\"],\n \"C_RAG_Score\": item[\"c_rag_score\"]\n }\n for item in demo_results\n])\n\nprint(\"FINAL ACME C-RAG RETRIEVAL DEMONSTRATION\")\nprint(\"=\" * 80)\n\ndisplay(demo_retrieval_df.round(4))\n\n\n# ------------------------------------------------------------\n# Save retrieval demonstration\n# ------------------------------------------------------------\n\ndemo_retrieval_df.to_csv(\n \"/kaggle/working/ACME_final_C_RAG_demo.csv\",\n index=False\n)\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_C_RAG_demo.csv\")\n\n\n# ------------------------------------------------------------\n# Final summary\n# ------------------------------------------------------------\n\nprint(\"\\nEND-TO-END ACME SUMMARY\")\nprint(\"-\" * 50)\nprint(\"Query:\", demo_query)\nprint(\"Attention:\", demo_attention)\nprint(\"Workload:\", demo_workload)\nprint(\"Fatigue:\", demo_fatigue)\nprint(\"Engagement:\", demo_engagement)\nprint(\"Cognitive Context:\", demo_context)\n\nprint(\"\\nRetrieved memories:\")\nfor rank, item in enumerate(demo_results, start=1):\n print(\n f\"{rank}. {item['id']} | \"\n f\"{item['topic']} | \"\n f\"C-RAG Score = {item['c_rag_score']:.4f}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:49:00.04662Z","iopub.execute_input":"2026-09-27T17:49:00.047547Z","iopub.status.idle":"2026-09-27T17:49:00.15704Z","shell.execute_reply.started":"2026-09-27T17:49:00.047505Z","shell.execute_reply":"2026-09-27T17:49:00.155461Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect the exact structure returned by final_c_rag_retrieve()\n\nprint(demo_results[0])\nprint(\"\\nAvailable keys:\")\nprint(demo_results[0].keys())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:49:32.068088Z","iopub.execute_input":"2026-09-27T17:49:32.06848Z","iopub.status.idle":"2026-09-27T17:49:32.076273Z","shell.execute_reply.started":"2026-09-27T17:49:32.068448Z","shell.execute_reply":"2026-09-27T17:49:32.074673Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 47 — FINAL END-TO-END ACME C-RAG DEMONSTRATION\n# ============================================================\n\ndemo_query = \"I keep getting recursion wrong. Can you help me understand it?\"\n\ndemo_attention = \"Focused\"\ndemo_workload = \"High\"\ndemo_fatigue = \"Non-drowsy\"\ndemo_engagement = \"Engaged\"\n\n# Calculate cognitive context\ndemo_context = calculate_full_cognitive_context(\n demo_attention,\n demo_workload,\n demo_fatigue,\n demo_engagement\n)\n\n# ------------------------------------------------------------\n# C-RAG retrieval\n# ------------------------------------------------------------\n\ndemo_results = final_c_rag_retrieve(\n demo_query,\n demo_attention,\n demo_workload,\n demo_fatigue,\n demo_engagement,\n top_k=3\n)\n\n# ------------------------------------------------------------\n# Create final demonstration table\n# ------------------------------------------------------------\n\ndemo_retrieval_df = pd.DataFrame([\n {\n \"Query\": demo_query,\n \"Attention\": demo_attention,\n \"Workload\": demo_workload,\n \"Fatigue\": demo_fatigue,\n \"Engagement\": demo_engagement,\n \"Cognitive_Context\": demo_context,\n \"Memory_ID\": item[\"id\"],\n \"Memory_Topic\": item[\"topic\"],\n \"Semantic_Similarity\": item[\"semantic_similarity\"],\n \"Memory_Score\": item[\"memory_score\"],\n \"C_RAG_Score\": item[\"c_rag_score\"]\n }\n for item in demo_results\n])\n\nprint(\"FINAL ACME C-RAG RETRIEVAL DEMONSTRATION\")\nprint(\"=\" * 80)\n\ndisplay(demo_retrieval_df.round(4))\n\n\n# ------------------------------------------------------------\n# Save final C-RAG demonstration\n# ------------------------------------------------------------\n\ndemo_retrieval_df.to_csv(\n \"/kaggle/working/ACME_final_C_RAG_demo.csv\",\n index=False\n)\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_C_RAG_demo.csv\")\n\n\n# ------------------------------------------------------------\n# End-to-end summary\n# ------------------------------------------------------------\n\nprint(\"\\nEND-TO-END ACME SUMMARY\")\nprint(\"-\" * 60)\n\nprint(\"Query:\", demo_query)\nprint(\"Attention:\", demo_attention)\nprint(\"Workload:\", demo_workload)\nprint(\"Fatigue:\", demo_fatigue)\nprint(\"Engagement:\", demo_engagement)\nprint(\"Cognitive Context:\", demo_context)\n\nprint(\"\\nRetrieved memories:\")\n\nfor rank, item in enumerate(demo_results, start=1):\n print(\n f\"{rank}. {item['id']} | \"\n f\"{item['topic']} | \"\n f\"Semantic = {item['semantic_similarity']:.4f} | \"\n f\"Memory = {item['memory_score']:.4f} | \"\n f\"C-RAG = {item['c_rag_score']:.4f}\"\n )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:50:15.300796Z","iopub.execute_input":"2026-09-27T17:50:15.301225Z","iopub.status.idle":"2026-09-27T17:50:15.352799Z","shell.execute_reply.started":"2026-09-27T17:50:15.301188Z","shell.execute_reply":"2026-09-27T17:50:15.351645Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 48 — FINAL GEMMA PERSONALIZED RESPONSE\n# ============================================================\n\n# Use the top retrieved memories from the final C-RAG demo\nretrieved_memory_text = \"\\n\".join([\n f\"- {item['topic']}: {item['text']}\"\n for item in demo_results\n])\n\ngemma_prompt = f\"\"\"\nYou are a personalized AI learning assistant.\n\nLearner cognitive state:\n- Attention: {demo_attention}\n- Workload: {demo_workload}\n- Fatigue: {demo_fatigue}\n- Engagement: {demo_engagement}\n- Cognitive context: {demo_context}\n\nLearner query:\n{demo_query}\n\nRelevant learner memories:\n{retrieved_memory_text}\n\nUse the learner's previous memories to personalize your response.\nExplain recursion from the basics in a simple, supportive way.\nDo not mention EEG, cognitive scores, retrieval systems, or internal memory mechanisms.\nKeep the response concise and educational.\n\"\"\"\n\nprint(\"GENERATING GEMMA RESPONSE...\")\nprint(\"=\" * 80)\n\n# Generate using the already-loaded Gemma model\nresponse = generate_gemma_response(\n gemma_prompt,\n max_new_tokens=300\n)\n\nprint(response)\n\n\n# ------------------------------------------------------------\n# SAVE RESPONSE\n# ------------------------------------------------------------\n\nwith open(\n \"/kaggle/working/ACME_final_Gemma_response.txt\",\n \"w\",\n encoding=\"utf-8\"\n) as f:\n f.write(response)\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_Gemma_response.txt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:50:58.949002Z","iopub.execute_input":"2026-09-27T17:50:58.949895Z","iopub.status.idle":"2026-09-27T17:50:58.964723Z","shell.execute_reply.started":"2026-09-27T17:50:58.949854Z","shell.execute_reply":"2026-09-27T17:50:58.963196Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Find existing Gemma-related variables and functions\n\nfor name in sorted(globals()):\n if \"gemma\" in name.lower():\n obj = globals()[name]\n print(f\"{name:45s} -> {type(obj).__name__}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:51:33.277427Z","iopub.execute_input":"2026-09-27T17:51:33.278015Z","iopub.status.idle":"2026-09-27T17:51:33.294801Z","shell.execute_reply.started":"2026-09-27T17:51:33.277967Z","shell.execute_reply":"2026-09-27T17:51:33.29299Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 48 — FINAL GEMMA PERSONALIZED RESPONSE\n# ============================================================\n\nimport torch\n\n# ------------------------------------------------------------\n# Build prompt from final C-RAG results\n# ------------------------------------------------------------\n\nretrieved_memory_text = \"\\n\".join([\n f\"- {item['topic']}: {item['text']}\"\n for item in demo_results\n])\n\ngemma_prompt = f\"\"\"\nYou are a personalized AI learning assistant.\n\nLearner cognitive state:\n- Attention: {demo_attention}\n- Workload: {demo_workload}\n- Fatigue: {demo_fatigue}\n- Engagement: {demo_engagement}\n- Cognitive context: {demo_context}\n\nLearner query:\n{demo_query}\n\nRelevant learner memories:\n{retrieved_memory_text}\n\nUse these previous learner memories to personalize the response.\n\nExplain recursion from the basics in a simple and supportive way.\nGive a short example if useful.\nDo not mention EEG, cognitive scores, retrieval systems, or internal memory mechanisms.\nKeep the response concise and educational.\n\"\"\"\n\n# ------------------------------------------------------------\n# Tokenize\n# ------------------------------------------------------------\n\ninputs = gemma_tokenizer(\n gemma_prompt,\n return_tensors=\"pt\"\n)\n\n# Move tensors to model device\ninputs = {\n key: value.to(gemma_model.device)\n for key, value in inputs.items()\n}\n\n# ------------------------------------------------------------\n# Generate\n# ------------------------------------------------------------\n\nprint(\"GENERATING FINAL GEMMA RESPONSE...\")\nprint(\"=\" * 80)\n\nwith torch.no_grad():\n output_ids = gemma_model.generate(\n **inputs,\n max_new_tokens=300,\n do_sample=False\n )\n\n# Remove prompt tokens so we only decode the answer\ngenerated_ids = output_ids[:, inputs[\"input_ids\"].shape[1]:]\n\nresponse = gemma_tokenizer.decode(\n generated_ids[0],\n skip_special_tokens=True\n)\n\nprint(response)\n\n\n# ------------------------------------------------------------\n# SAVE RESPONSE\n# ------------------------------------------------------------\n\nwith open(\n \"/kaggle/working/ACME_final_Gemma_response.txt\",\n \"w\",\n encoding=\"utf-8\"\n) as f:\n f.write(response)\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_Gemma_response.txt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:52:07.159737Z","iopub.execute_input":"2026-09-27T17:52:07.160828Z","iopub.status.idle":"2026-09-27T17:52:07.178139Z","shell.execute_reply.started":"2026-09-27T17:52:07.160781Z","shell.execute_reply":"2026-09-27T17:52:07.176328Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Inspect the Gemma tokenizer API already loaded in the notebook\n\nprint(type(gemma_tokenizer))\nprint(\"\\nTokenizer methods:\")\nprint([x for x in dir(gemma_tokenizer) if not x.startswith(\"_\")][:80])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:52:57.182348Z","iopub.execute_input":"2026-09-27T17:52:57.183429Z","iopub.status.idle":"2026-09-27T17:52:57.191695Z","shell.execute_reply.started":"2026-09-27T17:52:57.183376Z","shell.execute_reply":"2026-09-27T17:52:57.190538Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ============================================================\n# STEP 48 — FINAL GEMMA RESPONSE\n# ============================================================\n\nimport torch\n\nretrieved_memory_text = \"\\n\".join([\n f\"- {item['topic']}: {item['text']}\"\n for item in demo_results\n])\n\ngemma_prompt = f\"\"\"\nYou are a personalized AI learning assistant.\n\nLearner cognitive state:\n- Attention: {demo_attention}\n- Workload: {demo_workload}\n- Fatigue: {demo_fatigue}\n- Engagement: {demo_engagement}\n- Cognitive context: {demo_context}\n\nLearner query:\n{demo_query}\n\nRelevant learner memories:\n{retrieved_memory_text}\n\nUse the relevant learner memories to personalize your response.\n\nExplain recursion from the basics in a simple and supportive way.\nGive a short example if useful.\nDo not mention EEG, cognitive scores, retrieval systems, or internal memory mechanisms.\nKeep the response concise and educational.\n\"\"\"\n\nprint(\"Generating final Gemma response...\")\nprint(\"=\" * 80)\n\n# Official Gemma PyTorch API:\n# generate(prompts, device, output_len, temperature, top_p, top_k)\n\nwith torch.no_grad():\n output = gemma_model.generate(\n [gemma_prompt],\n device=\"cpu\",\n output_len=200,\n temperature=0.7,\n top_p=0.9,\n top_k=40\n )\n\n# Official Gemma returns generated text\nresponse = output[0]\n\nprint(response)\n\n# Save final response\nwith open(\n \"/kaggle/working/ACME_final_Gemma_response.txt\",\n \"w\",\n encoding=\"utf-8\"\n) as f:\n f.write(response)\n\nprint(\"\\nSaved:\")\nprint(\"ACME_final_Gemma_response.txt\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-09-27T17:55:33.998608Z","iopub.execute_input":"2026-09-27T17:55:33.998968Z","iopub.status.idle":"2026-09-27T17:57:51.786975Z","shell.execute_reply.started":"2026-09-27T17:55:33.99894Z","shell.execute_reply":"2026-09-27T17:57:51.782584Z"}},"outputs":[],"execution_count":null}]}