{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.11.11"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":102335,"databundleVersionId":12518947},{"sourceType":"datasetVersion","sourceId":12584022,"datasetId":7895595,"databundleVersionId":13178625},{"sourceType":"datasetVersion","sourceId":12584285,"datasetId":7947949,"databundleVersionId":13178933},{"sourceType":"kernelVersion","sourceId":252638509}],"dockerImageVersionId":31041,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true},"papermill":{"default_parameters":{},"duration":20.184915,"end_time":"2025-06-21T19:01:48.289795","environment_variables":{},"exception":null,"input_path":"__notebook__.ipynb","output_path":"__notebook__.ipynb","parameters":{},"start_time":"2025-06-21T19:01:28.10488","version":"2.6.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# CMI - LSTM Classifier Starter Inference\n\nThis is the submission notebook from the following starter training notebook: [CMI - LSTM Classifier Starter](https://www.kaggle.com/code/morodertobias/cmi-lstm-classifier-starter/).\n\n**All comments welcome!**\n\n## Change Log\n- V4: Use updated notebook V2.\n- V6: Use updated notebook V3.\n- V8: Use updated notebook V4.\n- ","metadata":{"papermill":{"duration":0.004663,"end_time":"2025-06-21T19:01:32.987977","exception":false,"start_time":"2025-06-21T19:01:32.983314","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## Import & Settings","metadata":{"papermill":{"duration":0.003421,"end_time":"2025-06-21T19:01:32.995342","exception":false,"start_time":"2025-06-21T19:01:32.991921","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport pathlib\nimport numpy as np\nimport pandas as pd\nimport polars as pl\nimport matplotlib.pyplot as plt\nfrom tqdm.auto import tqdm\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport kaggle_evaluation.cmi_inference_server\nimport joblib\nfrom pathlib import Path","metadata":{"papermill":{"duration":8.325273,"end_time":"2025-06-21T19:01:41.324371","exception":false,"start_time":"2025-06-21T19:01:32.999098","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:36:07.308998Z","iopub.execute_input":"2025-07-26T19:36:07.309164Z","iopub.status.idle":"2025-07-26T19:36:07.329184Z","shell.execute_reply.started":"2025-07-26T19:36:07.309151Z","shell.execute_reply":"2025-07-26T19:36:07.328460Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from expanded_cmi_utility_scripts import *","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:35:57.804300Z","iopub.execute_input":"2025-07-26T19:35:57.804736Z","iopub.status.idle":"2025-07-26T19:36:06.957222Z","shell.execute_reply.started":"2025-07-26T19:35:57.804705Z","shell.execute_reply":"2025-07-26T19:36:06.956626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"BASE_DIR = pathlib.Path(\"/kaggle/input/cmi-detect-behavior-with-sensor-data\")\nMODEL_DIR = pathlib.Path(\"/kaggle/input/lstm-repo\")\nDEVICE = torch.device(\"cuda\") \n\n# data\nID_COL = \"sequence_id\"\nSEQ_COL = \"sequence_counter\"\n\nCLASSES = [\n    \"Above ear - pull hair\",\n    \"Cheek - pinch skin\",\n    \"Drink from bottle/cup\",\n    \"Eyebrow - pull hair\",\n    \"Eyelash - pull hair\",\n    \"Feel around in tray and pull out an object\",\n    \"Forehead - pull hairline\",\n    \"Forehead - scratch\",\n    \"Glasses on/off\",\n    \"Neck - pinch skin\",\n    \"Neck - scratch\",\n    \"Pinch knee/leg skin\",\n    \"Pull air toward your face\",\n    \"Scratch knee/leg skin\",\n    \"Text on phone\",\n    \"Wave hello\",\n    \"Write name in air\",\n    \"Write name on leg\",\n]\n\n# model\nMAX_LENGTH = 55\nHIDDEN_SIZE = 256\nN_LAYERS = 1\n\nBIDIRECTIONAL = True\nDROP = 0.2\nNUM_CLASSES = len(CLASSES)","metadata":{"papermill":{"duration":0.014086,"end_time":"2025-06-21T19:01:41.342286","exception":false,"start_time":"2025-06-21T19:01:41.3282","status":"completed"},"tags":[],"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:36:19.129129Z","iopub.execute_input":"2025-07-26T19:36:19.129827Z","iopub.status.idle":"2025-07-26T19:36:19.134373Z","shell.execute_reply.started":"2025-07-26T19:36:19.129801Z","shell.execute_reply":"2025-07-26T19:36:19.133789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.listdir(MODEL_DIR)","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:36:20.980276Z","iopub.execute_input":"2025-07-26T19:36:20.980916Z","iopub.status.idle":"2025-07-26T19:36:20.988758Z","shell.execute_reply.started":"2025-07-26T19:36:20.980889Z","shell.execute_reply":"2025-07-26T19:36:20.988151Z"},"papermill":{"duration":0.017916,"end_time":"2025-06-21T19:01:41.363954","exception":false,"start_time":"2025-06-21T19:01:41.346038","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Prepare","metadata":{"papermill":{"duration":0.003498,"end_time":"2025-06-21T19:01:41.37147","exception":false,"start_time":"2025-06-21T19:01:41.367972","status":"completed"},"tags":[]}},{"cell_type":"code","source":"transformer = joblib.load(MODEL_DIR / \"second_lstm_preprocessing.joblib\")\ntransformer","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:36:22.827665Z","iopub.execute_input":"2025-07-26T19:36:22.827975Z","iopub.status.idle":"2025-07-26T19:36:22.883431Z","shell.execute_reply.started":"2025-07-26T19:36:22.827953Z","shell.execute_reply":"2025-07-26T19:36:22.882722Z"},"papermill":{"duration":1.897422,"end_time":"2025-06-21T19:01:43.272445","exception":false,"start_time":"2025-06-21T19:01:41.375023","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CLASS_NOS=18\nhidden_size=256\nimport torch.nn.init as init\nfrom functools import partial\nimport torch\nimport torch.nn as nn\nfrom torch.nn import LayerNorm\n\n\nclass GeneralRelu(nn.Module):\n    def __init__(self, leak=None, sub=None, maxv=None):\n        super().__init__()\n        self.leak,self.sub,self.maxv = leak,sub,maxv\n\n    def forward(self, x): \n        x = F.leaky_relu(x,self.leak) if self.leak is not None else F.relu(x)\n        if self.sub is not None: x -= self.sub\n        if self.maxv is not None: x.clamp_max_(self.maxv)\n        return x\nclass LayerNorm_M(nn.Module):\n    def __init__(self, hidden_size=256, eps=1e-5):\n        super().__init__()\n        self.eps = eps\n        self.gamma = nn.Parameter(torch.ones(1,1, hidden_size))  # scale\n        self.beta = nn.Parameter(torch.zeros(1,1, hidden_size))  # shift\n\n    def forward(self, x):\n        # x shape: [num_layers, batch_size, hidden_size]\n        mean = x.mean(dim=-1, keepdim=True)  # mean over hidden_size\n        std = x.std(dim=-1, keepdim=True)\n        return self.gamma * (x - mean) / (std + self.eps) + self.beta\n\n\n\nclass LSTMClassifier(nn.Module):\n    def __init__(self, input_size=28, hidden_size=256, num_layers=1, bidirectional=False, num_classes=18):\n        super().__init__()\n        self.hidden_size = hidden_size\n        self.num_layers = num_layers\n        self.bidirectional = bidirectional\n        self.num_directions = 2 if bidirectional else 1\n\n        # Create LSTM layers and layer norms manually\n        self.lstm_layers = nn.ModuleList()\n        self.layer_norms = nn.ModuleList()\n\n        for layer in range(num_layers):\n            lstm_input_size = input_size if layer == 0 else hidden_size * self.num_directions\n            lstm_layer = nn.LSTM(input_size=lstm_input_size,\n                        hidden_size=hidden_size,\n                        num_layers=1,\n                        bidirectional=bidirectional,\n                        dropout=0.2,\n                        batch_first=False)\n            self.lstm_layers.append(\n               lstm_layer\n            )\n            for name, param in lstm_layer.named_parameters():\n                if 'weight' in name:  # Apply only to weights, not biases\n                    init.xavier_normal_(param)\n            self.layer_norms.append(LayerNorm(hidden_size * self.num_directions))\n\n        self.dropout = nn.Dropout(p=0.3)\n        # self.leaky_act = GeneralRelu(leak=0.4, sub=0.1)\n        self.classifier = nn.Sequential(\n            nn.Linear(hidden_size * self.num_directions, hidden_size),\n            GeneralRelu(leak=0.4, sub=0.1),\n            nn.BatchNorm1d(num_features=hidden_size),\n            nn.Linear(hidden_size, num_classes)\n        )\n\n    def forward(self, x):\n        out = x  # (seq_len, batch, input_size)\n        h_n = None\n        for i, (lstm, norm) in enumerate(zip(self.lstm_layers, self.layer_norms)):\n            out, (h_n, _) = lstm(out)   # out: (seq_len, batch, hidden*2), h_n: (num_directions, batch, hidden)\n            out = norm(out)\n\n        # Get final hidden state from last layer\n        # h_n: (num_directions, batch, hidden_size)\n        h_final = h_n.transpose(0, 1).contiguous().view(x.size(1), -1)  # (batch, hidden_size * num_directions)\n\n        h_final = self.dropout(h_final)\n        return self.classifier(h_final)\n\ndef initialize_weights(module):\n    if isinstance(module, nn.Linear):  # Apply to Linear layers\n        init.kaiming_normal_(module.weight, nonlinearity='relu')\n        if module.bias is not None:\n            init.zeros_(module.bias)\n\n\nmodel = LSTMClassifier()\ninitialize_weights(model)\nmodel.eval()\nmodel.to(DEVICE)","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:36:25.048700Z","iopub.execute_input":"2025-07-26T19:36:25.049289Z","iopub.status.idle":"2025-07-26T19:36:25.504505Z","shell.execute_reply.started":"2025-07-26T19:36:25.049265Z","shell.execute_reply":"2025-07-26T19:36:25.503837Z"},"papermill":{"duration":0.051318,"end_time":"2025-06-21T19:01:43.328103","exception":false,"start_time":"2025-06-21T19:01:43.276785","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"os.listdir(MODEL_DIR)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:36:39.210599Z","iopub.execute_input":"2025-07-26T19:36:39.210888Z","iopub.status.idle":"2025-07-26T19:36:39.216360Z","shell.execute_reply.started":"2025-07-26T19:36:39.210868Z","shell.execute_reply":"2025-07-26T19:36:39.215625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"encoder_with_pooling\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:36:43.278871Z","iopub.execute_input":"2025-07-26T19:36:43.279145Z","iopub.status.idle":"2025-07-26T19:36:43.284017Z","shell.execute_reply.started":"2025-07-26T19:36:43.279124Z","shell.execute_reply":"2025-07-26T19:36:43.283298Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# defining and loading the autoencoder\nencoder_with_pooling.load_state_dict(torch.load(\"/kaggle/input/tof-autoencoder/autoencoder_second.pth\",weights_only=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:36:45.959002Z","iopub.execute_input":"2025-07-26T19:36:45.959776Z","iopub.status.idle":"2025-07-26T19:36:45.978565Z","shell.execute_reply.started":"2025-07-26T19:36:45.959729Z","shell.execute_reply":"2025-07-26T19:36:45.977641Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.load_state_dict(torch.load(MODEL_DIR / \"lstm_third_attempt.pth\", weights_only=True))","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:36:48.903809Z","iopub.execute_input":"2025-07-26T19:36:48.904095Z","iopub.status.idle":"2025-07-26T19:36:48.932328Z","shell.execute_reply.started":"2025-07-26T19:36:48.904076Z","shell.execute_reply":"2025-07-26T19:36:48.931786Z"},"papermill":{"duration":0.029664,"end_time":"2025-06-21T19:01:43.36255","exception":false,"start_time":"2025-06-21T19:01:43.332886","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### Direct test","metadata":{"papermill":{"duration":0.004039,"end_time":"2025-06-21T19:01:43.370963","exception":false,"start_time":"2025-06-21T19:01:43.366924","status":"completed"},"tags":[]}},{"cell_type":"code","source":"train = pl.read_csv( BASE_DIR / \"train.csv\", n_rows=1000)\nsequence = train.filter( pl.col(ID_COL) == \"SEQ_000092\")\nsequence","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:36:51.347214Z","iopub.execute_input":"2025-07-26T19:36:51.347958Z","iopub.status.idle":"2025-07-26T19:36:52.122517Z","shell.execute_reply.started":"2025-07-26T19:36:51.347932Z","shell.execute_reply":"2025-07-26T19:36:52.121911Z"},"papermill":{"duration":0.311706,"end_time":"2025-06-21T19:01:43.686648","exception":false,"start_time":"2025-06-21T19:01:43.374942","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = sequence.to_pandas()\ndf = df.sort_values(by=SEQ_COL)","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:41:13.697842Z","iopub.execute_input":"2025-07-26T19:41:13.698141Z","iopub.status.idle":"2025-07-26T19:41:13.709203Z","shell.execute_reply.started":"2025-07-26T19:41:13.698121Z","shell.execute_reply":"2025-07-26T19:41:13.708585Z"},"papermill":{"duration":0.070684,"end_time":"2025-06-21T19:01:43.762065","exception":false,"start_time":"2025-06-21T19:01:43.691381","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"headers = sequence.columns\ntof_cols = [header for header in headers if header.startswith(\"tof\")]\nprocessed_tof_vals = preprocess_tof_data(df,tof_cols)\n\nflattened_vals = transform_tof_data(encoder_with_pooling,processed_tof_vals)\ntof_features = [\"tof_feature_{}\".format(idx) for idx in range(flattened_vals.shape[-1])]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:41:15.691224Z","iopub.execute_input":"2025-07-26T19:41:15.691843Z","iopub.status.idle":"2025-07-26T19:41:15.699971Z","shell.execute_reply.started":"2025-07-26T19:41:15.691818Z","shell.execute_reply":"2025-07-26T19:41:15.699333Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df = transformer.transform(df[FEATURES[:-16]])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:41:27.867253Z","iopub.execute_input":"2025-07-26T19:41:27.867515Z","iopub.status.idle":"2025-07-26T19:41:27.877782Z","shell.execute_reply.started":"2025-07-26T19:41:27.867496Z","shell.execute_reply":"2025-07-26T19:41:27.877216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df[tof_features] = flattened_vals.numpy()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-07-26T19:41:41.825883Z","iopub.execute_input":"2025-07-26T19:41:41.826195Z","iopub.status.idle":"2025-07-26T19:41:41.835369Z","shell.execute_reply.started":"2025-07-26T19:41:41.826170Z","shell.execute_reply":"2025-07-26T19:41:41.834578Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\ndf = df[-MAX_LENGTH:]\nx = torch.tensor(df[FEATURES].values.astype(np.float32))\nx = x.unsqueeze(dim=1)\nx.shape","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:41:50.913439Z","iopub.execute_input":"2025-07-26T19:41:50.913676Z","iopub.status.idle":"2025-07-26T19:41:50.921046Z","shell.execute_reply.started":"2025-07-26T19:41:50.913661Z","shell.execute_reply":"2025-07-26T19:41:50.920478Z"},"papermill":{"duration":0.04637,"end_time":"2025-06-21T19:01:43.813282","exception":false,"start_time":"2025-06-21T19:01:43.766912","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"with torch.no_grad():\n    x = x.to(DEVICE)\n\n    logits = model(x).cpu()\nlogits","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:41:58.723296Z","iopub.execute_input":"2025-07-26T19:41:58.723576Z","iopub.status.idle":"2025-07-26T19:41:59.047172Z","shell.execute_reply.started":"2025-07-26T19:41:58.723557Z","shell.execute_reply":"2025-07-26T19:41:59.046465Z"},"papermill":{"duration":0.149952,"end_time":"2025-06-21T19:01:43.995096","exception":false,"start_time":"2025-06-21T19:01:43.845144","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CLASSES[int(torch.argmax(logits).item())]","metadata":{"execution":{"iopub.status.busy":"2025-07-26T19:42:01.719298Z","iopub.execute_input":"2025-07-26T19:42:01.719570Z","iopub.status.idle":"2025-07-26T19:42:01.727105Z","shell.execute_reply.started":"2025-07-26T19:42:01.719549Z","shell.execute_reply":"2025-07-26T19:42:01.726554Z"},"papermill":{"duration":0.01491,"end_time":"2025-06-21T19:01:44.014868","exception":false,"start_time":"2025-06-21T19:01:43.999958","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Wrap it into predict function","metadata":{"papermill":{"duration":0.004466,"end_time":"2025-06-21T19:01:44.024333","exception":false,"start_time":"2025-06-21T19:01:44.019867","status":"completed"},"tags":[]}},{"cell_type":"code","source":"def predict(sequence: pl.DataFrame, demographics: pl.DataFrame) -> str:\n    preprocess_tof_tens = lambda tens: torch.Tensor((tens-tof_data_mean)/tof_data_std).unsqueeze(1).float()\n    df = sequence.to_pandas()\n    df = df.sort_values(SEQ_COL)\n    headers = sequence.columns\n    tof_cols = [header for header in headers if header.startswith(\"tof\")]\n    processed_tof_vals = preprocess_tof_data(df,tof_cols)\n    flattened_vals = transform_tof_data(encoder_with_pooling,processed_tof_vals)\n    tof_features = [\"tof_feature_{}\".format(idx) for idx in range(flattened_vals.shape[-1])]\n    df = transformer.transform(df)\n    df[tof_features] = flattened_vals.numpy()\n    df = df[-MAX_LENGTH:]\n    x = torch.tensor(df[FEATURES].values.astype(np.float32))\n    x = x.unsqueeze(dim=1)\n    mask = torch.ones_like(x[..., -1])\n    model.eval()\n    with torch.no_grad():\n        x = x.to(DEVICE)\n        logits = model(x).cpu()\n    pred = CLASSES[int(torch.argmax(logits).item())]\n    print(pred)\n    return pred","metadata":{"execution":{"iopub.status.busy":"2025-07-26T15:11:11.013538Z","iopub.execute_input":"2025-07-26T15:11:11.013794Z","iopub.status.idle":"2025-07-26T15:11:11.020381Z","shell.execute_reply.started":"2025-07-26T15:11:11.013778Z","shell.execute_reply":"2025-07-26T15:11:11.019644Z"},"papermill":{"duration":0.011803,"end_time":"2025-06-21T19:01:44.04068","exception":false,"start_time":"2025-06-21T19:01:44.028877","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Run ","metadata":{"papermill":{"duration":0.004527,"end_time":"2025-06-21T19:01:44.049974","exception":false,"start_time":"2025-06-21T19:01:44.045447","status":"completed"},"tags":[]}},{"cell_type":"code","source":"inference_server = kaggle_evaluation.cmi_inference_server.CMIInferenceServer(predict)\n\nif os.getenv(\"KAGGLE_IS_COMPETITION_RERUN\"):\n    inference_server.serve()\nelse:\n    inference_server.run_local_gateway(\n        data_paths=(BASE_DIR / \"test.csv\", BASE_DIR / \"test_demographics.csv\")\n    )","metadata":{"execution":{"iopub.status.busy":"2025-07-18T15:33:59.098141Z","iopub.execute_input":"2025-07-18T15:33:59.098314Z","iopub.status.idle":"2025-07-18T15:34:00.324689Z","shell.execute_reply.started":"2025-07-18T15:33:59.098299Z","shell.execute_reply":"2025-07-18T15:34:00.324025Z"},"papermill":{"duration":1.172872,"end_time":"2025-06-21T19:01:45.227499","exception":false,"start_time":"2025-06-21T19:01:44.054627","status":"completed"},"tags":[],"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}