{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"},{"sourceId":7113901,"sourceType":"datasetVersion","datasetId":4102330}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport random\nfrom tqdm import tqdm\nimport torch\nfrom torch import tensor\nimport torch.nn as nn\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset\nfrom torch.utils.data import DataLoader\nimport torch.nn.functional as F\nfrom itertools import groupby\nimport gc as garbage\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\nnp.random.seed(0)\npd.set_option('display.precision', 2)  \npd.set_option('display.float_format', '{:.2f}'.format) \n\ndevice = 'mps' if torch.backends.mps.is_available() else 'cuda' if torch.cuda.is_available() else 'cpu'\n\n\npath_train_parquet = \"/kaggle/input/sleep-dataset/Zzzs_train.parquet\"\npath_train_events = \"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\"\npath_test_series_parquet = \"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\"\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-12-04T04:38:16.886133Z","iopub.execute_input":"2023-12-04T04:38:16.886588Z","iopub.status.idle":"2023-12-04T04:38:16.895364Z","shell.execute_reply.started":"2023-12-04T04:38:16.886544Z","shell.execute_reply":"2023-12-04T04:38:16.894211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(path_train_events)\n\n# Check for NaN values in the 'step' column for each row\nnan_values = train_events['step'].isna()\n\n# Group by 'series_id' and check if any group has at least one NaN value\ngrouped_nan_values = nan_values.groupby(train_events['series_id'])\nnan_group = grouped_nan_values.any()\n\n# Filter out 'series_id' values without any NaN values\ntrain_ids = nan_group[~nan_group].index.tolist()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:38:16.897279Z","iopub.execute_input":"2023-12-04T04:38:16.897586Z","iopub.status.idle":"2023-12-04T04:38:16.929975Z","shell.execute_reply.started":"2023-12-04T04:38:16.897559Z","shell.execute_reply":"2023-12-04T04:38:16.928792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_series_data(series_ids):\n    \n    # Define the parquet file path\n    parquet_file_path = path_train_parquet\n\n    # Read data from the parquet file using specified filters\n    series = pd.read_parquet(parquet_file_path, filters=[('series_id', 'in', series_ids)])\n\n    # Convert data types for selected columns\n    column_data_types = {'series_id': 'category', 'step': 'int16', 'awake': 'int16'}\n    series = series.astype(column_data_types)\n\n    # Call a function to add features to the data\n    series = add_features(series)\n\n    # Return the processed data\n    return series","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:38:16.931250Z","iopub.execute_input":"2023-12-04T04:38:16.931971Z","iopub.status.idle":"2023-12-04T04:38:16.938937Z","shell.execute_reply.started":"2023-12-04T04:38:16.931930Z","shell.execute_reply":"2023-12-04T04:38:16.937677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_features(dataframe):\n    # Convert 'timestamp' to datetime in UTC\n    dataframe[\"timestamp\"] = pd.to_datetime(dataframe[\"timestamp\"], utc=True)\n    \n    # Extract 'hour' and 'dayofweek' from 'timestamp'\n    dataframe[\"hour\"] = dataframe[\"timestamp\"].dt.hour\n    dataframe[\"dayofweek\"] = dataframe[\"timestamp\"].dt.dayofweek\n    \n    # Calculate 'anglez_times_enmo'\n    dataframe[\"anglez_times_enmo\"] = abs(dataframe[\"anglez\"] * dataframe[\"enmo\"])\n\n    # Define periods for rolling calculations\n    periods = 6\n    \n    # Loop through 'anglez' and 'enmo' columns\n    for col in ['anglez', 'enmo']:\n        # Calculate differences and rolling statistics for each column\n        dataframe[f\"{col}_diff\"] = dataframe.groupby('series_id')[col].diff(periods=periods).fillna(method=\"bfill\")\n        for stat in ['mean', 'min', 'max', 'std']:\n            dataframe[f\"{col}_{stat}\"] = dataframe[col].rolling(periods, center=True).agg(stat).fillna(method=\"bfill\").astype('float32')\n\n    # Loop through 'anglez_diff' and 'enmo_diff' columns\n    for col in ['anglez_diff', 'enmo_diff']:\n        # Calculate rolling mean for differences\n        dataframe[f\"{col}_rolling\"] = dataframe[col].rolling(periods, center=True).mean().fillna(method=\"bfill\").astype('float32')\n\n    # Return the dataframe with added features\n    return dataframe","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:38:16.940685Z","iopub.execute_input":"2023-12-04T04:38:16.941412Z","iopub.status.idle":"2023-12-04T04:38:16.954097Z","shell.execute_reply.started":"2023-12-04T04:38:16.941365Z","shell.execute_reply":"2023-12-04T04:38:16.953002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fetch data for the first 8 series using fetch_series_data function\ntrain_w_features = fetch_series_data(train_ids)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:38:16.957302Z","iopub.execute_input":"2023-12-04T04:38:16.958037Z","iopub.status.idle":"2023-12-04T04:39:49.466141Z","shell.execute_reply.started":"2023-12-04T04:38:16.957992Z","shell.execute_reply":"2023-12-04T04:39:49.464818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define Feature Columns\nselected_features = [\"step\", \"hour\", \"dayofweek\", \"anglez_times_enmo\",\n                   \"anglez\", \"anglez_diff\", \"anglez_mean\", \"anglez_min\", \"anglez_max\", \"anglez_std\", \"anglez_diff_rolling\",\n                   \"enmo\", \"enmo_diff\", \"enmo_mean\", \"enmo_min\", \"enmo_max\", \"enmo_std\", \"enmo_diff_rolling\"]\n\n# Define Target Column\ntarget_column = 'awake'\n\n# Initialize StandardScaler\nscaler = StandardScaler()\n\n# Standardize Features\ninputs = scaler.fit_transform(train_w_features[selected_features])\n# 'fit_transform': Compute the mean and standard deviation for scaling, then apply the transformation.\n\n# Extract Target Values\nlabels = train_w_features[target_column].values\n# Extracting the target column values (labels) that the model will learn to predict.\n\n# Convert the standardized feature matrix to a PyTorch tensor with dtype=torch.float32\ninputs = tensor(inputs, dtype=torch.float32)\n\n# Convert the target variable array to a PyTorch tensor with dtype=torch.long\nlabels = tensor(labels, dtype=torch.long)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:49.467584Z","iopub.execute_input":"2023-12-04T04:39:49.467942Z","iopub.status.idle":"2023-12-04T04:39:59.075335Z","shell.execute_reply.started":"2023-12-04T04:39:49.467910Z","shell.execute_reply":"2023-12-04T04:39:59.074326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_samples = len(inputs)\nsplit_index = int(total_samples * 0.8)\n\n# Split the tensors into training and validation sets\ninputs_train, inputs_val = torch.split(inputs, [split_index, total_samples - split_index])\nlabels_train, labels_val = torch.split(labels, [split_index, total_samples - split_index])","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.076921Z","iopub.execute_input":"2023-12-04T04:39:59.077251Z","iopub.status.idle":"2023-12-04T04:39:59.083920Z","shell.execute_reply.started":"2023-12-04T04:39:59.077221Z","shell.execute_reply":"2023-12-04T04:39:59.082582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a custom PyTorch Dataset for time series data\nclass TimeSeriesDataset(Dataset):\n    def __init__(self, features, labels):\n        # Initialize the dataset with features and labels\n        self.features = features\n        self.labels = labels\n\n    def __len__(self):\n        # Return the number of samples in the dataset\n        return len(self.features)\n\n    def __getitem__(self, index):\n        # Return a tuple of features and labels for a given index\n        return self.features[index], self.labels[index]\n\n# Create dataset instances for training and validation sets\ntrain_dataset = TimeSeriesDataset(inputs_train, labels_train)\nval_dataset = TimeSeriesDataset(inputs_val, labels_val)\n\n# Set batch size to represent a 1-hour window\nbatch_size = 12 * 60\n\n# Create data loaders for training and validation sets\ntrain_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False)\nval_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.087621Z","iopub.execute_input":"2023-12-04T04:39:59.088102Z","iopub.status.idle":"2023-12-04T04:39:59.096273Z","shell.execute_reply.started":"2023-12-04T04:39:59.088058Z","shell.execute_reply":"2023-12-04T04:39:59.095121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define a simple LSTM model using PyTorch's nn.Module\nclass LSTM(nn.Module):\n    def __init__(self, input_dim, hidden_dim, num_layers, output_dim):\n        # Initialize the LSTM model with specified input, hidden, and output dimensions\n        super(LSTM, self).__init__()\n        self.hidden_dim = hidden_dim\n        self.num_layers = num_layers\n        # LSTM layer with ReLU activation and fully connected layer\n        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True)\n        self.relu = nn.ReLU()\n        self.fc = nn.Linear(hidden_dim, output_dim)\n    \n    def forward(self, input_seq):\n        # Get the batch size from the input sequence\n        batch_size = input_seq.size(0)\n        # Initialize initial hidden and cell states\n        initial_hidden = torch.zeros(self.num_layers, batch_size, self.hidden_dim).to(device)\n        initial_cell = torch.zeros(self.num_layers, batch_size, self.hidden_dim).to(device)\n        # Add a dimension to make input_seq 3D\n        input_seq = input_seq[:, None, :]\n        # Forward pass through the LSTM layer and apply the fully connected layer\n        lstm_out, _ = self.lstm(input_seq, (initial_hidden, initial_cell))\n        lstm_out = self.fc(self.relu(lstm_out[:, -1, :])) \n        return lstm_out\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.097519Z","iopub.execute_input":"2023-12-04T04:39:59.097826Z","iopub.status.idle":"2023-12-04T04:39:59.113386Z","shell.execute_reply.started":"2023-12-04T04:39:59.097798Z","shell.execute_reply":"2023-12-04T04:39:59.112458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to calculate accuracy\ndef calculate_accuracy(predictions, labels):\n    # Calculate accuracy by comparing predicted labels to actual labels\n    return (predictions.argmax(axis=1) == labels).float().mean()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.114533Z","iopub.execute_input":"2023-12-04T04:39:59.115470Z","iopub.status.idle":"2023-12-04T04:39:59.130232Z","shell.execute_reply.started":"2023-12-04T04:39:59.115436Z","shell.execute_reply":"2023-12-04T04:39:59.129121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training function\ndef train(model, dataloader, loss_function, optimizer):\n    # Set the model to training mode\n    model.train()\n    total_loss = 0.0\n    total_correct = 0\n    total_samples = 0\n\n    # Iterate over batches in the training dataloader\n    progress_bar = tqdm(dataloader, desc='Training')\n    for inputs, labels in progress_bar:\n        # Move inputs and labels to the device (e.g., GPU)\n        inputs, labels = inputs.to(device), labels.to(device)\n\n        # Zero the gradients to avoid accumulation\n        optimizer.zero_grad()\n\n        # Forward pass: compute predicted outputs by passing inputs to the model\n        predictions = model(inputs)\n\n        # Compute the loss between predicted outputs and actual labels\n        loss = loss_function(predictions, labels)\n\n        # Backward pass: compute gradient of the loss with respect to model parameters\n        loss.backward()\n\n        # Update the model parameters using the optimizer\n        optimizer.step()\n\n        # Update statistics for monitoring training progress\n        total_loss += loss.item() * inputs.size(0)\n        total_correct += int(calculate_accuracy(predictions, labels) * inputs.size(0))\n        total_samples += inputs.size(0)\n\n        # Update the progress bar with current loss and accuracy\n        progress_bar.set_postfix({'Loss': total_loss / total_samples, 'Accuracy': total_correct / total_samples})\n\n    # Calculate average training loss and accuracy\n    return total_loss / total_samples, total_correct / total_samples","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.131561Z","iopub.execute_input":"2023-12-04T04:39:59.131908Z","iopub.status.idle":"2023-12-04T04:39:59.142967Z","shell.execute_reply.started":"2023-12-04T04:39:59.131876Z","shell.execute_reply":"2023-12-04T04:39:59.141819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluation function\ndef evaluate(model, dataloader, loss_function):\n    # Set the model to evaluation mode (no gradient computation)\n    model.eval()\n    total_loss = 0.0\n    total_correct = 0\n    total_samples = 0\n\n    # Iterate over batches in the validation dataloader\n    progress_bar = tqdm(dataloader, desc='Evaluating')\n    with torch.no_grad():\n        for inputs, labels in progress_bar:\n            # Move inputs and labels to the device (e.g., GPU)\n            inputs, labels = inputs.to(device), labels.to(device)\n\n            # Forward pass: compute predicted outputs by passing inputs to the model\n            predictions = model(inputs)\n\n            # Compute the loss between predicted outputs and actual labels\n            loss = loss_function(predictions, labels)\n\n            # Update statistics for monitoring evaluation progress\n            total_loss += loss.item() * inputs.size(0)\n            total_correct += int(calculate_accuracy(predictions, labels) * inputs.size(0))\n            total_samples += inputs.size(0)\n\n            # Update the progress bar with current loss and accuracy\n            progress_bar.set_postfix({'Loss': total_loss / total_samples, 'Accuracy': total_correct / total_samples})\n\n    # Calculate average validation loss and accuracy\n    return total_loss / total_samples, total_correct / total_samples\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.144369Z","iopub.execute_input":"2023-12-04T04:39:59.145091Z","iopub.status.idle":"2023-12-04T04:39:59.160556Z","shell.execute_reply.started":"2023-12-04T04:39:59.145054Z","shell.execute_reply":"2023-12-04T04:39:59.159278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training loop over multiple epochs\ndef fit(epochs, model, loss_function, optimizer, train_dl, valid_dl):\n    # Lists to store training and validation losses, and accuracies\n    train_losses, train_accuracies = [], []\n    valid_losses, valid_accuracies = [], []\n\n    for epoch in range(epochs):\n        print(f'\\nEpoch {epoch + 1}/{epochs}')\n\n        # Training\n        train_loss, train_acc = train(model, train_dl, loss_function, optimizer)\n        train_losses.append(train_loss)\n        train_accuracies.append(train_acc)\n\n        # Validation\n        valid_loss, valid_acc = evaluate(model, valid_dl, loss_function)\n        valid_losses.append(valid_loss)\n        valid_accuracies.append(valid_acc)\n\n        # Print and log training and validation metrics\n        print(f'Training Loss: {train_loss:.4f}, Accuracy: {train_acc:.4f}')\n        print(f'Validation Loss: {valid_loss:.4f}, Accuracy: {valid_acc:.4f}')\n\n    # Return the collected metrics for further analysis or plotting\n    return {\n        'train_losses': train_losses,\n        'train_accuracies': train_accuracies,\n        'valid_losses': valid_losses,\n        'valid_accuracies': valid_accuracies\n    }\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.162102Z","iopub.execute_input":"2023-12-04T04:39:59.162416Z","iopub.status.idle":"2023-12-04T04:39:59.174917Z","shell.execute_reply.started":"2023-12-04T04:39:59.162388Z","shell.execute_reply":"2023-12-04T04:39:59.174077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction function\ndef predict(inputs):\n    # Set the model to evaluation mode (no gradient computation)\n    model.eval()\n    with torch.no_grad():\n        # Move inputs to the device (e.g., GPU)\n        inputs = inputs.to(device)\n        \n        # Forward pass: compute predicted outputs by passing inputs to the model\n        predictions = model(inputs).to('cpu')\n\n        # Apply softmax to get probabilities and argmax to get predicted labels\n        scores = F.log_softmax(predictions, dim=-1).exp()\n        values = scores.argmax(axis=1)\n\n    # Return predicted probabilities and labels\n    return scores, values","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.175992Z","iopub.execute_input":"2023-12-04T04:39:59.176280Z","iopub.status.idle":"2023-12-04T04:39:59.189882Z","shell.execute_reply.started":"2023-12-04T04:39:59.176254Z","shell.execute_reply":"2023-12-04T04:39:59.188929Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = LSTM(input_dim = len(selected_features), hidden_dim = 32, num_layers = 2, output_dim = 2)\nopt = torch.optim.Adam(model.parameters(), lr=0.001)\nloss_function = F.cross_entropy\n\nmodel.to(device)\n\nepochs = 5 \nresult = fit(epochs, model, loss_function, opt, train_loader, val_loader)\n\n# Access the values from the result dictionary\nloss = result['valid_losses']  \nacc = result['valid_accuracies'] ","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:39:59.191350Z","iopub.execute_input":"2023-12-04T04:39:59.191787Z","iopub.status.idle":"2023-12-04T04:58:26.965470Z","shell.execute_reply.started":"2023-12-04T04:39:59.191755Z","shell.execute_reply":"2023-12-04T04:58:26.964767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs, labels = next(iter(train_loader))","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:26.969197Z","iopub.execute_input":"2023-12-04T04:58:26.969528Z","iopub.status.idle":"2023-12-04T04:58:26.980711Z","shell.execute_reply.started":"2023-12-04T04:58:26.969497Z","shell.execute_reply":"2023-12-04T04:58:26.979442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"score, predicted_value = predict(inputs)","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:26.982292Z","iopub.execute_input":"2023-12-04T04:58:26.983026Z","iopub.status.idle":"2023-12-04T04:58:26.991163Z","shell.execute_reply.started":"2023-12-04T04:58:26.982983Z","shell.execute_reply":"2023-12-04T04:58:26.989911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_transform(dataframe):\n    # Apply feature engineering to the input dataframe using add_features function\n    enhanced_dataframe = add_features(dataframe)\n    \n    # Extract selected features from the enhanced dataframe\n    selected_features_dataframe = enhanced_dataframe[selected_features]\n    \n    # Standardize the selected features using the pre-trained scaler\n    standardized_dataframe = scaler.transform(selected_features_dataframe)\n    \n    # Convert the standardized dataframe to a PyTorch tensor with float32 data type\n    tensor_data = torch.tensor(standardized_dataframe, dtype=torch.float32)\n    \n    # Return the resulting PyTorch tensor\n    return tensor_data","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:26.992659Z","iopub.execute_input":"2023-12-04T04:58:26.993021Z","iopub.status.idle":"2023-12-04T04:58:26.998841Z","shell.execute_reply.started":"2023-12-04T04:58:26.992989Z","shell.execute_reply":"2023-12-04T04:58:26.998011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_train_set(series_ids_to_predict):\n    for series_id in series_ids_to_predict:\n        # Fetch data for the current series_id\n        series_data = fetch_series_data([series_id])\n\n        # Transform the fetched data\n        transformed_data = data_transform(series_data)\n\n        # Predict using the model\n        prediction_score, predicted_values = predict(transformed_data.to(device))\n\n        # Get the actual values\n        actual_values = tensor(series_data['awake'].to_numpy())\n\n        # Calculate and print accuracy\n        accuracy_value = calculate_accuracy(prediction_score, actual_values)\n\n# Call the function with a subset of train_ids\npredict_train_set(train_ids[30:32])\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:27.000221Z","iopub.execute_input":"2023-12-04T04:58:27.000560Z","iopub.status.idle":"2023-12-04T04:58:39.299627Z","shell.execute_reply.started":"2023-12-04T04:58:27.000529Z","shell.execute_reply":"2023-12-04T04:58:39.298731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_w_features, inputs_train, labels_train\ngarbage.collect()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.304098Z","iopub.execute_input":"2023-12-04T04:58:39.304476Z","iopub.status.idle":"2023-12-04T04:58:39.451319Z","shell.execute_reply.started":"2023-12-04T04:58:39.304443Z","shell.execute_reply":"2023-12-04T04:58:39.450230Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_event_labels(dataframe):\n    lstCV = zip(dataframe.series_id, dataframe.smooth)\n    lstPOI = []\n    for (c, v), g in groupby(lstCV, lambda cv: (cv[0], cv[1] != 0 and not pd.isnull(cv[1]))):\n        llg = sum(1 for item in g)\n        if v is False:\n            lstPOI.extend([0] * llg)\n        else:\n            lstPOI.extend(['onset'] + (llg - 2) * [0] + ['wakeup'] if llg > 1 else [0])\n    return lstPOI\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.452742Z","iopub.execute_input":"2023-12-04T04:58:39.453167Z","iopub.status.idle":"2023-12-04T04:58:39.461883Z","shell.execute_reply.started":"2023-12-04T04:58:39.453126Z","shell.execute_reply":"2023-12-04T04:58:39.460869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_test_series(series_id):\n    return pd.read_parquet(\n        path_test_series_parquet, \n        columns=['series_id', 'timestamp', 'anglez', 'enmo', 'step'], \n        filters=[('series_id', '=', series_id)]\n    )","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.463553Z","iopub.execute_input":"2023-12-04T04:58:39.464307Z","iopub.status.idle":"2023-12-04T04:58:39.475903Z","shell.execute_reply.started":"2023-12-04T04:58:39.464268Z","shell.execute_reply":"2023-12-04T04:58:39.474883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def process_series(series):\n    inputs_test = data_transform(series)\n    score, _ = predict(inputs_test.to(device))\n    \n    series[\"not_awake\"] = score[:, 0]\n    series['awake'] = score[:, 1]\n\n    smoothing_length = 2 * 230\n    series[\"score\"] = series[\"awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    series[\"smooth\"] = series[\"not_awake\"].rolling(smoothing_length, center=True).mean().fillna(method=\"bfill\").fillna(method=\"ffill\")\n    series[\"smooth\"] = series[\"smooth\"].round()\n    series[\"event\"] = generate_event_labels(series)\n    \n    return series\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.477224Z","iopub.execute_input":"2023-12-04T04:58:39.478310Z","iopub.status.idle":"2023-12-04T04:58:39.487980Z","shell.execute_reply.started":"2023-12-04T04:58:39.478265Z","shell.execute_reply":"2023-12-04T04:58:39.487042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def save_series_to_csv(series, series_id):\n    dataframe = series.loc[series[\"event\"] != 0][[\"series_id\", \"step\", \"event\", \"score\"]].copy().reset_index(drop=True)\n    dataframe.to_csv(f'{series_id}.csv', index=False)\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.489294Z","iopub.execute_input":"2023-12-04T04:58:39.490318Z","iopub.status.idle":"2023-12-04T04:58:39.500337Z","shell.execute_reply.started":"2023-12-04T04:58:39.490245Z","shell.execute_reply":"2023-12-04T04:58:39.499041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def series_prediction(test_series_ids):\n    for series_id in test_series_ids:\n        series = load_test_series(series_id)\n        processed_series = process_series(series)\n        save_series_to_csv(processed_series, series_id)\n        del processed_series\n        garbage.collect()\n    return","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.501598Z","iopub.execute_input":"2023-12-04T04:58:39.502437Z","iopub.status.idle":"2023-12-04T04:58:39.510963Z","shell.execute_reply.started":"2023-12-04T04:58:39.502397Z","shell.execute_reply":"2023-12-04T04:58:39.509795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def all_series_predictions(test_series):\n    submission = pd.read_csv(f'{test_series[0]}.csv')\n    for series in test_series[1:]:\n        temp = pd.read_csv(f'{series}.csv')\n        submission = pd.concat([submission, temp], ignore_index=True)\n        del temp\n        garbage.collect()\n    submission = submission.copy().reset_index(drop=True).reset_index(names=\"row_id\")\n    submission.to_csv('submission.csv', index=False)\n    return submission","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.512293Z","iopub.execute_input":"2023-12-04T04:58:39.513299Z","iopub.status.idle":"2023-12-04T04:58:39.523121Z","shell.execute_reply.started":"2023-12-04T04:58:39.513259Z","shell.execute_reply":"2023-12-04T04:58:39.521775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_series_ids = pd.read_parquet(path_test_series_parquet, columns=['series_id'])['series_id'].unique()\n","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.524508Z","iopub.execute_input":"2023-12-04T04:58:39.526105Z","iopub.status.idle":"2023-12-04T04:58:39.545789Z","shell.execute_reply.started":"2023-12-04T04:58:39.526064Z","shell.execute_reply":"2023-12-04T04:58:39.544584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_prediction(test_series_ids)\nsubmission = all_series_predictions(test_series_ids)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2023-12-04T04:58:39.547135Z","iopub.execute_input":"2023-12-04T04:58:39.547738Z","iopub.status.idle":"2023-12-04T04:58:40.291769Z","shell.execute_reply.started":"2023-12-04T04:58:39.547699Z","shell.execute_reply":"2023-12-04T04:58:40.290768Z"},"trusted":true},"execution_count":null,"outputs":[]}]}