{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30579,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-01-15T09:33:35.452671Z","iopub.execute_input":"2024-01-15T09:33:35.453455Z","iopub.status.idle":"2024-01-15T09:33:35.461903Z","shell.execute_reply.started":"2024-01-15T09:33:35.453424Z","shell.execute_reply":"2024-01-15T09:33:35.460846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os\nimport joblib\nimport random\nimport math\nfrom tqdm.auto import tqdm\n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\n# from timm.scheduler import CosineLRScheduler\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa\nimport ctypes","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:33:35.464118Z","iopub.execute_input":"2024-01-15T09:33:35.464380Z","iopub.status.idle":"2024-01-15T09:33:35.480356Z","shell.execute_reply.started":"2024-01-15T09:33:35.464356Z","shell.execute_reply":"2024-01-15T09:33:35.479686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    SPLIT_DIR = \"/kaggle/working/split/\"\n    TEST_DIR = \"/kaggle/working/test/\"\n\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"\n\n    @staticmethod\n    def get_series_filename(series_id):\n        f = f'{series_id}_test_series.parquet'\n        return PATHS.SPLIT_DIR + f\n    def get_test_filename(series_id):\n        f = f'{series_id}_test_series.parquet'\n        return PATHS.TEST_DIR + f","metadata":{"execution":{"iopub.status.busy":"2024-01-15T11:12:36.592723Z","iopub.execute_input":"2024-01-15T11:12:36.593628Z","iopub.status.idle":"2024-01-15T11:12:36.599601Z","shell.execute_reply.started":"2024-01-15T11:12:36.593598Z","shell.execute_reply":"2024-01-15T11:12:36.598594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    DEMO_MODE = True","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:33:35.493571Z","iopub.execute_input":"2024-01-15T09:33:35.493829Z","iopub.status.idle":"2024-01-15T09:33:35.509991Z","shell.execute_reply.started":"2024-01-15T09:33:35.493799Z","shell.execute_reply":"2024-01-15T09:33:35.509155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class data_reader:\n    def __init__(self, demo_mode):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Percentage of removed steps : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n#         print(data.isna().any())\n#         data = data.bfill()\n        return data\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_steps = next(pf.iter_batches(batch_size=20_000)) \n                data = pa.Table.from_batches([demo_steps]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nsteps=20_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n                \n        gc.collect()\n        if data_props[\"has_timestamp\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n            gc.collect()\n        data = self.reduce_memory_usage(data)\n        return data","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:33:35.535144Z","iopub.execute_input":"2024-01-15T09:33:35.535486Z","iopub.status.idle":"2024-01-15T09:33:35.557602Z","shell.execute_reply.started":"2024-01-15T09:33:35.535456Z","shell.execute_reply":"2024-01-15T09:33:35.556923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = data_reader(demo_mode=False)\nseries = reader.load_data(data_name=\"train_series\")\nevents = reader.load_data(data_name=\"train_events\")","metadata":{"execution":{"iopub.status.busy":"2024-01-15T11:06:14.510090Z","iopub.execute_input":"2024-01-15T11:06:14.510913Z","iopub.status.idle":"2024-01-15T11:09:03.374314Z","shell.execute_reply.started":"2024-01-15T11:06:14.510883Z","shell.execute_reply":"2024-01-15T11:09:03.373384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(PATHS.SPLIT_DIR)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:55:50.915853Z","iopub.execute_input":"2024-01-15T09:55:50.916215Z","iopub.status.idle":"2024-01-15T09:55:50.921036Z","shell.execute_reply.started":"2024-01-15T09:55:50.916187Z","shell.execute_reply":"2024-01-15T09:55:50.920046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\ndata = []\nids = series.series_id.unique()\n\nfor viz_id in tqdm(ids):\n    viz_targets = []\n    viz_events = events[events.series_id == viz_id]\n    viz_series = series.loc[(series.series_id==viz_id)].copy().reset_index()\n    # viz_series['dt'] = pd.to_datetime(viz_series.timestamp,format = '%Y-%m-%dT%H:%M:%S%z').astype(\"datetime64[ns, UTC-04:00]\")\n    # viz_series['hour'] = viz_series['dt'].dt.hour\n\n    check = 0\n    for i in range(len(viz_events)-1):\n        if viz_events.iloc[i].event =='onset' and viz_events.iloc[i+1].event =='wakeup' and viz_events.iloc[i].night==viz_events.iloc[i+1].night:\n            start,end = viz_events.timestamp.iloc[i],viz_events.timestamp.iloc[i+1]\n\n            start_id = viz_series.loc[viz_series.timestamp ==start].index.values[0]\n            end_id = viz_series.loc[viz_series.timestamp ==end].index.values[0]\n            viz_targets.append((start_id,end_id))\n            check+=1\n    targets.append(viz_targets)\n    viz_series['anglez'] = viz_series['anglez'].astype('float')\n    viz_series['enmo'] = viz_series['enmo'].astype('float')\n    viz_series[['anglez','enmo','step']].to_parquet(PATHS.get_series_filename(viz_id))\n    # print(type(viz_series[['anglez','enmo','step']]))\n    # print(viz_series[['anglez','enmo','step']])\n    data.append(viz_series[['anglez','enmo','step']]) #you can include features like hour or minnute or second if you want to\n# joblib.dump((targets,data,ids), 'train_data.pkl')\nlen(data)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T11:09:06.411474Z","iopub.execute_input":"2024-01-15T11:09:06.411877Z","iopub.status.idle":"2024-01-15T11:10:05.652145Z","shell.execute_reply.started":"2024-01-15T11:09:06.411845Z","shell.execute_reply":"2024-01-15T11:10:05.651128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"训练框架部分","metadata":{}},{"cell_type":"code","source":"class CFG:\n    DEMO_MODE = False\n    VERBOSE = True\n\n    SEED = 42\n    DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'\n\nCFG.DEVICE","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:19.810263Z","iopub.execute_input":"2024-01-15T09:59:19.811175Z","iopub.status.idle":"2024-01-15T09:59:19.878390Z","shell.execute_reply.started":"2024-01-15T09:59:19.811143Z","shell.execute_reply":"2024-01-15T09:59:19.877309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def torch_fix_seed(seed=36):\n    # Python random\n    random.seed(seed)\n    # Numpy\n    np.random.seed(seed)\n    # Pytorch\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    # torch.backends.cudnn.deterministic = True\n    # torch.use_deterministic_algorithms = True\n    # torch.backends.cudnn.benchmark = True\n\ntorch_fix_seed(CFG.SEED)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:22.027391Z","iopub.execute_input":"2024-01-15T09:59:22.027768Z","iopub.status.idle":"2024-01-15T09:59:22.060411Z","shell.execute_reply.started":"2024-01-15T09:59:22.027741Z","shell.execute_reply":"2024-01-15T09:59:22.059716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_memory():\n    gc.collect()\n    ctypes.CDLL(\"libc.so.6\").malloc_trim(0)\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:24.547310Z","iopub.execute_input":"2024-01-15T09:59:24.547988Z","iopub.status.idle":"2024-01-15T09:59:24.552371Z","shell.execute_reply.started":"2024-01-15T09:59:24.547953Z","shell.execute_reply":"2024-01-15T09:59:24.551473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = pd.read_csv(PATHS.TRAIN_EVENTS)\nlen(train_events)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:26.647662Z","iopub.execute_input":"2024-01-15T09:59:26.648590Z","iopub.status.idle":"2024-01-15T09:59:26.674517Z","shell.execute_reply.started":"2024-01-15T09:59:26.648555Z","shell.execute_reply":"2024-01-15T09:59:26.673599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_longest_continuous(gdf):\n    \"\"\"Function will find longest subsequence of a training set where there is an event each night\"\"\"\n    c = sorted(list(set(gdf[gdf['event'] == 'onset'].dropna().night.unique()) & set(gdf[gdf['event'] == 'wakeup'].dropna().night.unique())))\n    start = -1\n    end = -1\n    m = 0\n    save = 0,0\n    last = start\n    for x in c:\n        if x == last + 1:\n            end = x\n            last = x\n        else:\n            v = end - start\n            if v > m:\n                save = start, end\n                m = v\n            start = x\n            end = x\n            last = x\n\n    v = end - start\n    if v > m:\n        save = start, end\n        m = v\n\n    #     print(f'Max length is {m} from {save}')\n    #     print(c)\n    return save","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:29.842357Z","iopub.execute_input":"2024-01-15T09:59:29.842981Z","iopub.status.idle":"2024-01-15T09:59:29.850759Z","shell.execute_reply.started":"2024-01-15T09:59:29.842947Z","shell.execute_reply":"2024-01-15T09:59:29.849828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_series = []\ncontinuous = {}\nfor series_id, gdf in train_events.groupby('series_id'):\n    tmp = gdf.dropna()\n    # print(series_id, len(tmp))\n    if len(tmp) == 0:\n        drop_series.append(series_id)\n    else:\n        start, end = get_longest_continuous(gdf)\n        if end - start == 0:\n            drop_series.append(series_id)\n        else:\n            continuous[series_id] = start, end\n\nprint(f'Drop {len(drop_series)}')","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:35.935106Z","iopub.execute_input":"2024-01-15T09:59:35.935443Z","iopub.status.idle":"2024-01-15T09:59:36.619891Z","shell.execute_reply.started":"2024-01-15T09:59:35.935419Z","shell.execute_reply":"2024-01-15T09:59:36.618954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"series_ids = train_events.series_id.unique()\nlen(series_ids)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:38.760788Z","iopub.execute_input":"2024-01-15T09:59:38.761186Z","iopub.status.idle":"2024-01-15T09:59:38.775831Z","shell.execute_reply.started":"2024-01-15T09:59:38.761155Z","shell.execute_reply":"2024-01-15T09:59:38.774865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def compare_predictions(valid_ds, i, net):\n    \"\"\"Utility function to return dataframes of predicted and actual events with their distributions\"\"\"\n    net.eval()\n    with torch.no_grad():\n        X, Y = valid_ds[i]\n        Y = Y.to(CFG.DEVICE, non_blocking=True)\n        pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n\n        h = None\n\n        seq_len = X.shape[0]\n        # max_chunk_size = seq_len\n        for j in range(0, seq_len, max_chunk_size):\n            X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n            y_pred = net(X_chunk)\n            pred[j: j+max_chunk_size, :] = y_pred\n\n            del X_chunk, y_pred\n        # clean_memory()\n    res_df = pd.DataFrame(torch.softmax(pred.cpu(), axis=0).numpy(), columns=['wakeup_val', 'onset_val'])\n    act_df = pd.DataFrame(Y.cpu().numpy(), columns=['wakeup_val', 'onset_val'])\n    return res_df, act_df\n\ndef get_predictions(res_df, target, SIGMA):\n    \"\"\"Function will take a predicted dataframe, and find local maxima to get event location.  The score is determined by the area under the curve\"\"\"\n    q = res_df[target].max() * 0.1\n    tmp = res_df.loc[res_df[target] > q].copy()\n    # print(f'Target max = {q}, len = {len(tmp)}')\n    tmp['gap'] = tmp['step'].diff()\n    tmp = tmp[tmp['gap'] > 5*5]\n    # print(f'Target max = {q}, len = {len(tmp)}')\n    res = []\n    for i in range(len(tmp) + 1):\n        start_i = 0 if i == 0 else tmp['step'].iloc[i-1]\n        end_i = tmp['step'].iloc[i] if i < len(tmp) else res_df['step'].max()\n        v = res_df.loc[(res_df['step'] > start_i) & (res_df['step'] < end_i)]\n        if v[target].max() > q:\n            # print('Locate in ', start_i, end_i)\n            idx = v.idxmax()[target]\n            step = v.loc[idx, 'step']\n            span = 3*SIGMA\n            score = res_df.loc[(res_df['step'] > step - span) & (res_df['step'] < step + span), target].sum()\n            res.append([step, target, score])\n\n    return res\n\ndef create_predictions(test_ds, i, net):\n    \"\"\"Function to create the prediction dataframe\"\"\"\n    net.eval()\n    with torch.no_grad():\n        X = test_ds[i]\n        pred = torch.zeros(X.shape).to(CFG.DEVICE, non_blocking=True)\n\n        h = None\n\n        seq_len = X.shape[0]\n        for j in range(0, seq_len, max_chunk_size):\n            X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n            y_pred = net(X_chunk)\n            pred[j: j+max_chunk_size, :] = y_pred\n\n            del X_chunk, y_pred\n        # clean_memory()\n    res_df = pd.DataFrame(torch.softmax(pred.cpu(), axis=0).numpy(), columns=['wakeup_val', 'onset_val'])\n    return res_df","metadata":{"execution":{"iopub.status.busy":"2024-01-15T11:34:58.289798Z","iopub.execute_input":"2024-01-15T11:34:58.290724Z","iopub.status.idle":"2024-01-15T11:34:58.793725Z","shell.execute_reply.started":"2024-01-15T11:34:58.290682Z","shell.execute_reply":"2024-01-15T11:34:58.792750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDatasetTrain(Dataset):\n    \"\"\"\n    Dataset for Child Mind Sleep States.  We have it output just the raw anglez and enmo variables.\n    \n    :param series_ids: list of series ids in this set\n    :param events: The events dataframe\n    :param len_mult: The total length of the sequence must be a multiple of this integer\n    :param continuous: dictionary of series_id to start and end points, if we want to trim to continuous series\n    :param sigma: The width of the distribution to use for output.\n    \n    \"\"\"\n    def __init__(\n            self,\n            series_ids,\n            events,\n            len_mult,\n            continuous = None,\n            sigma = None\n    ):\n        self.series_ids = series_ids\n        self.continuous = continuous\n        self.len_mult = len_mult\n        if events is not None:\n            self.events = events\n            self.sigma = sigma\n        else:\n            self.events = None\n            self.sigma = None\n\n    def load_data(self, series_id):\n        filename = PATHS.get_series_filename(series_id)\n        data = pd.read_parquet(filename)\n        if self.events is not None:\n            if self.continuous is not None:\n                start, end = self.continuous[series_id]\n            else:\n                start, end = 0, 1000000\n            gap = 6*60*12\n            tmp = self.events[(self.events.series_id == series_id) & (self.events.night >= start) & (self.events.night <= end)]\n            data = data[(data.step > (tmp.step.min() - gap)) & (data.step < (tmp.step.max() + gap))]\n\n            data = data.set_index(['step']).join(tmp.set_index(['step'])[['event', 'night']]).reset_index()\n            norm = 1/ np.sqrt(pi / self.sigma)\n            for evt in ['wakeup', 'onset']:\n                steps = data[data.event == evt]['step'].values\n                col = f'{evt}_val'\n                data[col] = 0.0\n                for i in steps:\n                    x = 0.5*((data.step.astype(np.int64) - i)/self.sigma)**2\n                    data[col] += np.exp(-x)*norm\n                data[col] /= data[col].sum()\n\n        n = int((len(data) // len_mult) * len_mult)\n\n        return data.iloc[:n]\n\n    def __len__(self):\n        return len(self.series_ids)\n\n    def __getitem__(self, index):\n        series_id = self.series_ids[index]\n        data = self.load_data(series_id)\n        X = data[['anglez','enmo']].values.astype(np.float32)\n        X = torch.from_numpy(X)\n        if self.sigma is not None:\n            Y = data[['wakeup_val', 'onset_val']].values.astype(np.float32)\n            Y = torch.from_numpy(Y)\n            return X, Y\n        else:\n            return X","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:44.201450Z","iopub.execute_input":"2024-01-15T09:59:44.202086Z","iopub.status.idle":"2024-01-15T09:59:44.216956Z","shell.execute_reply.started":"2024-01-15T09:59:44.202055Z","shell.execute_reply":"2024-01-15T09:59:44.216063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len_mult = 64\nmax_chunk_size = 24*60*12\nmin_interval = 30\nprint(max_chunk_size)\n\nuseable_series_ids = [s for s in series_ids if s not in drop_series]\nif CFG.DEMO_MODE:\n    useable_series_ids = useable_series_ids[:75]\n\nnp.random.shuffle(useable_series_ids)\nlen(useable_series_ids)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:47.092310Z","iopub.execute_input":"2024-01-15T09:59:47.092670Z","iopub.status.idle":"2024-01-15T09:59:47.100896Z","shell.execute_reply.started":"2024-01-15T09:59:47.092639Z","shell.execute_reply":"2024-01-15T09:59:47.099926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class EncoderLayer(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1):\n        super(EncoderLayer, self).__init__()\n        self.conv = nn.Conv1d(in_channels, out_channels, kernel_size, stride=stride, padding=padding, dilation=dilation)\n        self.ln = nn.LayerNorm(out_channels)\n        self.relu = nn.ReLU(inplace=True)\n        # self.print_shape = print_shape\n    def forward(self, x):\n        x = self.conv(x)\n        x = self.ln(x.transpose(-1,-2))\n        x = self.relu(x.transpose(-1,-2))\n        return x\n\nclass DecoderLayer(nn.Module):\n    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1):\n        super(DecoderLayer, self).__init__()\n        self.conv = nn.Sequential(\n            nn.ConvTranspose1d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, output_padding=1),\n            nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=padding, dilation=dilation),\n            nn.ReLU(inplace=True),\n            nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=padding, dilation=dilation),\n            nn.ReLU(inplace=True)\n        )\n    def forward(self, x):\n        x = self.conv(x)\n        return x\n\nclass LSTM_Encoder(nn.Module):\n    def __init__(self, lstm_channels, num_layers=1):\n        super(LSTM_Encoder, self).__init__()\n        self.lstm = nn.LSTM(input_size=lstm_channels, hidden_size=lstm_channels, batch_first=True)\n        self.fc = nn.Sequential(\n            nn.Linear(lstm_channels, lstm_channels*2),\n            nn.LayerNorm(lstm_channels*2),\n            nn.ReLU(inplace=True),\n            nn.Linear(lstm_channels*2, lstm_channels),\n            nn.LayerNorm(lstm_channels),\n            nn.ReLU(inplace=True)\n        )\n    def forward(self, x):\n        xx, _ = self.lstm(x)\n        x = x + xx\n        x = self.fc(x)\n        return x\n\n# 基于unet的复合模型\nclass UNet(nn.Module):\n\n    def __init__(self, in_channels: int, out_channels: int):\n\n        super().__init__()\n        self.in_channels = in_channels\n        self.out_channels = out_channels\n        num = 4\n        ksize1 = 17\n        ksize2 = 11\n        ksize3 = 7\n\n        # Encoder\n        down_conv1_channels = num*2\n        self.down_conv1 = EncoderLayer(in_channels, down_conv1_channels, kernel_size=ksize1, padding=ksize1//2, stride=2)\n        down_conv2_channels = down_conv1_channels*2\n        self.down_conv2 = EncoderLayer(down_conv1_channels, down_conv2_channels, kernel_size=ksize2, padding=ksize2//2, stride=2)\n        down_conv3_channels = down_conv2_channels*2\n        self.down_conv3 = EncoderLayer(down_conv2_channels, down_conv3_channels, kernel_size=ksize3, padding=ksize3//2, stride=2)\n\n        # LSTM\n        lstm_channels = down_conv3_channels\n        self.lstm = LSTM_Encoder(lstm_channels)\n\n        # Decoder\n        up_conv3_channels = lstm_channels//2\n        self.up_conv3 = DecoderLayer(lstm_channels, up_conv3_channels, kernel_size=ksize3, padding=ksize3//2, stride=2)\n        up_conv2_channels = up_conv3_channels//2\n        self.up_conv2 = DecoderLayer(up_conv3_channels, up_conv2_channels, kernel_size=ksize2, padding=ksize2//2, stride=2)\n        up_conv1_channels = up_conv2_channels//2\n        self.up_conv1 = DecoderLayer(up_conv2_channels, up_conv1_channels, kernel_size=ksize1, padding=ksize1//2, stride=2)\n        # self.concat1 = CropAndConcat()\n        # self.concat2 = CropAndConcat()\n        # self.concat3 = CropAndConcat()\n        # self.concat4 = CropAndConcat()\n        self.final_conv = nn.Conv1d(up_conv1_channels, out_channels, kernel_size=1)\n\n    def forward(self, x: torch.Tensor):\n        x = x.unsqueeze(0).permute(0,2,1)\n        # print(f'Forward: {x.shape}')\n        # Encoder\n        x = self.down_conv1(x)\n        # print(f'down1: {x.shape}')\n        x = self.down_conv2(x)\n        x = self.down_conv3(x)\n        # print(f'Encoder: {x.shape}')\n        # LSTM\n        x = self.lstm(x.permute(0,2,1))\n        x = x.permute(0,2,1)\n        # x = self.lstm(x.permute(1,0))\n        # x = x.permute(1,0)\n        # print(f'LSTM: {x.shape}')\n        # print(x)\n\n        # Decoder\n        x = self.up_conv3(x)\n        # print(f'up1: {x.shape}')\n        # print(x)\n        x = self.up_conv2(x)\n        x = self.up_conv1(x)\n        # print(f'Decoder: {x.shape}')\n        # print(x)\n        x = self.final_conv(x)\n        # print(x)\n        x = x.permute(0,2,1).squeeze(0)\n        # print(f'Final: {x.shape}')\n        return x\n","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:48.865621Z","iopub.execute_input":"2024-01-15T09:59:48.866581Z","iopub.status.idle":"2024-01-15T09:59:48.889773Z","shell.execute_reply.started":"2024-01-15T09:59:48.866546Z","shell.execute_reply":"2024-01-15T09:59:48.888819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_sigma(epoch):\n    if epoch < 4:\n        return 90\n    elif epoch < 7:\n        return 60\n    return 36","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:52.132951Z","iopub.execute_input":"2024-01-15T09:59:52.133308Z","iopub.status.idle":"2024-01-15T09:59:52.138317Z","shell.execute_reply.started":"2024-01-15T09:59:52.133277Z","shell.execute_reply":"2024-01-15T09:59:52.137327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"EPOCHS = 20\nloss_fct = nn.KLDivLoss(reduction='sum')\n\nfor fold, valid_series_ids in enumerate(np.array_split(useable_series_ids, 5)):\n    print(f'Fold {fold}')\n    train_series_ids = [s for s in useable_series_ids if s not in valid_series_ids]\n    # net = GRUNET(arch=arch,out_channels=2, hidden_size=hidden_size, kernel_size=kernel_size, stride=stride,\n    #              dconv_padding=dconv_padding, n_layers=n_layers, bidir=True, print_shape=False).to(CFG.DEVICE)\n    net = UNet(2, 2).to(CFG.DEVICE)\n    learning_rate = 1.e-3\n    clip_val = 2.\n    weight_decay=0.0\n    optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate, weight_decay=weight_decay)\n\n    WARMUP_PROP = 0.1\n    train_size = len(train_series_ids)\n    steps = train_size * EPOCHS\n    warmup_steps = int(steps * WARMUP_PROP)\n    # scheduler = CosineLRScheduler(optimizer,t_initial= steps,warmup_t=warmup_steps, warmup_lr_init=1e-5,lr_min=1e-6,)\n    m = nn.LogSoftmax(dim=0)\n\n    train_loss_history = []\n    valid_loss_history = []\n    learning_rate_history = []\n    mAP_history = []\n\n    # with torch.autograd.detect_anomaly(check_nan=True):\n    for epoch in range(EPOCHS):\n        SIGMA = get_sigma(epoch)\n\n        np.random.shuffle(train_series_ids)\n        train_ds = SleepDatasetTrain(train_series_ids, events=train_events, len_mult=len_mult, continuous=continuous, sigma=SIGMA)\n        valid_ds = SleepDatasetTrain(valid_series_ids, events=train_events, len_mult=len_mult, sigma=SIGMA)\n        print(f'Epoch {epoch}, sigma = {SIGMA}')\n        net.train()\n        train_loss = 0\n\n        for i in tqdm(range(len(train_ds))):\n            net.train()\n            X, Y = train_ds[i]\n            Y = Y.to(CFG.DEVICE, non_blocking=True)\n            # print(X.shape)\n            # print(Y.shape)\n            if not np.isfinite(Y.sum().cpu()):\n                print(f'Nan Target {i}')\n\n            pred = torch.zeros(Y.shape).to(CFG.DEVICE, non_blocking=True)\n            optimizer.zero_grad()\n            # scheduler.step(i+train_size*epoch)\n            h = None\n\n            seq_len = X.shape[0]\n            max_chunk_size = seq_len\n            for j in range(0, seq_len, max_chunk_size):\n                X_chunk = X[j: j + max_chunk_size].float().to(CFG.DEVICE, non_blocking=True)\n                # y_pred, h = net(X_chunk, h)\n                y_pred = net(X_chunk)\n                # h = [hi.detach() for hi in h]\n                pred[j: j+max_chunk_size, :] = y_pred\n\n                del X_chunk, y_pred\n\n            if not np.isfinite(pred.sum().cpu().detach()):\n                print(f'Nan Pred before logsoftmax {i}')\n            pred = m(pred.float())\n            if not np.isfinite(pred.sum().cpu().detach()):\n                print(f'Nan Pred after logsoftmax {i}')\n            loss = loss_fct(pred.float(), Y.float())\n            loss.backward()\n            train_loss += loss.item()\n\n            nn.utils.clip_grad_norm_(net.parameters(), max_norm=clip_val)\n            optimizer.step()\n\n            del pred, loss, Y, X, h\n            # clean_memory()\n            \n        train_loss /= len(train_ds)\n        print(f'Epoch {epoch} train loss = {train_loss}')\n        train_loss_history.append(train_loss)\n        print(f'Learning Rate = {optimizer.param_groups[0][\"lr\"]}')\n        learning_rate_history.append(optimizer.param_groups[0][\"lr\"])\n        net.eval()\n        print('Evaluate Validation Loss and mAP')\n        torch.save(net.state_dict(), f'model_epoch{epoch}.pth')\n\n    torch.save(net.state_dict(), f'model_UNet{fold}.pth')\n#     iplot({'data': [Scatter(y=train_loss_history, name='train'), Scatter(y=valid_loss_history, name='valid')], 'layout': Layout(title=f'KLDivLoss {fold}')})\n#     iplot({'data': [Scatter(y=learning_rate_history, name='lr')], 'layout': Layout(title=f'Learning Rate {fold}')})\n    # iplot({'data': [Scatter(y=mAP_history, name='mAP')], 'layout': Layout(title=f'Event mAP {fold}')})\n    print('Break after 1 to save GPU!')\n    break","metadata":{"execution":{"iopub.status.busy":"2024-01-15T09:59:53.710913Z","iopub.execute_input":"2024-01-15T09:59:53.711250Z","iopub.status.idle":"2024-01-15T10:12:38.223702Z","shell.execute_reply.started":"2024-01-15T09:59:53.711224Z","shell.execute_reply":"2024-01-15T10:12:38.222630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"推理","metadata":{}},{"cell_type":"code","source":"reader = data_reader(demo_mode=False)\nseries = reader.load_data(data_name=\"test_series\")","metadata":{"execution":{"iopub.status.busy":"2024-01-15T14:19:34.827164Z","iopub.execute_input":"2024-01-15T14:19:34.827890Z","iopub.status.idle":"2024-01-15T14:19:35.751890Z","shell.execute_reply.started":"2024-01-15T14:19:34.827853Z","shell.execute_reply":"2024-01-15T14:19:35.750947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.mkdir(PATHS.TEST_DIR)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T14:05:00.330549Z","iopub.execute_input":"2024-01-15T14:05:00.331328Z","iopub.status.idle":"2024-01-15T14:05:00.368399Z","shell.execute_reply.started":"2024-01-15T14:05:00.331293Z","shell.execute_reply":"2024-01-15T14:05:00.367101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\ndata = []\nids = series.series_id.unique()\n\nprint(ids)\n\nfor viz_id in tqdm(ids):\n    viz_targets = []\n    viz_events = events[events.series_id == viz_id]\n    viz_series = series.loc[(series.series_id==viz_id)].copy().reset_index()\n    # viz_series['dt'] = pd.to_datetime(viz_series.timestamp,format = '%Y-%m-%dT%H:%M:%S%z').astype(\"datetime64[ns, UTC-04:00]\")\n    # viz_series['hour'] = viz_series['dt'].dt.hour\n\n#     check = 0\n#     for i in range(len(viz_events)-1):\n#         if viz_events.iloc[i].event =='onset' and viz_events.iloc[i+1].event =='wakeup' and viz_events.iloc[i].night==viz_events.iloc[i+1].night:\n#             start,end = viz_events.timestamp.iloc[i],viz_events.timestamp.iloc[i+1]\n\n#             start_id = viz_series.loc[viz_series.timestamp ==start].index.values[0]\n#             end_id = viz_series.loc[viz_series.timestamp ==end].index.values[0]\n#             viz_targets.append((start_id,end_id))\n#             check+=1\n    targets.append(viz_targets)\n    viz_series['anglez'] = viz_series['anglez'].astype('float')\n    viz_series['enmo'] = viz_series['enmo'].astype('float')\n    viz_series['step'] = viz_series['step'].astype('float')\n#     print(viz_series)\n#     print(PATHS.get_series_filename(viz_id))\n    viz_series[['anglez','enmo','step']].to_parquet(PATHS.get_test_filename(viz_id))\n    # print(type(viz_series[['anglez','enmo','step']]))\n    # print(viz_series[['anglez','enmo','step']])\n    data.append(viz_series[['anglez','enmo','step']]) #you can include features like hour or minnute or second if you want to\n# joblib.dump((targets,data,ids), 'train_data.pkl')\nlen(data)\ntest_series_ids = ids","metadata":{"execution":{"iopub.status.busy":"2024-01-15T14:19:37.793167Z","iopub.execute_input":"2024-01-15T14:19:37.794037Z","iopub.status.idle":"2024-01-15T14:19:38.232512Z","shell.execute_reply.started":"2024-01-15T14:19:37.794001Z","shell.execute_reply":"2024-01-15T14:19:38.231574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SleepDatasetTest(Dataset):\n    \"\"\"\n    Dataset for Child Mind Sleep States.  We have it output just the raw anglez and enmo variables.\n    \n    :param series_ids: list of series ids in this set\n    :param events: The events dataframe\n    :param len_mult: The total length of the sequence must be a multiple of this integer\n    :param continuous: dictionary of series_id to start and end points, if we want to trim to continuous series\n    :param sigma: The width of the distribution to use for output.\n    \n    \"\"\"\n    def __init__(\n            self,\n            series_ids,\n            len_mult\n    ):\n        self.series_ids = series_ids\n        self.continuous = continuous\n        self.len_mult = len_mult\n\n    def load_data(self, series_id):\n        filename = PATHS.get_test_filename(series_id)\n        data = pd.read_parquet(filename)\n        n = int((len(data) // len_mult) * len_mult)\n        return data.iloc[:n]\n\n    def __len__(self):\n        return len(self.series_ids)\n\n    def __getitem__(self, index):\n        series_id = self.series_ids[index]\n        data = self.load_data(series_id)\n        X = data[['anglez','enmo']].values.astype(np.float32)\n        X = torch.from_numpy(X)\n        return X","metadata":{"execution":{"iopub.status.busy":"2024-01-15T11:32:02.777128Z","iopub.execute_input":"2024-01-15T11:32:02.777540Z","iopub.status.idle":"2024-01-15T11:32:02.786736Z","shell.execute_reply.started":"2024-01-15T11:32:02.777505Z","shell.execute_reply":"2024-01-15T11:32:02.785707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"net = UNet(2, 2).to(CFG.DEVICE)\nnet.load_state_dict(torch.load(f'model_epoch{19}.pth'))\ntest_ds = SleepDatasetTest(test_series_ids, len_mult=1)\n\n# for i in range(len(test_ds)):\n#     res_df = create_predictions(test_ds, i, net)\n#     print(res_df)\n#     if (i==0):\n#         res_df.to_csv(\"submission.csv\", index=False, header=True)\n#     else:\n#         res_df.to_csv(\"submission.csv\", index=False, header=False, mode='a+')\n\n\nsubmission = pd.DataFrame()\n\nnet.eval()\nfor i in tqdm(range(len(test_ds))):\n    series_id = test_ds.series_ids[i]\n    print(series_id)\n    \n    X = test_ds[i].to(CFG.DEVICE)\n    seq_len = X.shape[0]\n    pred = net(X.float())\n    pred = torch.softmax(pred.cpu(), axis=0).detach().numpy()\n#     pred = torch.zeros((len(X),2)).half()\n#     for j in range(0, seq_len, seq_len):\n#         y_pred, h = model(X[j: j + max_chunk_size].float())\n#         h = [hi.detach() for hi in h]\n#         pred[j : j + max_chunk_size] = y_pred.detach()\n#         del y_pred;gc.collect()\n#     del h,X;gc.collect()\n#     pred = pred.numpy()\n    \n    series_id = ids[i]\n    \n    days = len(pred)/(17280/12)\n    scores0,scores1 = np.zeros(len(pred),dtype=np.float16),np.zeros(len(pred),dtype=np.float16)\n    for index in range(len(pred)):\n        if pred[index,0]==max(pred[max(0,index-min_interval):index+min_interval,0]):\n            scores0[index] = max(pred[max(0,index-min_interval):index+min_interval,0])\n        if pred[index,1]==max(pred[max(0,index-min_interval):index+min_interval,1]):\n            scores1[index] = max(pred[max(0,index-min_interval):index+min_interval,1])\n    candidates_onset = np.argsort(scores0)[-max(1,round(days)):]\n    candidates_wakeup = np.argsort(scores1)[-max(1,round(days)):]\n#     print(candidates_onset)\n#     print(scores0)\n#     print(scores1)\n    onset = pd.DataFrame({'step':candidates_onset})\n    onset['event'] = 'onset'\n    onset['series_id'] = series_id\n    onset['score']= scores0[candidates_onset]\n    wakeup = pd.DataFrame({'step':candidates_wakeup})\n    wakeup['event'] = 'wakeup'\n    wakeup['series_id'] = series_id\n    wakeup['score']= scores1[candidates_wakeup]\n    submission = pd.concat([submission,onset,wakeup],axis=0)\n    del onset,wakeup,candidates_onset,candidates_wakeup,scores0,scores1,pred,series_id,\n    gc.collect()\nsubmission = submission.sort_values(['series_id','step']).reset_index(drop=True)\nsubmission['row_id'] = submission.index.astype(int)\nsubmission['score'] = submission['score'].fillna(submission['score'].mean())\nsubmission = submission[['row_id','series_id','step','event','score']]\nsubmission.to_csv('submission.csv',index=False)\nprint(submission)","metadata":{"execution":{"iopub.status.busy":"2024-01-15T14:20:05.960193Z","iopub.execute_input":"2024-01-15T14:20:05.960949Z","iopub.status.idle":"2024-01-15T14:20:07.391526Z","shell.execute_reply.started":"2024-01-15T14:20:05.960916Z","shell.execute_reply":"2024-01-15T14:20:07.390437Z"},"trusted":true},"execution_count":null,"outputs":[]}]}