{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30558,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#import os\n#import py7zr\n#import sys\n#from tqdm import tqdm\nfrom contextlib import contextmanager\nimport gc\nimport numpy as np\nimport pandas as pd\n#import matplotlib.pyplot as plt\nfrom scipy.signal import convolve, convolve2d\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\n\n#import tensorflow as tf\nfrom keras import layers, models, utils, optimizers\nfrom keras.layers import (\n    Input,\n    Dense,\n    Concatenate,\n    LSTM,\n    Conv1D,\n    UpSampling1D,\n    Flatten,\n    MaxPooling1D,\n    Dropout,\n)\nfrom keras.models import Model, Sequential\nfrom keras.utils import to_categorical\nfrom keras.optimizers import RMSprop\nimport os\nimport re\nimport pyarrow.parquet as pq\n\nfrom datetime import datetime\nfrom datetime import timedelta\nfrom dateutil import parser\nfrom tqdm import tqdm\n\nTRAIN_DATA_PATH = \"/kaggle/input/child-mind-institute-detect-sleep-states/train_series.parquet\"\n\ndf_train_original = pd.read_parquet(TRAIN_DATA_PATH)\n#print(df_train_original)\n\nTRAIN_EVENTS_PATH = \"/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv\"\ndf_train_events_original = pd.read_csv(TRAIN_EVENTS_PATH)\n#print(df_train_events_original)\n\ndf_grouped_train = df_train_original.groupby(\"series_id\")\ndf_grouped_events = df_train_events_original.groupby(\"series_id\")\nserires_id_original = df_grouped_events.groups.keys()\n\ndel df_train_original,df_train_events_original\ngc.collect()\n\nseries_id = []\nfor key in serires_id_original:\n    df = df_grouped_events.get_group(key)\n    if df[\"timestamp\"].isna().any():\n        pass\n    else:\n        series_id.append(key)\n#print(f\"Valid Series ID: {len(df_grouped_events)} => {len(series_id)}\")\nDAY_LENGTH = 17280\n#print(f\"DAY_LENGTH = {DAY_LENGTH}\")\n\n# series_id_test = series_id[:5]\n# series_id = series_id[6:]\n\ndf = {}\nfor id in series_id:\n    df[id] = {\n        \"train\": df_grouped_train.get_group(id),\n        \"events\": df_grouped_events.get_group(id),\n    }\n\ndel df_grouped_train,df_grouped_events\ngc.collect()\n\ndef to_datetime(x):\n    tz = parser.parse(x[\"train\"][\"timestamp\"].iloc[0]).tzinfo\n    x[\"train\"].loc[:, \"timestamp\"] = pd.to_datetime(x[\"train\"][\"timestamp\"], utc=True)\n    x[\"events\"].loc[:, \"timestamp\"] = pd.to_datetime(x[\"events\"][\"timestamp\"], utc=True)\n    x[\"train\"].set_index(\"timestamp\", inplace=True)\n    x[\"events\"].set_index(\"timestamp\", inplace=True)\n    array_df_train = x[\"train\"].resample(\n        \"D\", origin=datetime(1970, 1, 1, 12, 0, tzinfo=tz)\n    )\n    array_df_events = x[\"events\"].resample(\n        \"D\", origin=datetime(1970, 1, 1, 12, 0, tzinfo=tz)\n    )\n    train_list = [item for name, item in array_df_train][1:-1]\n    events_list = [item for name, item in array_df_events][1:-1]\n    if len(train_list) * len(events_list) == 0:\n        return {}\n    else:\n        return {\n            \"train\": pd.concat(train_list),\n            \"events\": pd.concat(events_list),\n        }\n\n\ndf = {key: to_datetime(items) for key, items in df.items()}\ndf = {key: value for key, value in df.items() if value}\n\n\ndef set_onset_wakeup(x):\n    status = 0 if x[\"events\"].iloc[0][\"event\"] == \"onset\" else 1\n    event_counter = 0\n    event_counter_max = len(x[\"events\"][\"event\"])\n    onset_wakeup = []\n\n    for row in x[\"train\"].itertuples():\n        if event_counter < event_counter_max:\n            if x[\"events\"].index[event_counter] < row[0]:\n                event_counter += 1\n                status = (status + 1) % 2\n        onset_wakeup.append(status)\n    x[\"train\"].loc[:, \"onset_wakeup\"] = onset_wakeup\n    return x\n\n\ndf = {key: set_onset_wakeup(items) for key, items in df.items()}\n\ndf = pd.concat([item[\"train\"] for key, item in df.items()], axis=0)\n\ndf[\"enmo\"] = df[\"enmo\"] / df[\"enmo\"].max()\ndf[\"anglez\"] = df[\"anglez\"] / df[\"anglez\"].max()\n\nFEATURES = [\"enmo\", \"anglez\"]  # timestamp列は使っていません\nTARGET = \"onset_wakeup\"\nLOAD_FOLDERS = -1  # 読み込むseries_id数　あまり大きくすると止まります\nTEST_DAY = 5  # テストデータとして使用する日数 データの分割、結果表示に使用されます\n\nFEATURES = [\"enmo\", \"anglez\"]\nTARGET = \"onset_wakeup\"\n\nX_train, X_test, y_train, y_test = train_test_split(\n    df[FEATURES],\n    df[TARGET],\n    test_size=TEST_DAY / round(df.shape[0] / 17820, 0),\n    shuffle=False,\n    random_state=42,\n)\n\nX_train = X_train.to_numpy().reshape(-1, 1, 2)\nX_test = X_test.to_numpy().reshape(-1, 1, 2)\ny_train = y_train.to_numpy().reshape(-1, 1, 1)\ny_test = y_test.to_numpy().reshape(-1, 1, 1)\n\nINPUT_WINDOW_SIZE = 1200\nFILTERS = 100  # filters: フィルター（ニューロン）の数\nKERNEL_SIZE = 120  # kernel_size: フィルターのウィンドウサイズ（フィルターが見る入力のサイズ）10min=120\nACTIVATION = \"relu\"  # 活性化関数 relu sigmoid softmax tanh\n# POOLING_SIZE = 1  # プーリングサイズ　特徴量マップをダウンサイズする時の窓サイズ\n# STRIDES_SIZE = 1  # ストライドサイズ　特徴量マップをダウンサイズする時のステップサイズ\nOPTIMIZER = \"adam\"  # 最適化関数 adam\nEPOCH = 10  # 学習の繰り返し回数\n\nX_train = X_train.reshape(-1, 2)\nX_test = X_test.reshape(-1, 2)\n\n# X_train = X_train / np.max(X_train, axis=0)\n\n# 標準化のためのスケーラーの初期化\n\nscaler = StandardScaler()\n\n# トレーニングデータを用いて平均と標準偏差を計算し、それを用いてデータを標準化\n\nX_train_scaled = scaler.fit_transform(X_train)\n\n# テストデータも同様に標準化\n\nX_test_scaled = scaler.transform(X_test)\n\nX_train, X_test = X_train_scaled.reshape(-1, 1, 2), X_test_scaled.reshape(-1, 1, 2)\n\ndel X_train_scaled, X_test_scaled\ngc.collect()\n\n\narray = X_train.reshape(-1, 2)\nkernel = np.ones((INPUT_WINDOW_SIZE, 1)) / INPUT_WINDOW_SIZE  # カーネルの定義（ウィンドウサイズ3x1）\narray = convolve2d(array, kernel, mode=\"same\")\nX_train = array.reshape(-1, 1, 2)\n\narray = X_test.reshape(-1, 2)\n# kernel = np.ones((INPUT_WINDOW_SIZE, 1)) / INPUT_WINDOW_SIZE  # カーネルの定義（ウィンドウサイズ3x1）\narray = convolve2d(array, kernel, mode=\"same\")\nX_test = array.reshape(-1, 1, 2)\n#plt.plot(X_test.reshape(-1, 2))\n\n# モデルの作成\nmodel = Sequential()\n\n# Conv1D層の追加\n# filters: フィルター（ニューロン）の数\n# kernel_size: フィルターのウィンドウサイズ（フィルターが見る入力のサイズ）\n# activation: 活性化関数\n# input_shape: 入力データの形状（特徴量の数, チャネル数）(hyperParamater)\n# padding: 入力データと出力データのサイズが同じになる\nmodel.add(\n    Conv1D(\n        filters=FILTERS,\n        kernel_size=KERNEL_SIZE,\n        activation=ACTIVATION,\n        input_shape=(1, 2),\n        padding=\"same\",\n    )\n)\n\n# MaxPooling1D層の追加　特徴マップのダウンサンプリングを行う層(反応性を鈍くする)\n# pool_size: プーリングウィンドウのサイズ\n# model.add(MaxPooling1D(pool_size=POOLING_SIZE, strides=STRIDES_SIZE))\n\n# Flatten層の追加 畳みこまれたデータが1次元のベクトルに変換される層\nmodel.add(Flatten())\n\n# 出力層の追加\nmodel.add(Dense(100, activation=\"sigmoid\"))\n\n\n# 出力層の追加\n# クラス分類の場合は、出力のユニット数はクラスの数に対応\nmodel.add(Dense(1, activation=\"sigmoid\"))\n\n# モデルのコンパイル\nmodel.compile(optimizer=OPTIMIZER, loss=\"binary_crossentropy\", metrics=[\"accuracy\"])\n\nmodel.fit(\n    X_train, y_train, epochs=EPOCH, batch_size=DAY_LENGTH, verbose=0\n)  # , validation_data=(X_val, y_val)\n\nloss, accuracy = model.evaluate(X_test, y_test)\n\nresult = model.predict(X_test)\n\nOUTPUT_WINDOW_SIZE = 1200  # 出力結果に適用する移動平均の窓幅\nBinarization = 1  # 二値化処理　01\n\n# resultを1次元に変換\nresult2 = np.squeeze(result)\n# 移動平均の計算\nresult2 = np.convolve(\n    result2, np.ones(OUTPUT_WINDOW_SIZE) / OUTPUT_WINDOW_SIZE, mode=\"valid\"\n)\nif Binarization:\n    # 2値化処理(平均値を閾値とする)\n    # result2 = np.where(result > np.mean(result2), 1, 0)\n    result2 = np.where(result2 > 0.7, 1, 0)\n# answerのインデックスをリセット(df読み込み時の結合に17820周期になってる)\nanswer = pd.Series(y_test.flatten()).reset_index(drop=True)\n\n#plt.plot(result)\n#plt.plot(result2)\n#plt.plot(answer)\n\ndel X_train, X_test, y_train, y_test \ngc.collect()\n\n# # inputとして想定されるデータ\n# df_test = pd.concat([df_grouped_train.get_group(id) for id in series_id_test])\n# df_test.reset_index(drop=True)\n# df_test\n\nTRAIN_TEST_PATH = \"/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet\"\ndf_test_original = pd.read_parquet(TRAIN_TEST_PATH)  # ここがTRAIN_DATA_PATHになっていた。\n# →変更後test_listでエラーになる。\n#print(df_test_original)\ndf_test = df_test_original\ndel df_test_original \ngc.collect()\n\n# timestamp列をstring->datetimeに変換\ndef totimestamp(df_test):\n    df_test[\"timestamp\"] = pd.to_datetime(df_test[\"timestamp\"], utc=True)\n    return df_test\n\n\n# timestamp列をインデックスに設定\ndef setIndex(df_test):\n    df_test.set_index(\"timestamp\", inplace=True)\n    return df_test\n\n\ntz = parser.parse(df_test[\"timestamp\"].iloc[0]).tzinfo\ndf_test = totimestamp(df_test)\ndf_test = setIndex(df_test)\ndf_test\n\n\n# テストデータの17280*n行からの不足分を0埋めで補完する\ndef make_testlist(df_test):\n    df_test_groupby_id = df_test.groupby(\"series_id\")\n    key_list = list(df_test_groupby_id.groups.keys())\n\n    test_list = []\n    df_test_groupby_id.apply(lambda x: test_list.append(x))\n\n    for i in range(len(test_list)):\n        num = DAY_LENGTH - len(test_list[i]) % DAY_LENGTH\n        len_ = len(test_list[i])\n        df_ = pd.DataFrame(\n            {\n                \"series_id\": [test_list[i][\"series_id\"].iloc[0]] * num,\n                \"step\": [index + len_ for index in range(num)],\n                \"anglez\": [0] * num,\n                \"enmo\": [0] * num,\n            }\n        )\n        test_list[i] = pd.concat([test_list[i], df_], axis=0, ignore_index=True)\n        print(test_list[i])\n    return test_list\n\n\ntest_list = make_testlist(df_test)\n\n\n# 一日ごとに移動平均を処理する\ndef arrays_(array):\n    nparray = model.predict(\n        array[[\"enmo\", \"anglez\"]].values.reshape(-1, 1, 2)\n    ).flatten()\n    print(nparray)\n    nparray = np.convolve(\n        nparray,\n        np.ones(OUTPUT_WINDOW_SIZE) / OUTPUT_WINDOW_SIZE,\n        mode=\"same\",\n    )\n    nparray = nparray / nparray.max()\n    nparray = np.where(\n        nparray > 0.5,\n        1,\n        0,\n    )\n    df_ = pd.DataFrame(\n        nparray,\n        columns=[\"onset_wakeup\"],\n        index=array.index,\n    )\n    df = pd.concat([df_, pd.DataFrame(array)], ignore_index=False, axis=1)\n    #         df[\"onset_wakeup\"].plot()\n    #         arrays.append(df)\n    return df\n\n\ndef series_(df_series):\n    print(len(df_series))\n    chunks = np.array_split(df_series, len(df_series) // DAY_LENGTH)\n    arrays = []\n    for array in chunks:\n        arrays.append(arrays_(array))\n    #     series.append(arrays)\n    return arrays\n\n\nOUTPUT_WINDOW_SIZE = 120\n#窓幅　1200→120に変更\nseries = []\nfor df_series in test_list:\n    series.append(series_(df_series))\n    \n# 入眠・起床を検知する\nbuf2 = 0\ndf_detected_event = pd.DataFrame(columns=[\"series_id\", \"step\", \"event\"])\nprint(df_detected_event)\n\ndef detect(df_detected_event, row, buf2):\n    buf = row[\"onset_wakeup\"]\n\n    if buf != buf2:\n        df_detected_event = pd.concat(\n            [\n                df_detected_event,\n                pd.Series(\n                    [\n                        row[\"series_id\"],\n                        row[\"step\"],\n                        \"wakeup\" if row[\"onset_wakeup\"] == 1 else \"onset\",\n                    ],\n                    index=[\"series_id\", \"step\", \"event\"],\n                )\n                .to_frame()\n                .T,\n            ]\n        )\n        buf2 = buf\n    return df_detected_event, buf2\nfor arrays in series:\n    for df in arrays:\n        for index, row in df.iterrows():\n            df_detected_event, buf2 = detect(df_detected_event, row, buf2)\ndf_detected_event[\"score\"] = 0.5\ndf_detected_event\n\n# 出力処理\n#df_detected_event[\"row_id\"] = range(len(df_detected_event))\n#df_detected_event.insert(0, 'row_id', ['0', '1', '2', '3', '4', '5'])\ndf_detected_event.insert(0, 'row_id', range(len(df_detected_event)))\n\ndf_detected_event.to_csv(\n    \"/kaggle/working/submission.csv\",\n    index=False,\n    header=[\"row_id\", \"series_id\", \"step\", \"event\", \"score\"],\n)","metadata":{"execution":{"iopub.status.busy":"2023-12-12T06:17:55.427502Z","iopub.execute_input":"2023-12-12T06:17:55.427910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}