{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n\nimport lightgbm as lgb\n\n\nimport gc\nimport plotly.express as px\nfrom joblib import Parallel, delayed\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"686b6d59-3edb-42a2-a60e-3dc7c6d9f5ab","_cell_guid":"11a27c0e-ce3b-4ea8-80e9-aef25607846c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:01:51.211531Z","iopub.execute_input":"2023-11-13T11:01:51.212313Z","iopub.status.idle":"2023-11-13T11:01:53.517692Z","shell.execute_reply.started":"2023-11-13T11:01:51.21226Z","shell.execute_reply":"2023-11-13T11:01:53.516672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp_from_low =['anglezdiffabs_amplit_720', 'anglezdiffabs_amplit_3600', 'anglezdiffabs_min_3600', 'anglezdiffabs_min_360',\n 'anglezdiffabs_min_720', 'anglezdiffabs_min_1800', 'anglezdiffabs_amplit_1800', 'anglezdiffabs_min_2880',\n 'anglezdiffabs_amplit_2880', 'anglezdiffabs_min_60', 'anglezdiffabs_amplit_60', 'anglezdiffabs_amplit_360',\n 'anglezdiffabs_max_60', 'anglez_diff_60_mean', 'anglezdiffabs', 'anglez_diff_60_max',\n 'enmo_diff_60_max', 'anglez_amplit_60', 'anglezdiffabs_median_60', 'enmo_amplit_720',\n 'anglezdiffabs_diff_60_mean', 'enmo_diff_360_mean', 'anglezdiffabs_diff_60_max', 'anglezdiffabs_max_360',\n 'anglezdiffabs_amplit_60_min', 'anglezdiffabs_diff_360_max', 'anglezdiffabs_diff_360_mean',\n 'anglezdiffabs_var_360', 'anglez_diff_360_mean', 'anglezdiffabs_var_60', 'enmo_amplit_60',\n 'enmo_mean_60', 'anglez_var_720', 'anglezdiffabs_max_720', 'anglez_diff_720_mean',\n 'anglez_diff_360_max', 'anglezdiffabs_var_720', 'enmo_var_60', 'anglez_amplit_360',\n 'anglez_mean_720', 'anglez_amplit_720', 'enmo_max_60', 'anglez_var_360',\n 'enmo_median_60', 'anglez_amplit_60_min', 'anglez_mean_360', 'enmo_max_720',\n 'anglez_mean_60', 'enmo_min_60', 'enmo_amplit_360', 'enmo_diff_720_mean',\n 'enmo_min_720', 'enmo_max_360', 'anglezdiffabs_mean_60', 'enmo_amplit_1800',\n 'enmo_diff_360_max', 'anglezdiffabs_amplit_360_min', 'anglez_max_360', 'anglez_var_60',\n 'anglez_amplit_1800', 'enmo_median_360', 'enmo_min_360', 'anglez_diff_720_max',\n 'anglezdiffabs_mean_360', 'anglez_max_60', 'enmo_diff_60_mean', 'anglez_median_60',\n 'anglez_min_360', 'anglez_max_720', 'anglezdiffabs_diff_720_mean', 'enmo_amplit_60_min',\n 'enmo_var_720', 'anglez_median_720', 'anglez_median_360', 'enmo_diff_720_max',\n 'anglezdiffabs_amplit_720_min', 'anglezdiffabs_max_1800', 'anglezdiffabs_diff_720_max', 'anglez_var_1800',\n 'enmo_median_720', 'anglez_mean_1800', 'enmo_mean_360', 'enmo_min_1800',\n 'enmo_var_1800', 'enmo_median_1800', 'enmo_max_1800', 'enmo_amplit_2880',\n 'enmo_mean_1800', 'anglezdiffabs_diff_1800_mean', 'enmo_min_2880', 'anglez_max_1800',\n 'anglezdiffabs_var_1800', 'anglez_mean_2880', 'anglez_var_2880', 'enmo_diff_1800_mean',\n 'anglezdiffabs_max_2880', 'anglez_amplit_2880', 'enmo_amplit_3600', 'enmo_mean_2880',\n 'enmo_median_2880', 'anglez_amplit_720_min', 'enmo_var_360',\n 'enmo_mean_720', 'anglez_diff_1800_mean', 'anglez_min_60', 'enmo_max_2880',\n 'anglez_min_1800', 'anglez_diff_1800_max', 'enmo_min_3600', 'anglezdiffabs_var_2880',\n 'enmo_var_2880', 'lids', 'enmo_diff_2880_mean', 'anglez_max_2880',\n 'anglez_amplit_3600', 'enmo_median_3600', 'anglez_min_2880', 'enmo_max_3600',\n 'enmo_diff_1800_max', 'anglez_mean_3600', 'anglez_amplit_1800_min', 'anglezdiffabs_diff_2880_mean',\n 'anglezdiffabs_mean_720', 'anglezdiffabs_amplit_1800_min', 'anglez_min_720', 'anglezdiffabs_median_2880',\n 'anglezdiffabs_diff_3600_mean', 'anglez_var_3600', 'anglez_median_1800', 'anglezdiffabs_median_360',\n 'anglezdiffabs_diff_1800_max', 'anglez_diff_2880_mean', 'anglez_mad_3600', 'anglez_max_3600',\n 'anglezdiffabs_mean_1800', 'enmo_amplit_720_min', 'anglez_median_3600', 'enmo_diff_2880_max',\n 'enmo_mean_3600', 'anglez_diff_3600_mean', 'anglez_diff_2880_max', 'anglez_amplit_2880_min',\n 'anglezdiffabs_amplit_2880_min', 'enmo_mad_3600', 'anglezdiffabs_diff_2880_max', 'enmo_var_3600',\n 'enmo_amplit_1800_min', 'anglezdiffabs_diff_3600_max', 'enmo_amplit_360_min', 'anglez_diff_3600_max',\n 'anglez_median_2880', 'anglez_min_3600', 'enmo_diff_3600_mean', 'anglezdiffabs_max_3600',\n 'enmo_diff_3600_max', 'anglezdiffabs_var_3600', 'enmo_amplit_2880_min', 'anglezdiffabs_mean_2880',\n 'anglez_amplit_360_min', 'anglez_amplit_3600_min', 'anglezdiffabs_amplit_3600_min', 'enmo_amplit_3600_min',\n 'anglezdiffabs_median_720', 'anglezdiffabs_median_3600', 'anglezdiffabs_median_1800', 'hour',\n 'anglezdiffabs_mean_3600', 'anglezdiffabs_mad_3600'] #168 features\n\n\ndef drop_feat_poor(df, N):\n\n    N = min(N, len(feat_imp_from_low))\n\n    cols_to_remove = feat_imp_from_low[:N]\n\n    df = df.drop(columns=cols_to_remove)\n\n    return df\n\n\nw_size = 2000\ndrop_poor_no = 165","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:01:53.519771Z","iopub.execute_input":"2023-11-13T11:01:53.520108Z","iopub.status.idle":"2023-11-13T11:01:53.53653Z","shell.execute_reply.started":"2023-11-13T11:01:53.520079Z","shell.execute_reply":"2023-11-13T11:01:53.5352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_parquet('/kaggle/input/merged-data/merged_data.parquet')\n\ndata = data.iloc[::60]\n\ndata = data.rename(columns={'series_id': 'series_id_no'})","metadata":{"_uuid":"31f7dc84-6de7-4d23-80c4-d3f02770a0dc","_cell_guid":"19b95b4f-7738-4ada-b742-b961222cbac1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:01:53.537854Z","iopub.execute_input":"2023-11-13T11:01:53.538178Z","iopub.status.idle":"2023-11-13T11:01:57.397446Z","shell.execute_reply.started":"2023-11-13T11:01:53.538151Z","shell.execute_reply":"2023-11-13T11:01:57.396539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEV = True # 테스트시 True, 제출시 False\n\nif DEV:\n    unique_series_ids = data['series_id_no'].unique()\n    selected_series_ids = unique_series_ids[:10]\n    data = data[data['series_id_no'].isin(selected_series_ids)]","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:01:57.400236Z","iopub.execute_input":"2023-11-13T11:01:57.40072Z","iopub.status.idle":"2023-11-13T11:01:57.43485Z","shell.execute_reply.started":"2023-11-13T11:01:57.400679Z","shell.execute_reply":"2023-11-13T11:01:57.433867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng(df):\n    \n    df['series_id_no'] = df['series_id_no'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df['hour'] = df[\"timestamp\"].dt.hour\n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float32)\n    \n    for col in ['enmo', 'anglez', 'anglezdiffabs']:\n        \n        # periods in seconds        \n        periods = [60,360,720,1800,2880,3600] \n        \n        for n in periods:\n            \n            rol_args = {'window':f'{n+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['median', 'mean', 'max', 'min', 'var']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n            \n            if n == max(periods):\n                df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n            \n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n            df[f'{col}_amplit_{n}_min'] = df[f'{col}_amplit_{n}'].rolling(**rol_args).min().astype(np.float32).values\n            \n            # if col in ['enmo', 'anglez']:\n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).max().astype(np.float32)\n            df[f'{col}_diff_{n}_mean'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).mean().astype(np.float32)\n\n    \n            gc.collect()\n    \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:01:57.436318Z","iopub.execute_input":"2023-11-13T11:01:57.436779Z","iopub.status.idle":"2023-11-13T11:01:57.456324Z","shell.execute_reply.started":"2023-11-13T11:01:57.436748Z","shell.execute_reply":"2023-11-13T11:01:57.454709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef feat_eng_by_id_no(idx):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df  = pd.read_parquet(data, filters=[('series_id_no','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = feat_eng(df)\n    \n    return df\n'''\n\n\n\ndef feat_eng_by_id_no(df, idx):\n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    # Filter the dataframe based on 'idx'\n    filtered_df = df[df['series_id_no'] == idx].copy()\n    filtered_df['event'] = filtered_df['event'].astype(np.int8)\n    filtered_df = feat_eng(filtered_df)\n    \n    return filtered_df","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:01:57.458311Z","iopub.execute_input":"2023-11-13T11:01:57.459547Z","iopub.status.idle":"2023-11-13T11:01:57.468342Z","shell.execute_reply.started":"2023-11-13T11:01:57.459503Z","shell.execute_reply":"2023-11-13T11:01:57.467334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# weird_series = ['31011ade7c0a', 'a596ad0b82aa']\n\n# series_id = [s for s in series_id if s not in weird_series]","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:01:57.469986Z","iopub.execute_input":"2023-11-13T11:01:57.470374Z","iopub.status.idle":"2023-11-13T11:01:57.476928Z","shell.execute_reply.started":"2023-11-13T11:01:57.470333Z","shell.execute_reply":"2023-11-13T11:01:57.47576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n'''\ndata = Parallel(n_jobs=6)(delayed(feat_eng_by_id_no)(i) for i in series_id_no)\ndata = pd.concat(data, ignore_index=True)\n'''\n\n\nseries_id_no = data['series_id_no'].unique()\n\n\ndata = Parallel(n_jobs=6)(delayed(feat_eng_by_id_no)(data, i) for i in series_id_no)\ndata = pd.concat(data, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:01:57.478786Z","iopub.execute_input":"2023-11-13T11:01:57.479173Z","iopub.status.idle":"2023-11-13T11:02:10.860643Z","shell.execute_reply.started":"2023-11-13T11:01:57.479129Z","shell.execute_reply":"2023-11-13T11:02:10.85936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = drop_feat_poor(data, drop_poor_no)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:02:10.862794Z","iopub.execute_input":"2023-11-13T11:02:10.863263Z","iopub.status.idle":"2023-11-13T11:02:10.87291Z","shell.execute_reply.started":"2023-11-13T11:02:10.863221Z","shell.execute_reply":"2023-11-13T11:02:10.871731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:02:10.878824Z","iopub.execute_input":"2023-11-13T11:02:10.879218Z","iopub.status.idle":"2023-11-13T11:02:10.90745Z","shell.execute_reply.started":"2023-11-13T11:02:10.879171Z","shell.execute_reply":"2023-11-13T11:02:10.906163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndef extract_fft_features(conc, window_size):\n    features = []\n    series_id_count = {}  # Dictionary to keep track of counts for each series_id_no\n\n    for window in conc:\n        series_id_no = window[0][0]\n\n        # Increment count for the current series_id_no or initialize if not present\n        if series_id_no in series_id_count:\n            series_id_count[series_id_no] += 1\n        else:\n            series_id_count[series_id_no] = 0\n\n        # Create the modified series_id_no\n        modified_series_id_no = f\"{series_id_no}_{series_id_count[series_id_no]}\"\n\n        fft_enmo = np.fft.fft(window[2])\n        fft_anglez = np.fft.fft(window[1])\n\n        half_len = len(fft_enmo) // 2\n        fft_enmo_half = fft_enmo[:half_len]\n        fft_anglez_half = fft_anglez[:half_len]\n\n        amp_enmo = np.abs(fft_enmo_half)\n        angle_enmo = np.angle(fft_enmo_half)\n        amp_anglez = np.abs(fft_anglez_half)\n        angle_anglez = np.angle(fft_anglez_half)\n\n        indices_enmo = np.argsort(amp_enmo)[::-1][:3]\n        indices_anglez = np.argsort(amp_anglez)[::-1][:3]\n\n        top_amp_enmo = amp_enmo[indices_enmo]\n        top_angle_enmo = angle_enmo[indices_enmo]\n        top_amp_anglez = amp_anglez[indices_anglez]\n        top_angle_anglez = angle_anglez[indices_anglez]\n\n        features.append(np.concatenate(([modified_series_id_no], top_amp_enmo, top_angle_enmo, top_amp_anglez, top_angle_anglez)))\n\n    # Dynamic column names based on window size\n    columns = ['series_id_no'] + [f'{feature}_{window_size}' for feature in [\n        'enmo_amp_top1', 'enmo_amp_top2', 'enmo_amp_top3',\n        'enmo_angle_top1', 'enmo_angle_top2', 'enmo_angle_top3',\n        'anglez_amp_top1', 'anglez_amp_top2', 'anglez_amp_top3',\n        'anglez_angle_top1', 'anglez_angle_top2', 'anglez_angle_top3'\n    ]]\n\n    return pd.DataFrame(features, columns=columns)\n","metadata":{"_uuid":"e5358b6d-a6b6-4a27-ba6e-fd563f6df9d3","_cell_guid":"9be5c1c0-4609-492d-ba49-7e5abe8aac5c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:02:10.908888Z","iopub.execute_input":"2023-11-13T11:02:10.909232Z","iopub.status.idle":"2023-11-13T11:02:10.923339Z","shell.execute_reply.started":"2023-11-13T11:02:10.909205Z","shell.execute_reply":"2023-11-13T11:02:10.922029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndfs = [data.query(f'series_id_no=={i}') for i in range(len(data.series_id_no.unique()))]\n\n'''\ntrain_windows = [np.lib.stride_tricks.sliding_window_view(df, w_size, axis=0) for df in dfs if len(df) > w_size]\n\ntrain_conc = np.concatenate(train_windows)\n\nsliced_array = train_conc[:, :, 0]\n\ntrain = pd.DataFrame(sliced_array)\n\ntrain.columns = data.columns\n'''","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:02:10.925005Z","iopub.execute_input":"2023-11-13T11:02:10.925375Z","iopub.status.idle":"2023-11-13T11:02:10.963379Z","shell.execute_reply.started":"2023-11-13T11:02:10.925344Z","shell.execute_reply":"2023-11-13T11:02:10.962048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w_sizes = [2000, 1000, 500, 250]\nX_fft_dict = {}\n\nfor w_size in w_sizes:\n    windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, [1, 4, 5]], w_size, axis=0) for df in dfs if len(df) > w_size]\n    conc = np.concatenate(windows)\n    X_fft_one = extract_fft_features(conc, w_size)\n    X_fft_dict[w_size] = X_fft_one\n","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:02:10.964656Z","iopub.execute_input":"2023-11-13T11:02:10.964961Z","iopub.status.idle":"2023-11-13T11:02:37.829814Z","shell.execute_reply.started":"2023-11-13T11:02:10.964934Z","shell.execute_reply":"2023-11-13T11:02:37.828341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_fft_dict","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:02:37.832009Z","iopub.execute_input":"2023-11-13T11:02:37.832644Z","iopub.status.idle":"2023-11-13T11:02:37.874703Z","shell.execute_reply.started":"2023-11-13T11:02:37.832595Z","shell.execute_reply":"2023-11-13T11:02:37.873462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from functools import reduce\n\n# Assuming 'results' is your dictionary with the DataFrames\ndataframes = list(X_fft_dict.values())  # Get the list of DataFrames from the dictionary\n\n# Use functools.reduce to iteratively join the DataFrames on 'series_id_no'\nX_fft = reduce(lambda left, right: pd.merge(left, right, on='series_id_no', how='outer'), dataframes)\n\n# Drop any rows that have missing data\nX_fft.dropna(inplace=True)\n\n\n# Select all columns except 'series_id_no'\ncols_to_convert = X_fft.columns.drop('series_id_no')\n\n# Apply pd.to_numeric to these columns only\nX_fft[cols_to_convert] = X_fft[cols_to_convert].apply(pd.to_numeric, errors='coerce')","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:04:45.70856Z","iopub.execute_input":"2023-11-13T11:04:45.708972Z","iopub.status.idle":"2023-11-13T11:04:47.551907Z","shell.execute_reply.started":"2023-11-13T11:04:45.708936Z","shell.execute_reply":"2023-11-13T11:04:47.550671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_fft","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:04:48.993561Z","iopub.execute_input":"2023-11-13T11:04:48.994019Z","iopub.status.idle":"2023-11-13T11:04:49.052515Z","shell.execute_reply.started":"2023-11-13T11:04:48.993975Z","shell.execute_reply":"2023-11-13T11:04:49.051127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_feat = data.copy()  # Create a copy of the original DataFrame\nseries_id_count = {}  # Dictionary to keep track of counts for each series_id_no\n\nfor index, row in X_feat.iterrows():\n    series_id_no = row['series_id_no']\n\n    # Increment count for the current series_id_no or initialize if not present\n    if series_id_no in series_id_count:\n        series_id_count[series_id_no] += 1\n    else:\n        series_id_count[series_id_no] = 0\n\n    # Create the modified series_id_no\n    modified_series_id_no = f\"{series_id_no}_{series_id_count[series_id_no]}\"\n    X_feat.at[index, 'series_id_no'] = modified_series_id_no\n\n# Display the modified DataFrame\nprint(X_feat)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:05:13.894934Z","iopub.execute_input":"2023-11-13T11:05:13.895363Z","iopub.status.idle":"2023-11-13T11:05:18.302955Z","shell.execute_reply.started":"2023-11-13T11:05:13.89533Z","shell.execute_reply":"2023-11-13T11:05:18.301598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Merge the DataFrames on 'series_id_no'\nX = pd.merge(X_feat, X_fft, on='series_id_no', how='inner')\n\n\n# Assuming X is your merged DataFrame\nX['series_id_no'] = X['series_id_no'].astype(str).apply(lambda x: x.split('_')[0])\n\n# Convert the modified 'series_id_no' back to its original numeric type if needed\nX['series_id_no'] = pd.to_numeric(X['series_id_no'])\n\n# Display the modified DataFrame\nprint(X)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:05:25.610704Z","iopub.execute_input":"2023-11-13T11:05:25.611164Z","iopub.status.idle":"2023-11-13T11:05:25.730242Z","shell.execute_reply.started":"2023-11-13T11:05:25.611129Z","shell.execute_reply":"2023-11-13T11:05:25.72836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['series_id_no', 'step', 'timestamp']\n\n\ny = X['event']\n\nX = X.drop(columns=drop_cols+['event'])","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:05:42.520499Z","iopub.execute_input":"2023-11-13T11:05:42.520981Z","iopub.status.idle":"2023-11-13T11:05:42.535985Z","shell.execute_reply.started":"2023-11-13T11:05:42.520941Z","shell.execute_reply":"2023-11-13T11:05:42.534587Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params1 = {    \n    'boosting_type':'gbdt',\n    'num_leaves':31,\n    'max_depth':6,\n    'learning_rate':0.02,\n    'n_estimators':100,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.9,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n             }\nimport xgboost as xgb\n\n\n# Training classifier\n\n\nmodel = lgb.LGBMClassifier(random_state=42, **lgb_params1)\n\n\n'''\nformer parameter values\n\nlgb_params1 = {    \n    'boosting_type':'gbdt',\n    'num_leaves':40,\n    'max_depth':6,\n    'learning_rate':0.02,\n    'n_estimators':1000,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.9,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n             }\nimport xgboost as xgb\n\n\ndefault parameter values\n    \n    'boosting_type': 'gbdt', # Gradient Boosting Decision Tree\n    'num_leaves': 31, # Max number of leaves in one tree\n    'max_depth': -1, # No limit, unless specified\n    'learning_rate': 0.1, # The efficiency of the model learning\n    'n_estimators': 100, # Number of boosted trees to fit\n    'subsample_for_bin': 200000, # Number of samples for constructing bins\n    'objective': None, # The loss to minimize during optimization\n    'class_weight': None, # Weights associated with classes\n    'min_split_gain': 0., # The minimum loss reduction required to make a further partition on a leaf node of the tree\n    'min_child_weight': 1e-3, # Minimum sum of instance weight (hessian) needed in a child (leaf)\n    'min_child_samples': 20, # Minimum number of data needed in a child (leaf)\n    'subsample': 1., # Subsample ratio of the training instance\n    'subsample_freq': 0, # Frequence of subsample, <=0 means no enable\n    'colsample_bytree': 1., # Subsample ratio of columns when constructing each tree\n    'reg_alpha': 0., # L1 regularization term on weights\n    'reg_lambda': 0., # L2 regularization term on weights\n    'random_state': None, # Random number seed\n    # ... there are other parameters as well which have default values\n    \n\n\nan advice from a team member\n\n추가로, lgb_params가 너무 깊습니다.\n먼저 lgb_params의 leaves와 estimators 를 좀 낮춘 후 진행하시는 걸 추천드립니다.\n파라미터에 대한 최적화는 일단 스코어링이 진행되고 나서 값을 조정해도 되니, 최대한 간단한 값으로 하시는 걸 추천드립니다.\n'''\n","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:05:50.864512Z","iopub.execute_input":"2023-11-13T11:05:50.864977Z","iopub.status.idle":"2023-11-13T11:05:50.909448Z","shell.execute_reply.started":"2023-11-13T11:05:50.864942Z","shell.execute_reply":"2023-11-13T11:05:50.908084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel.fit(X, y)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:05:51.819049Z","iopub.execute_input":"2023-11-13T11:05:51.820472Z","iopub.status.idle":"2023-11-13T11:05:53.557426Z","shell.execute_reply.started":"2023-11-13T11:05:51.820421Z","shell.execute_reply":"2023-11-13T11:05:53.555846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = []\n\nfeat_imp = pd.Series(model.booster_.feature_importance(importance_type='gain'), index=X.columns).sort_values()\nfeats.append(feat_imp)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:05:54.997641Z","iopub.execute_input":"2023-11-13T11:05:54.998049Z","iopub.status.idle":"2023-11-13T11:05:55.006262Z","shell.execute_reply.started":"2023-11-13T11:05:54.998011Z","shell.execute_reply":"2023-11-13T11:05:55.004383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp = pd.Series(pd.concat(feats, axis=1).mean(axis=1), index=feats[0].index).sort_values()\nprint('Columns with poor contribution', feat_imp[feat_imp<0.001].index, sep='\\n')\nfig = px.bar(x=feat_imp, y=feat_imp.index, orientation='h')\nfig.show()\n\n#according to this analysis, seems like 'amplitude' feature has more contribution than 'angle'","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:05:56.916716Z","iopub.execute_input":"2023-11-13T11:05:56.917253Z","iopub.status.idle":"2023-11-13T11:05:59.182021Z","shell.execute_reply.started":"2023-11-13T11:05:56.91721Z","shell.execute_reply":"2023-11-13T11:05:59.18068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp.sort_values().head(10)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:06:30.650783Z","iopub.execute_input":"2023-11-13T11:06:30.651722Z","iopub.status.idle":"2023-11-13T11:06:30.665679Z","shell.execute_reply.started":"2023-11-13T11:06:30.651678Z","shell.execute_reply":"2023-11-13T11:06:30.664528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del X, y\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:06:31.806238Z","iopub.execute_input":"2023-11-13T11:06:31.806892Z","iopub.status.idle":"2023-11-13T11:06:31.977185Z","shell.execute_reply.started":"2023-11-13T11:06:31.806855Z","shell.execute_reply":"2023-11-13T11:06:31.975752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, [1, 4, 5]], w_size, axis=0) for df in dfs if len(df) > w_size]\n\nconc = np.concatenate(windows)\n\nX_fft = extract_fft_features(conc)","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:03:07.292998Z","iopub.status.idle":"2023-11-13T11:03:07.293863Z","shell.execute_reply.started":"2023-11-13T11:03:07.293634Z","shell.execute_reply":"2023-11-13T11:03:07.293656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['series_id_no', 'step', 'timestamp']\n\nX_feat = train\nX_fft = extract_fft_features(conc)\n\nX = pd.concat([X_feat, X_fft], axis=1)\n\nX = X.drop(columns=drop_cols+['event'])\n\nfor col in X.columns:\n    X[col] = pd.to_numeric(X[col], errors='coerce')","metadata":{"_uuid":"f454069a-2ecc-4cf8-81d3-38a23f83bfe3","_cell_guid":"2b6febbb-9ff1-4901-99ec-84cd46e6bad3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.295181Z","iopub.status.idle":"2023-11-13T11:03:07.296112Z","shell.execute_reply.started":"2023-11-13T11:03:07.295899Z","shell.execute_reply":"2023-11-13T11:03:07.29592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_fft","metadata":{"execution":{"iopub.status.busy":"2023-11-13T11:03:07.297252Z","iopub.status.idle":"2023-11-13T11:03:07.298067Z","shell.execute_reply.started":"2023-11-13T11:03:07.297848Z","shell.execute_reply":"2023-11-13T11:03:07.297871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate y for model.fit(X, y)\n\nevent_windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, 3], w_size, axis=0) for df in dfs if len(df) > w_size] ####################################### window size\n\nevent_conc = np.concatenate(event_windows)\n\nfirst_values = event_conc[:, 0]\n\ntarget = pd.DataFrame(first_values, columns=['event'])\n\ny = target.iloc[:,0] #y 그냥 넣으면 안되는지 확인 y = target['event'] 하면 안되는지 확인\n\ngc.collect()","metadata":{"_uuid":"d65cb9ad-11b9-4859-83fc-6fabea0a5b03","_cell_guid":"1aed7f4b-5989-488d-bf19-e7c19f78090c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.301144Z","iopub.status.idle":"2023-11-13T11:03:07.301684Z","shell.execute_reply.started":"2023-11-13T11:03:07.301448Z","shell.execute_reply":"2023-11-13T11:03:07.301472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params1 = {    \n    'boosting_type':'gbdt',\n    'num_leaves':31,\n    'max_depth':6,\n    'learning_rate':0.02,\n    'n_estimators':100,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.9,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n             }\nimport xgboost as xgb\n\n\n# Training classifier\n\n\nmodel = lgb.LGBMClassifier(random_state=42, **lgb_params1)\n\n\n'''\nformer parameter values\n\nlgb_params1 = {    \n    'boosting_type':'gbdt',\n    'num_leaves':40,\n    'max_depth':6,\n    'learning_rate':0.02,\n    'n_estimators':1000,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.9,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n             }\nimport xgboost as xgb\n\n\ndefault parameter values\n    \n    'boosting_type': 'gbdt', # Gradient Boosting Decision Tree\n    'num_leaves': 31, # Max number of leaves in one tree\n    'max_depth': -1, # No limit, unless specified\n    'learning_rate': 0.1, # The efficiency of the model learning\n    'n_estimators': 100, # Number of boosted trees to fit\n    'subsample_for_bin': 200000, # Number of samples for constructing bins\n    'objective': None, # The loss to minimize during optimization\n    'class_weight': None, # Weights associated with classes\n    'min_split_gain': 0., # The minimum loss reduction required to make a further partition on a leaf node of the tree\n    'min_child_weight': 1e-3, # Minimum sum of instance weight (hessian) needed in a child (leaf)\n    'min_child_samples': 20, # Minimum number of data needed in a child (leaf)\n    'subsample': 1., # Subsample ratio of the training instance\n    'subsample_freq': 0, # Frequence of subsample, <=0 means no enable\n    'colsample_bytree': 1., # Subsample ratio of columns when constructing each tree\n    'reg_alpha': 0., # L1 regularization term on weights\n    'reg_lambda': 0., # L2 regularization term on weights\n    'random_state': None, # Random number seed\n    # ... there are other parameters as well which have default values\n    \n\n\nan advice from a team member\n\n추가로, lgb_params가 너무 깊습니다.\n먼저 lgb_params의 leaves와 estimators 를 좀 낮춘 후 진행하시는 걸 추천드립니다.\n파라미터에 대한 최적화는 일단 스코어링이 진행되고 나서 값을 조정해도 되니, 최대한 간단한 값으로 하시는 걸 추천드립니다.\n'''\n","metadata":{"_uuid":"f598095f-5162-4a98-90d0-4fe9f08a1e8b","_cell_guid":"301f5001-124e-4aa2-8ca5-7b80c75bb14a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.303609Z","iopub.status.idle":"2023-11-13T11:03:07.304065Z","shell.execute_reply.started":"2023-11-13T11:03:07.303849Z","shell.execute_reply":"2023-11-13T11:03:07.303871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel.fit(X, y)","metadata":{"_uuid":"ef230a84-a6a5-4b5f-ab03-a4aa94ab6f29","_cell_guid":"0c6c1dd7-e069-49a0-a340-01f9cc5730a6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.305828Z","iopub.status.idle":"2023-11-13T11:03:07.306248Z","shell.execute_reply.started":"2023-11-13T11:03:07.306055Z","shell.execute_reply":"2023-11-13T11:03:07.306074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = []\n\nfeat_imp = pd.Series(model.booster_.feature_importance(importance_type='gain'), index=X.columns).sort_values()\nfeats.append(feat_imp)","metadata":{"_uuid":"23f3745f-aacc-410e-bfe6-1f9a6e67304a","_cell_guid":"ea5dfcee-cad4-4b5b-b748-7a5836242f7c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.308377Z","iopub.status.idle":"2023-11-13T11:03:07.308846Z","shell.execute_reply.started":"2023-11-13T11:03:07.308624Z","shell.execute_reply":"2023-11-13T11:03:07.308645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp = pd.Series(pd.concat(feats, axis=1).mean(axis=1), index=feats[0].index).sort_values()\nprint('Columns with poor contribution', feat_imp[feat_imp<0.001].index, sep='\\n')\nfig = px.bar(x=feat_imp, y=feat_imp.index, orientation='h')\nfig.show()\n\n#according to this analysis, seems like 'amplitude' feature has more contribution than 'angle'","metadata":{"_uuid":"c280f03d-1d26-4288-a398-a4ff7798d321","_cell_guid":"99565f82-b9af-4c3a-8198-759a9e8f36aa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.310893Z","iopub.status.idle":"2023-11-13T11:03:07.312047Z","shell.execute_reply.started":"2023-11-13T11:03:07.311817Z","shell.execute_reply":"2023-11-13T11:03:07.31184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp.sort_values().head(10)","metadata":{"_uuid":"5b4c5867-c85f-43b0-b60c-99c73381d506","_cell_guid":"36272cc1-16c0-49e2-96bf-162ccf997d19","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.313212Z","iopub.status.idle":"2023-11-13T11:03:07.314089Z","shell.execute_reply.started":"2023-11-13T11:03:07.313873Z","shell.execute_reply":"2023-11-13T11:03:07.313895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del X, y\ngc.collect()","metadata":{"_uuid":"69d7b97a-a3c8-4bc2-8bee-21005fd4121e","_cell_guid":"bbfbb946-105b-4003-be76-10b8ac046b49","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.315925Z","iopub.status.idle":"2023-11-13T11:03:07.316707Z","shell.execute_reply.started":"2023-11-13T11:03:07.316495Z","shell.execute_reply":"2023-11-13T11:03:07.316519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_events(idx, classifier, file='test_series.parquet') :\n    \n    test  = pd.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/{file}',\n                    filters=[('series_id','=',idx)])\n    \n    codes, uniques = pd.factorize(test['series_id'])\n    test.insert(test.columns.get_loc('series_id') + 1, 'series_id_no', codes)\n    test.drop('series_id', axis=1, inplace=True)    \n    \n    \n    train = feat_eng(test)    \n    train = drop_feat_poor(train, drop_poor_no) ##########\n    \n    dfs = [train.query(f'series_id_no=={i}') for i in range(len(train.series_id_no.unique()))]\n    feat_train_windows = [np.lib.stride_tricks.sliding_window_view(df, w_size, axis=0) for df in dfs if len(df) > w_size] ####################################### window size\n    feat_train_conc = np.concatenate(feat_train_windows)\n    \n    sliced_array = feat_train_conc[:, :, 0]\n    \n    feat_train = pd.DataFrame(sliced_array)\n    \n    column_names = data.columns.drop('event')\n    \n    feat_train.columns = column_names\n\n    \n\n    X_feat = feat_train    \n    \n    \n    \n\n    fft_windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, [3, 4]], w_size, axis=0) for df in dfs if len(df) > w_size] ####################################### window size\n    fft_conc = np.concatenate(fft_windows)\n    \n    \n    X_fft = extract_fft_features(fft_conc)\n    \n    \n\n    X_test = pd.concat([X_feat, X_fft], axis=1)\n\n    X_test = X_test.drop(columns=drop_cols)\n\n    for col in X_test.columns:\n        X_test[col] = pd.to_numeric(X[col], errors='coerce')\n    \n    \n    \n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]    \n    \n    \n    \n\n    windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, :3], w_size, axis=0) for df in dfs if len(df) > w_size] ####################################### window size\n\n    flattened_data = [(window[0][0], window[1][0], window[2][0]) for window in windows[0]]\n    \n    test = pd.DataFrame(flattened_data, columns=['series_id', 'step', 'timestamp'])\n    \n    if test['timestamp'].dtype != '<M8[ns]':  # '<M8[ns]' is numpy's datetime64\n        test['timestamp'] = pd.to_datetime(test['timestamp'])\n    \n    \n    \n    \n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    \n    test['pred_diff'] = test['prediction'].diff()\n    \n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    test = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('timestamp')\n\n    return test","metadata":{"_uuid":"9a8d385c-77e4-410b-9985-7d83cb914f4e","_cell_guid":"5e863e37-674e-4ba2-a5ac-f614ab05f9b7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.318115Z","iopub.status.idle":"2023-11-13T11:03:07.319027Z","shell.execute_reply.started":"2023-11-13T11:03:07.318808Z","shell.execute_reply":"2023-11-13T11:03:07.318832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_sub = ['series_id','step','event','score']\n\nseries_id  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id'])\nseries_id = series_id.series_id.unique()\n\ntests = []\n\nfor idx in series_id: \n\n    test =  get_events(idx, model)\n    tests.append(test[cols_sub])","metadata":{"_uuid":"8edf989d-08d7-4ccf-afd2-17a6e9dbeb13","_cell_guid":"dad39ed9-8159-4d8a-9ecf-18362a6daa2e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.320741Z","iopub.status.idle":"2023-11-13T11:03:07.32118Z","shell.execute_reply.started":"2023-11-13T11:03:07.320974Z","shell.execute_reply":"2023-11-13T11:03:07.320996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat(tests, ignore_index=True).reset_index(names='row_id')\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"_uuid":"e978c88f-3785-400f-b53b-12775dc93da6","_cell_guid":"136594b9-bac9-4749-ba5c-e1d30199f85a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-13T11:03:07.323228Z","iopub.status.idle":"2023-11-13T11:03:07.323963Z","shell.execute_reply.started":"2023-11-13T11:03:07.323734Z","shell.execute_reply":"2023-11-13T11:03:07.323757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}