{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n\nimport lightgbm as lgb\nimport gc\nimport plotly.express as px\nfrom joblib import Parallel, delayed\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"686b6d59-3edb-42a2-a60e-3dc7c6d9f5ab","_cell_guid":"11a27c0e-ce3b-4ea8-80e9-aef25607846c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:54:31.739045Z","iopub.execute_input":"2023-11-12T03:54:31.739600Z","iopub.status.idle":"2023-11-12T03:54:35.942161Z","shell.execute_reply.started":"2023-11-12T03:54:31.739562Z","shell.execute_reply":"2023-11-12T03:54:35.940921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp_from_low =['anglezdiffabs_amplit_720', 'anglezdiffabs_amplit_3600', 'anglezdiffabs_min_3600', 'anglezdiffabs_min_360',\n 'anglezdiffabs_min_720', 'anglezdiffabs_min_1800', 'anglezdiffabs_amplit_1800', 'anglezdiffabs_min_2880',\n 'anglezdiffabs_amplit_2880', 'anglezdiffabs_min_60', 'anglezdiffabs_amplit_60', 'anglezdiffabs_amplit_360',\n 'anglezdiffabs_max_60', 'anglez_diff_60_mean', 'anglezdiffabs', 'anglez_diff_60_max',\n 'enmo_diff_60_max', 'anglez_amplit_60', 'anglezdiffabs_median_60', 'enmo_amplit_720',\n 'anglezdiffabs_diff_60_mean', 'enmo_diff_360_mean', 'anglezdiffabs_diff_60_max', 'anglezdiffabs_max_360',\n 'anglezdiffabs_amplit_60_min', 'anglezdiffabs_diff_360_max', 'anglezdiffabs_diff_360_mean', 'enmo',\n 'anglezdiffabs_var_360', 'anglez_diff_360_mean', 'anglezdiffabs_var_60', 'enmo_amplit_60',\n 'enmo_mean_60', 'anglez_var_720', 'anglezdiffabs_max_720', 'anglez_diff_720_mean',\n 'anglez_diff_360_max', 'anglezdiffabs_var_720', 'enmo_var_60', 'anglez_amplit_360',\n 'anglez_mean_720', 'anglez_amplit_720', 'enmo_max_60', 'anglez_var_360',\n 'enmo_median_60', 'anglez_amplit_60_min', 'anglez_mean_360', 'enmo_max_720',\n 'anglez_mean_60', 'enmo_min_60', 'enmo_amplit_360', 'enmo_diff_720_mean',\n 'enmo_min_720', 'enmo_max_360', 'anglezdiffabs_mean_60', 'enmo_amplit_1800',\n 'enmo_diff_360_max', 'anglezdiffabs_amplit_360_min', 'anglez_max_360', 'anglez_var_60',\n 'anglez_amplit_1800', 'enmo_median_360', 'enmo_min_360', 'anglez_diff_720_max',\n 'anglezdiffabs_mean_360', 'anglez_max_60', 'enmo_diff_60_mean', 'anglez_median_60',\n 'anglez_min_360', 'anglez_max_720', 'anglezdiffabs_diff_720_mean', 'enmo_amplit_60_min',\n 'enmo_var_720', 'anglez_median_720', 'anglez_median_360', 'enmo_diff_720_max',\n 'anglezdiffabs_amplit_720_min', 'anglezdiffabs_max_1800', 'anglezdiffabs_diff_720_max', 'anglez_var_1800',\n 'enmo_median_720', 'anglez_mean_1800', 'enmo_mean_360', 'enmo_min_1800',\n 'enmo_var_1800', 'enmo_median_1800', 'enmo_max_1800', 'enmo_amplit_2880',\n 'enmo_mean_1800', 'anglezdiffabs_diff_1800_mean', 'enmo_min_2880', 'anglez_max_1800',\n 'anglezdiffabs_var_1800', 'anglez_mean_2880', 'anglez_var_2880', 'enmo_diff_1800_mean',\n 'anglezdiffabs_max_2880', 'anglez_amplit_2880', 'enmo_amplit_3600', 'enmo_mean_2880',\n 'enmo_median_2880', 'anglez', 'anglez_amplit_720_min', 'enmo_var_360',\n 'enmo_mean_720', 'anglez_diff_1800_mean', 'anglez_min_60', 'enmo_max_2880',\n 'anglez_min_1800', 'anglez_diff_1800_max', 'enmo_min_3600', 'anglezdiffabs_var_2880',\n 'enmo_var_2880', 'lids', 'enmo_diff_2880_mean', 'anglez_max_2880',\n 'anglez_amplit_3600', 'enmo_median_3600', 'anglez_min_2880', 'enmo_max_3600',\n 'enmo_diff_1800_max', 'anglez_mean_3600', 'anglez_amplit_1800_min', 'anglezdiffabs_diff_2880_mean',\n 'anglezdiffabs_mean_720', 'anglezdiffabs_amplit_1800_min', 'anglez_min_720', 'anglezdiffabs_median_2880',\n 'anglezdiffabs_diff_3600_mean', 'anglez_var_3600', 'anglez_median_1800', 'anglezdiffabs_median_360',\n 'anglezdiffabs_diff_1800_max', 'anglez_diff_2880_mean', 'anglez_mad_3600', 'anglez_max_3600',\n 'anglezdiffabs_mean_1800', 'enmo_amplit_720_min', 'anglez_median_3600', 'enmo_diff_2880_max',\n 'enmo_mean_3600', 'anglez_diff_3600_mean', 'anglez_diff_2880_max', 'anglez_amplit_2880_min',\n 'anglezdiffabs_amplit_2880_min', 'enmo_mad_3600', 'anglezdiffabs_diff_2880_max', 'enmo_var_3600',\n 'enmo_amplit_1800_min', 'anglezdiffabs_diff_3600_max', 'enmo_amplit_360_min', 'anglez_diff_3600_max',\n 'anglez_median_2880', 'anglez_min_3600', 'enmo_diff_3600_mean', 'anglezdiffabs_max_3600',\n 'enmo_diff_3600_max', 'anglezdiffabs_var_3600', 'enmo_amplit_2880_min', 'anglezdiffabs_mean_2880',\n 'anglez_amplit_360_min', 'anglez_amplit_3600_min', 'anglezdiffabs_amplit_3600_min', 'enmo_amplit_3600_min',\n 'anglezdiffabs_median_720', 'anglezdiffabs_median_3600', 'anglezdiffabs_median_1800', 'hour',\n 'anglezdiffabs_mean_3600', 'anglezdiffabs_mad_3600']\n\n\ndef drop_feat_poor(df, N):\n\n    N = min(N, len(feat_imp_from_low))\n\n    cols_to_remove = feat_imp_from_low[:N]\n\n    df = df.drop(columns=cols_to_remove)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:54:35.944448Z","iopub.execute_input":"2023-11-12T03:54:35.944757Z","iopub.status.idle":"2023-11-12T03:54:35.958643Z","shell.execute_reply.started":"2023-11-12T03:54:35.944732Z","shell.execute_reply":"2023-11-12T03:54:35.957693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = pd.read_parquet('/kaggle/input/merged-data/merged_data.parquet')\n\ndata = data.iloc[::60]\n\ndata = data.rename(columns={'series_id': 'series_id_no'})","metadata":{"_uuid":"31f7dc84-6de7-4d23-80c4-d3f02770a0dc","_cell_guid":"19b95b4f-7738-4ada-b742-b961222cbac1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:54:35.960058Z","iopub.execute_input":"2023-11-12T03:54:35.960499Z","iopub.status.idle":"2023-11-12T03:54:47.194723Z","shell.execute_reply.started":"2023-11-12T03:54:35.960452Z","shell.execute_reply":"2023-11-12T03:54:47.193223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEV = False # 테스트시 True, 제출시 False\n\nif DEV:\n    unique_series_ids = data['series_id_no'].unique()\n    selected_series_ids = unique_series_ids[:10]\n    data = data[data['series_id_no'].isin(selected_series_ids)]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:54:47.197843Z","iopub.execute_input":"2023-11-12T03:54:47.198289Z","iopub.status.idle":"2023-11-12T03:54:47.238282Z","shell.execute_reply.started":"2023-11-12T03:54:47.198236Z","shell.execute_reply":"2023-11-12T03:54:47.237108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def feat_eng(df):\n    \n    df['series_id_no'] = df['series_id_no'].astype('category')\n    df['timestamp'] = pd.to_datetime(df['timestamp']).apply(lambda t: t.tz_localize(None))\n    df['hour'] = df[\"timestamp\"].dt.hour\n    \n    df.sort_values(['timestamp'], inplace=True)\n    df.set_index('timestamp', inplace=True)\n    \n    df['lids'] = np.maximum(0., df['enmo'] - 0.02)\n    df['lids'] = df['lids'].rolling(f'{120*5}s', center=True, min_periods=1).agg('sum')\n    df['lids'] = 100 / (df['lids'] + 1)\n    df['lids'] = df['lids'].rolling(f'{360*5}s', center=True, min_periods=1).agg('mean').astype(np.float32)\n    \n    df[\"enmo\"] = (df[\"enmo\"]*1000).astype(np.int16)\n    df[\"anglez\"] = df[\"anglez\"].astype(np.int16)\n    df[\"anglezdiffabs\"] = df[\"anglez\"].diff().abs().astype(np.float32)\n    \n    for col in ['enmo', 'anglez', 'anglezdiffabs']:\n        \n        # periods in seconds        \n        periods = [60,360,720,1800,2880,3600] \n        \n        for n in periods:\n            \n            rol_args = {'window':f'{n+5}s', 'min_periods':10, 'center':True}\n            \n            for agg in ['median', 'mean', 'max', 'min', 'var']:\n                df[f'{col}_{agg}_{n}'] = df[col].rolling(**rol_args).agg(agg).astype(np.float32).values\n                gc.collect()\n            \n            if n == max(periods):\n                df[f'{col}_mad_{n}'] = (df[col] - df[f'{col}_median_{n}']).abs().rolling(**rol_args).median().astype(np.float32)\n            \n            df[f'{col}_amplit_{n}'] = df[f'{col}_max_{n}']-df[f'{col}_min_{n}']\n            df[f'{col}_amplit_{n}_min'] = df[f'{col}_amplit_{n}'].rolling(**rol_args).min().astype(np.float32).values\n            \n            # if col in ['enmo', 'anglez']:\n            df[f'{col}_diff_{n}_max'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).max().astype(np.float32)\n            df[f'{col}_diff_{n}_mean'] = df[f'{col}_max_{n}'].diff().abs().rolling(**rol_args).mean().astype(np.float32)\n\n    \n            gc.collect()\n    \n    df.reset_index(inplace=True)\n    df.dropna(inplace=True)\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:54:47.239947Z","iopub.execute_input":"2023-11-12T03:54:47.240581Z","iopub.status.idle":"2023-11-12T03:54:47.260367Z","shell.execute_reply.started":"2023-11-12T03:54:47.240542Z","shell.execute_reply":"2023-11-12T03:54:47.259190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ndef feat_eng_by_id_no(idx):\n    \n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    df  = pd.read_parquet(data, filters=[('series_id_no','=',idx)])\n    df['awake'] = df['awake'].astype(np.int8)\n    df = feat_eng(df)\n    \n    return df\n'''\n\n\n\ndef feat_eng_by_id_no(df, idx):\n    from warnings import simplefilter \n    simplefilter(action=\"ignore\", category=pd.errors.PerformanceWarning)\n    \n    # Filter the dataframe based on 'idx'\n    filtered_df = df[df['series_id_no'] == idx].copy()\n    filtered_df['event'] = filtered_df['event'].astype(np.int8)\n    filtered_df = feat_eng(filtered_df)\n    \n    return filtered_df","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:54:47.262099Z","iopub.execute_input":"2023-11-12T03:54:47.262503Z","iopub.status.idle":"2023-11-12T03:54:47.276741Z","shell.execute_reply.started":"2023-11-12T03:54:47.262465Z","shell.execute_reply":"2023-11-12T03:54:47.275511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# weird_series = ['31011ade7c0a', 'a596ad0b82aa']\n\n# series_id = [s for s in series_id if s not in weird_series]","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:54:47.278141Z","iopub.execute_input":"2023-11-12T03:54:47.278466Z","iopub.status.idle":"2023-11-12T03:54:47.289255Z","shell.execute_reply.started":"2023-11-12T03:54:47.278440Z","shell.execute_reply":"2023-11-12T03:54:47.288314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n'''\ndata = Parallel(n_jobs=6)(delayed(feat_eng_by_id_no)(i) for i in series_id_no)\ndata = pd.concat(data, ignore_index=True)\n'''\n\n\nseries_id_no = data['series_id_no'].unique()\n\n\ndata = Parallel(n_jobs=6)(delayed(feat_eng_by_id_no)(data, i) for i in series_id_no)\ndata = pd.concat(data, ignore_index=True)","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:54:47.292139Z","iopub.execute_input":"2023-11-12T03:54:47.293219Z","iopub.status.idle":"2023-11-12T03:54:59.698397Z","shell.execute_reply.started":"2023-11-12T03:54:47.293169Z","shell.execute_reply":"2023-11-12T03:54:59.696707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data = drop_feat_poor(data, 150) ##########","metadata":{"execution":{"iopub.status.busy":"2023-11-12T03:54:59.700786Z","iopub.execute_input":"2023-11-12T03:54:59.701324Z","iopub.status.idle":"2023-11-12T03:54:59.717821Z","shell.execute_reply.started":"2023-11-12T03:54:59.701277Z","shell.execute_reply":"2023-11-12T03:54:59.716663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [data.query(f'series_id_no=={i}') for i in range(len(data.series_id_no.unique()))]\n\ntrain_windows = [np.lib.stride_tricks.sliding_window_view(df, 20, axis=0) for df in dfs if len(df) > 20]\n\ntrain_conc = np.concatenate(train_windows)\n\n\nsliced_array = train_conc[:, :, 0]\n\ntrain = pd.DataFrame(sliced_array)\n\ntrain.columns = data.columns","metadata":{"_uuid":"0b462dee-124a-46c3-9ef9-8f7a6b264633","_cell_guid":"a2ec7970-9cb4-448d-ba6a-fd7667f40d40","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:54:59.719468Z","iopub.execute_input":"2023-11-12T03:54:59.719801Z","iopub.status.idle":"2023-11-12T03:55:00.627080Z","shell.execute_reply.started":"2023-11-12T03:54:59.719757Z","shell.execute_reply":"2023-11-12T03:55:00.625997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, [4, 5]], 20, axis=0) for df in dfs if len(df) > 20]\n\nconc = np.concatenate(windows)","metadata":{"_uuid":"3e608c68-45d7-4244-aad5-f1986a017aa0","_cell_guid":"314d35ee-4a6d-4ed5-920f-85b8da64e3da","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:00.631160Z","iopub.execute_input":"2023-11-12T03:55:00.631447Z","iopub.status.idle":"2023-11-12T03:55:00.650572Z","shell.execute_reply.started":"2023-11-12T03:55:00.631424Z","shell.execute_reply":"2023-11-12T03:55:00.649464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract_fft_features(df, window_size=20, chunk_size=10000):\n\n    new_data = np.zeros((df.shape[0], 12))\n\n    # Calculate the number of chunks needed\n    number_of_chunks = df.shape[0] // chunk_size + (1 if df.shape[0] % chunk_size else 0)\n\n    for chunk in range(number_of_chunks):\n        # Determine the start and end index for the chunk\n        start = chunk * chunk_size\n        end = min(start + chunk_size, df.shape[0])  # Ensure we don't go past the end of the array\n\n        # Apply FFT on 'enmo' and 'anglez' columns for the current chunk\n        fft_enmo = np.fft.fft(df[start:end, 0, :], axis=1)\n        fft_anglez = np.fft.fft(df[start:end, 1, :], axis=1)\n\n        # Calculate amplitudes and angles\n        amp_enmo = 2 * np.abs(fft_enmo) / window_size\n        angle_enmo = np.angle(fft_enmo)\n        amp_anglez = 2 * np.abs(fft_anglez) / window_size\n        angle_anglez = np.angle(fft_anglez)\n\n        # Extract the top 3 amplitudes and corresponding angles for 'enmo' and 'anglez'\n        top3_amp_enmo_indices = np.argsort(-amp_enmo, axis=1)[:, :3]\n        top3_angle_enmo_indices = np.argsort(-angle_enmo, axis=1)[:, :3]\n        top3_amp_anglez_indices = np.argsort(-amp_anglez, axis=1)[:, :3]\n        top3_angle_anglez_indices = np.argsort(-angle_anglez, axis=1)[:, :3]\n\n        # Fill the new_data with the top 3 amplitudes and angles\n        for i in range(end - start):\n            idx = start + i  # Index in the full array\n            new_data[idx, 0:3] = amp_enmo[i, top3_amp_enmo_indices[i]]\n            new_data[idx, 3:6] = angle_enmo[i, top3_angle_enmo_indices[i]]\n            new_data[idx, 6:9] = amp_anglez[i, top3_amp_anglez_indices[i]]\n            new_data[idx, 9:12] = angle_anglez[i, top3_angle_anglez_indices[i]]\n\n    # Convert the new_data array into a pandas DataFrame\n    column_names = [\n        'enmo_amp_top1', 'enmo_amp_top2', 'enmo_amp_top3',\n        'enmo_angle_top1', 'enmo_angle_top2', 'enmo_angle_top3',\n        'anglez_amp_top1', 'anglez_amp_top2', 'anglez_amp_top3',\n        'anglez_angle_top1', 'anglez_angle_top2', 'anglez_angle_top3'\n    ]\n    train = pd.DataFrame(new_data, columns=column_names)\n    \n    return train","metadata":{"_uuid":"e5358b6d-a6b6-4a27-ba6e-fd563f6df9d3","_cell_guid":"9be5c1c0-4609-492d-ba49-7e5abe8aac5c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:00.652235Z","iopub.execute_input":"2023-11-12T03:55:00.652718Z","iopub.status.idle":"2023-11-12T03:55:00.668507Z","shell.execute_reply.started":"2023-11-12T03:55:00.652674Z","shell.execute_reply":"2023-11-12T03:55:00.666971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"drop_cols = ['series_id_no', 'step', 'timestamp']\n\nX_feat = train\nX_fft = extract_fft_features(conc)\n\nX = pd.concat([X_feat, X_fft], axis=1)\n\nX = X.drop(columns=drop_cols+['event'])\n\nfor col in X.columns:\n    X[col] = pd.to_numeric(X[col], errors='coerce')","metadata":{"_uuid":"f454069a-2ecc-4cf8-81d3-38a23f83bfe3","_cell_guid":"2b6febbb-9ff1-4901-99ec-84cd46e6bad3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:00.670053Z","iopub.execute_input":"2023-11-12T03:55:00.670470Z","iopub.status.idle":"2023-11-12T03:55:01.522580Z","shell.execute_reply.started":"2023-11-12T03:55:00.670434Z","shell.execute_reply":"2023-11-12T03:55:01.521328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# generate y for model.fit(X, y)\n\nevent_windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, 3], 20, axis=0) for df in dfs if len(df) > 20]\n\nevent_conc = np.concatenate(event_windows)\n\nfirst_values = event_conc[:, 0]\n\ntarget = pd.DataFrame(first_values, columns=['event'])\n\ny = target.iloc[:,0] #y 그냥 넣으면 안되는지 확인 y = target['event'] 하면 안되는지 확인\n\ngc.collect()","metadata":{"_uuid":"d65cb9ad-11b9-4859-83fc-6fabea0a5b03","_cell_guid":"1aed7f4b-5989-488d-bf19-e7c19f78090c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:01.524267Z","iopub.execute_input":"2023-11-12T03:55:01.524566Z","iopub.status.idle":"2023-11-12T03:55:01.656253Z","shell.execute_reply.started":"2023-11-12T03:55:01.524541Z","shell.execute_reply":"2023-11-12T03:55:01.655108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params1 = {    \n    'boosting_type':'gbdt',\n    'num_leaves':40,\n    'max_depth':6,\n    'learning_rate':0.02,\n    'n_estimators':1000,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.9,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n             }\nimport xgboost as xgb\n\n\n# Training classifier\n\n\nmodel = lgb.LGBMClassifier(random_state=42, **lgb_params1)\n\n\n'''\nformer parameter values\n\nlgb_params1 = {    \n    'boosting_type':'gbdt',\n    'num_leaves':40,\n    'max_depth':6,\n    'learning_rate':0.02,\n    'n_estimators':1000,\n    'subsample_for_bin':200000,\n    'min_child_weight':0.001,\n    'min_child_samples':20,\n    'subsample':0.9,\n#     'colsample_bytree':0.7,\n    'reg_alpha':0.05,\n    'reg_lambda':0.05,\n             }\nimport xgboost as xgb\n\n\ndefault parameter values\n    \n    'boosting_type': 'gbdt', # Gradient Boosting Decision Tree\n    'num_leaves': 31, # Max number of leaves in one tree\n    'max_depth': -1, # No limit, unless specified\n    'learning_rate': 0.1, # The efficiency of the model learning\n    'n_estimators': 100, # Number of boosted trees to fit\n    'subsample_for_bin': 200000, # Number of samples for constructing bins\n    'objective': None, # The loss to minimize during optimization\n    'class_weight': None, # Weights associated with classes\n    'min_split_gain': 0., # The minimum loss reduction required to make a further partition on a leaf node of the tree\n    'min_child_weight': 1e-3, # Minimum sum of instance weight (hessian) needed in a child (leaf)\n    'min_child_samples': 20, # Minimum number of data needed in a child (leaf)\n    'subsample': 1., # Subsample ratio of the training instance\n    'subsample_freq': 0, # Frequence of subsample, <=0 means no enable\n    'colsample_bytree': 1., # Subsample ratio of columns when constructing each tree\n    'reg_alpha': 0., # L1 regularization term on weights\n    'reg_lambda': 0., # L2 regularization term on weights\n    'random_state': None, # Random number seed\n    # ... there are other parameters as well which have default values\n    \n\n\nan advice from a team member\n\n추가로, lgb_params가 너무 깊습니다.\n먼저 lgb_params의 leaves와 estimators 를 좀 낮춘 후 진행하시는 걸 추천드립니다.\n파라미터에 대한 최적화는 일단 스코어링이 진행되고 나서 값을 조정해도 되니, 최대한 간단한 값으로 하시는 걸 추천드립니다.\n'''\n","metadata":{"_uuid":"f598095f-5162-4a98-90d0-4fe9f08a1e8b","_cell_guid":"301f5001-124e-4aa2-8ca5-7b80c75bb14a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:01.657800Z","iopub.execute_input":"2023-11-12T03:55:01.658131Z","iopub.status.idle":"2023-11-12T03:55:01.845447Z","shell.execute_reply.started":"2023-11-12T03:55:01.658105Z","shell.execute_reply":"2023-11-12T03:55:01.844249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nmodel.fit(X, y)","metadata":{"_uuid":"ef230a84-a6a5-4b5f-ab03-a4aa94ab6f29","_cell_guid":"0c6c1dd7-e069-49a0-a340-01f9cc5730a6","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:01.847002Z","iopub.execute_input":"2023-11-12T03:55:01.847460Z","iopub.status.idle":"2023-11-12T03:55:13.215532Z","shell.execute_reply.started":"2023-11-12T03:55:01.847421Z","shell.execute_reply":"2023-11-12T03:55:13.214054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feats = []\n\nfeat_imp = pd.Series(model.booster_.feature_importance(importance_type='gain'), index=X.columns).sort_values()\nfeats.append(feat_imp)","metadata":{"_uuid":"23f3745f-aacc-410e-bfe6-1f9a6e67304a","_cell_guid":"ea5dfcee-cad4-4b5b-b748-7a5836242f7c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:13.217510Z","iopub.execute_input":"2023-11-12T03:55:13.217981Z","iopub.status.idle":"2023-11-12T03:55:13.226504Z","shell.execute_reply.started":"2023-11-12T03:55:13.217939Z","shell.execute_reply":"2023-11-12T03:55:13.225196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp = pd.Series(pd.concat(feats, axis=1).mean(axis=1), index=feats[0].index).sort_values()\nprint('Columns with poor contribution', feat_imp[feat_imp<0.001].index, sep='\\n')\nfig = px.bar(x=feat_imp, y=feat_imp.index, orientation='h')\nfig.show()\n\n#according to this analysis, seems like 'amplitude' feature has more contribution than 'angle'","metadata":{"_uuid":"c280f03d-1d26-4288-a398-a4ff7798d321","_cell_guid":"99565f82-b9af-4c3a-8198-759a9e8f36aa","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:13.227945Z","iopub.execute_input":"2023-11-12T03:55:13.228291Z","iopub.status.idle":"2023-11-12T03:55:14.989275Z","shell.execute_reply.started":"2023-11-12T03:55:13.228264Z","shell.execute_reply":"2023-11-12T03:55:14.988120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feat_imp.sort_values().head(10)","metadata":{"_uuid":"5b4c5867-c85f-43b0-b60c-99c73381d506","_cell_guid":"36272cc1-16c0-49e2-96bf-162ccf997d19","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:14.990739Z","iopub.execute_input":"2023-11-12T03:55:14.991470Z","iopub.status.idle":"2023-11-12T03:55:15.008098Z","shell.execute_reply.started":"2023-11-12T03:55:14.991439Z","shell.execute_reply":"2023-11-12T03:55:15.004262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del X, y\ngc.collect()","metadata":{"_uuid":"69d7b97a-a3c8-4bc2-8bee-21005fd4121e","_cell_guid":"bbfbb946-105b-4003-be76-10b8ac046b49","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:15.009770Z","iopub.execute_input":"2023-11-12T03:55:15.010231Z","iopub.status.idle":"2023-11-12T03:55:15.151701Z","shell.execute_reply.started":"2023-11-12T03:55:15.010202Z","shell.execute_reply":"2023-11-12T03:55:15.150578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_events(idx, classifier, file='test_series.parquet') :\n    \n    test  = pd.read_parquet(f'/kaggle/input/child-mind-institute-detect-sleep-states/{file}',\n                    filters=[('series_id','=',idx)])\n    \n    id = test['series_id'].iloc[0] ######################추가\n    codes, uniques = pd.factorize(test['series_id'])\n    test.insert(test.columns.get_loc('series_id') + 1, 'series_id_no', codes)\n    test.drop('series_id', axis=1, inplace=True)    \n    \n    \n    train = feat_eng(test)    \n    train = drop_feat_poor(train, 150) ##########\n    \n    dfs = [train.query(f'series_id_no=={i}') for i in range(len(train.series_id_no.unique()))]\n    feat_train_windows = [np.lib.stride_tricks.sliding_window_view(df, 20, axis=0) for df in dfs if len(df) > 20]\n    feat_train_conc = np.concatenate(feat_train_windows)\n    \n    sliced_array = feat_train_conc[:, :, 0]\n    \n    feat_train = pd.DataFrame(sliced_array)\n    \n    column_names = data.columns.drop('event')\n    \n    feat_train.columns = column_names\n\n    \n\n    X_feat = feat_train    \n    \n    \n    \n\n    fft_windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, [3, 4]], 20, axis=0) for df in dfs if len(df) > 20]\n    fft_conc = np.concatenate(fft_windows)\n    \n    \n    X_fft = extract_fft_features(fft_conc)\n    \n    \n\n    X_test = pd.concat([X_feat, X_fft], axis=1)\n\n    X_test = X_test.drop(columns=drop_cols)\n\n    for col in X_test.columns:\n        X_test[col] = pd.to_numeric(X[col], errors='coerce')\n    \n    \n    \n    preds, probs = classifier.predict(X_test), classifier.predict_proba(X_test)[:, 1]    \n    \n    \n    \n\n    windows = [np.lib.stride_tricks.sliding_window_view(df.iloc[:, :3], 20, axis=0) for df in dfs if len(df) > 20]\n\n    flattened_data = [(window[0][0], window[1][0], window[2][0]) for window in windows[0]]\n    \n    test = pd.DataFrame(flattened_data, columns=['series_id', 'step', 'timestamp'])\n    \n    if test['timestamp'].dtype != '<M8[ns]':  # '<M8[ns]' is numpy's datetime64\n        test['timestamp'] = pd.to_datetime(test['timestamp'])\n        \n    test['series_id'] = id #################추가\n    \n    \n    \n    test['prediction'] = preds\n    test['prediction'] = test['prediction'].rolling(360+1, center=True).median()\n    test['probability'] = probs\n    \n    test = test[test['prediction']!=2]\n    \n    test.loc[test['prediction']==0, 'probability'] = 1-test.loc[test['prediction']==0, 'probability']\n    test['score'] = test['probability'].rolling(60*12*5, center=True, min_periods=10).mean().bfill().ffill()\n\n    \n    test['pred_diff'] = test['prediction'].diff()\n    \n    test['event'] = test['pred_diff'].replace({1:'wakeup', -1:'onset', 0:np.nan})\n    \n    test_wakeup = test[test['event']=='wakeup'].groupby(test['timestamp'].dt.date).agg('first')\n    test_onset = test[test['event']=='onset'].groupby(test['timestamp'].dt.date).agg('last')\n    test = pd.concat([test_wakeup, test_onset], ignore_index=True).sort_values('timestamp')\n\n    return test","metadata":{"_uuid":"9a8d385c-77e4-410b-9985-7d83cb914f4e","_cell_guid":"5e863e37-674e-4ba2-a5ac-f614ab05f9b7","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:15.152962Z","iopub.execute_input":"2023-11-12T03:55:15.153277Z","iopub.status.idle":"2023-11-12T03:55:15.175121Z","shell.execute_reply.started":"2023-11-12T03:55:15.153252Z","shell.execute_reply":"2023-11-12T03:55:15.173958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_sub = ['series_id','step','event','score']\n\nseries_id  = pd.read_parquet('/kaggle/input/child-mind-institute-detect-sleep-states/test_series.parquet', columns=['series_id'])\nseries_id = series_id.series_id.unique()\n\ntests = []\n\nfor idx in series_id: \n\n    test =  get_events(idx, model)\n    tests.append(test[cols_sub])","metadata":{"_uuid":"8edf989d-08d7-4ccf-afd2-17a6e9dbeb13","_cell_guid":"dad39ed9-8159-4d8a-9ecf-18362a6daa2e","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:15.176413Z","iopub.execute_input":"2023-11-12T03:55:15.176721Z","iopub.status.idle":"2023-11-12T03:55:56.528087Z","shell.execute_reply.started":"2023-11-12T03:55:15.176695Z","shell.execute_reply":"2023-11-12T03:55:56.526869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat(tests, ignore_index=True).reset_index(names='row_id')\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"_uuid":"e978c88f-3785-400f-b53b-12775dc93da6","_cell_guid":"136594b9-bac9-4749-ba5c-e1d30199f85a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2023-11-12T03:55:56.529381Z","iopub.execute_input":"2023-11-12T03:55:56.529693Z","iopub.status.idle":"2023-11-12T03:55:56.549088Z","shell.execute_reply.started":"2023-11-12T03:55:56.529667Z","shell.execute_reply":"2023-11-12T03:55:56.547933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}