{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport gc\nimport time\nimport json\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport os\nimport joblib\nimport random\nimport math\nfrom tqdm.auto import tqdm \n\nfrom scipy.interpolate import interp1d\n\nfrom math import pi, sqrt, exp\nimport sklearn,sklearn.model_selection\nimport torch\nfrom torch import nn,Tensor\nimport torch.nn.functional as F\nfrom torch.utils.data import DataLoader, Dataset, SubsetRandomSampler\nfrom sklearn.metrics import average_precision_score\nfrom timm.scheduler import CosineLRScheduler\nplt.style.use(\"ggplot\")\n\nfrom pyarrow.parquet import ParquetFile\nimport pyarrow as pa \nimport ctypes","metadata":{"execution":{"iopub.status.busy":"2023-10-31T07:58:12.482282Z","iopub.execute_input":"2023-10-31T07:58:12.482707Z","iopub.status.idle":"2023-10-31T07:58:21.072134Z","shell.execute_reply.started":"2023-10-31T07:58:12.482672Z","shell.execute_reply":"2023-10-31T07:58:21.070977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PATHS:\n    MAIN_DIR = \"/kaggle/input/child-mind-institute-detect-sleep-states/\"\n    # CSV FILES : \n    SUBMISSION = MAIN_DIR + \"sample_submission.csv\"\n    TRAIN_EVENTS = MAIN_DIR + \"train_events.csv\"\n    # PARQUET FILES:\n    TRAIN_SERIES = MAIN_DIR + \"train_series.parquet\"\n    TEST_SERIES = MAIN_DIR + \"test_series.parquet\"","metadata":{"execution":{"iopub.status.busy":"2023-10-31T07:58:21.074017Z","iopub.execute_input":"2023-10-31T07:58:21.074402Z","iopub.status.idle":"2023-10-31T07:58:21.080093Z","shell.execute_reply.started":"2023-10-31T07:58:21.074355Z","shell.execute_reply":"2023-10-31T07:58:21.079169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_dir = 'train_csvs'\nos.makedirs(out_dir, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2023-10-31T07:58:21.081470Z","iopub.execute_input":"2023-10-31T07:58:21.081992Z","iopub.status.idle":"2023-10-31T07:58:21.096627Z","shell.execute_reply.started":"2023-10-31T07:58:21.081961Z","shell.execute_reply":"2023-10-31T07:58:21.095290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class data_reader:\n    def __init__(self, demo_mode):\n        super().__init__()\n        # MAPPING FOR DATA LOADING :\n        self.names_mapping = {\n            \"submission\" : {\"path\" : PATHS.SUBMISSION, \"is_parquet\" : False, \"has_timestamp\" : False}, \n            \"train_events\" : {\"path\" : PATHS.TRAIN_EVENTS, \"is_parquet\" : False, \"has_timestamp\" : True},\n            \"train_series\" : {\"path\" : PATHS.TRAIN_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True},\n            \"test_series\" : {\"path\" : PATHS.TEST_SERIES, \"is_parquet\" : True, \"has_timestamp\" : True}\n        }\n        self.valid_names = [\"submission\", \"train_events\", \"train_series\", \"test_series\"]\n        self.demo_mode = demo_mode\n    \n    def verify(self, data_name):\n        \"function for data name verification\"\n        if data_name not in self.valid_names:\n            print(\"PLEASE ENTER A VALID DATASET NAME, VALID NAMES ARE : \", valid_names)\n        return\n    \n    def cleaning(self, data):\n        \"cleaning function : drop na values\"\n        before_cleaning = len(data)\n        print(\"Number of missing timestamps : \", len(data[data[\"timestamp\"].isna()]))\n        data = data.dropna(subset=[\"timestamp\"])\n        after_cleaning = len(data)\n        print(\"Percentage of removed steps : {:.1f}%\".format(100 * (before_cleaning - after_cleaning) / before_cleaning) )\n#         print(data.isna().any())\n#         data = data.bfill()\n        return data\n    \n    @staticmethod\n    def reduce_memory_usage(data):\n        \"iterate through all the columns of a dataframe and modify the data type to reduce memory usage.\"\n        start_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n        for col in data.columns:\n            col_type = data[col].dtype    \n            if col_type != object:\n                c_min = data[col].min()\n                c_max = data[col].max()\n                if str(col_type)[:3] == 'int':\n                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                        data[col] = data[col].astype(np.int8)\n                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                        data[col] = data[col].astype(np.int16)\n                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                        data[col] = data[col].astype(np.int32)\n                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                        data[col] = data[col].astype(np.int64)  \n                else:\n                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                        data[col] = data[col].astype(np.float16)\n                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                        data[col] = data[col].astype(np.float32)\n                    else:\n                        data[col] = data[col].astype(np.float64)\n            else:\n                data[col] = data[col].astype('category')\n\n        end_mem = data.memory_usage().sum() / 1024**2\n        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n        return data\n    \n    def load_data(self, data_name):\n        \"function for data loading\"\n        self.verify(data_name)\n        data_props = self.names_mapping[data_name]\n        if data_props[\"is_parquet\"]:\n            if self.demo_mode:\n                pf = ParquetFile(data_props[\"path\"]) \n                demo_steps = next(pf.iter_batches(batch_size=20_000)) \n                data = pa.Table.from_batches([demo_steps]).to_pandas()\n            else:\n                data = pd.read_parquet(data_props[\"path\"])\n        else:\n            if self.demo_mode:\n                data = pd.read_csv(data_props[\"path\"], nsteps=20_000)\n            else:\n                data = pd.read_csv(data_props[\"path\"])\n                \n        gc.collect()\n        if data_props[\"has_timestamp\"]:\n            print('cleaning')\n            data = self.cleaning(data)\n            gc.collect()\n        #data = self.reduce_memory_usage(data)\n        return data","metadata":{"execution":{"iopub.status.busy":"2023-10-31T07:58:21.099576Z","iopub.execute_input":"2023-10-31T07:58:21.099964Z","iopub.status.idle":"2023-10-31T07:58:21.129517Z","shell.execute_reply.started":"2023-10-31T07:58:21.099932Z","shell.execute_reply":"2023-10-31T07:58:21.128019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"reader = data_reader(demo_mode=False)\nseries = reader.load_data(data_name=\"train_series\")\nevents = reader.load_data(data_name=\"train_events\")","metadata":{"execution":{"iopub.status.busy":"2023-10-31T07:58:21.131129Z","iopub.execute_input":"2023-10-31T07:58:21.131633Z","iopub.status.idle":"2023-10-31T07:59:54.404186Z","shell.execute_reply.started":"2023-10-31T07:58:21.131593Z","shell.execute_reply":"2023-10-31T07:59:54.402700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SIGMA = 720 # 12 * 60\ndef gauss(n=SIGMA,sigma=SIGMA*0.15):\n    # guassian distribution function\n    r = range(-int(n/2),int(n/2)+1)\n    return [1 / (sigma * sqrt(2*pi)) * exp(-float(x)**2/(2*sigma**2)) for x in r]","metadata":{"execution":{"iopub.status.busy":"2023-10-31T07:59:54.406407Z","iopub.execute_input":"2023-10-31T07:59:54.406903Z","iopub.status.idle":"2023-10-31T07:59:54.415259Z","shell.execute_reply.started":"2023-10-31T07:59:54.406859Z","shell.execute_reply":"2023-10-31T07:59:54.413546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = []\ndata = []\nids = series.series_id.unique()\n\nenmo_dfs = pd.DataFrame()\n\nfor j, viz_id in tqdm(enumerate(ids), total=len(ids)):\n    viz_targets = []\n    viz_events = events[events.series_id == viz_id]\n    viz_series = series.loc[(series.series_id==viz_id)].copy().reset_index()\n    viz_series['dt'] = pd.to_datetime(viz_series.timestamp,format = '%Y-%m-%dT%H:%M:%S%z').astype(\"datetime64[ns, UTC-04:00]\")\n    viz_series['hour'] = viz_series['dt'].dt.hour+viz_series['dt'].dt.minute / 60\n\n    check = 0\n    for i in range(len(viz_events)-1):\n        if viz_events.iloc[i].event =='onset' and viz_events.iloc[i+1].event =='wakeup' and viz_events.iloc[i].night==viz_events.iloc[i+1].night:\n            start,end = viz_events.timestamp.iloc[i],viz_events.timestamp.iloc[i+1]\n\n            start_id = viz_series.loc[viz_series.timestamp ==start].index.values[0]\n            end_id = viz_series.loc[viz_series.timestamp ==end].index.values[0]\n            viz_targets.append((start_id,end_id))\n            check+=1\n\n    target_guassian = np.zeros((len(viz_series),2))\n\n    for s,e in viz_targets:\n        st1,st2 = max(0,s-SIGMA//2),s+SIGMA//2+1\n        ed1,ed2 = e-SIGMA//2,min(len(viz_series),e+SIGMA//2+1)\n        target_guassian[st1:st2,0] = gauss()[st1-(s-SIGMA//2):]\n        target_guassian[ed1:ed2,1] = gauss()[:SIGMA+1-((e+SIGMA//2+1)-ed2)]\n\n    target_guassian /= np.max(target_guassian + 1e-12)\n\n    viz_series['onset'] = target_guassian[:,0]\n    viz_series['wakeup'] = target_guassian[:,1]\n\n    new_df = viz_series[['step', 'anglez', 'enmo', 'hour', 'onset', 'wakeup']]\n    new_df.to_csv(f'{out_dir}/{viz_id}.csv', index=False)\n\n    enmo_dfs = pd.concat([enmo_dfs, new_df] ,axis=0)","metadata":{"execution":{"iopub.status.busy":"2023-10-31T07:59:54.416894Z","iopub.execute_input":"2023-10-31T07:59:54.417366Z","iopub.status.idle":"2023-10-31T11:11:38.662292Z","shell.execute_reply.started":"2023-10-31T07:59:54.417324Z","shell.execute_reply":"2023-10-31T11:11:38.660859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls\n","metadata":{"execution":{"iopub.status.busy":"2023-10-31T11:15:17.973513Z","iopub.execute_input":"2023-10-31T11:15:17.973989Z","iopub.status.idle":"2023-10-31T11:15:19.183632Z","shell.execute_reply.started":"2023-10-31T11:15:17.973954Z","shell.execute_reply":"2023-10-31T11:15:19.182480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport zipfile\n\n# 指定要压缩的CSV文件所在的目录\ncsv_dir = 'train_csvs'\n\n# 指定每批次要压缩的文件数量\nbatch_size = 20  # 每批压缩20个文件\n\n# 获取CSV文件列表\ncsv_files = [f for f in os.listdir(csv_dir) if f.endswith('.csv')]\n\n# 分批次进行压缩\nfor i in range(0, len(csv_files), batch_size):\n    batch_files = csv_files[i:i+batch_size]  # 获取当前批次的文件列表\n    zip_name = f'batch_{i//batch_size + 1}.zip'  # 生成ZIP文件名\n    \n    # 创建ZIP文件\n    with zipfile.ZipFile(zip_name, 'w', zipfile.ZIP_DEFLATED) as zipf:\n        # 逐个文件进行压缩\n        for file in batch_files:\n            file_path = os.path.join(csv_dir, file)  # 构建文件的完整路径\n            zipf.write(file_path, arcname=file)  # 将文件添加到ZIP中\n","metadata":{"execution":{"iopub.status.busy":"2023-10-31T11:18:16.810012Z","iopub.execute_input":"2023-10-31T11:18:16.810443Z","iopub.status.idle":"2023-10-31T11:24:24.892965Z","shell.execute_reply.started":"2023-10-31T11:18:16.810409Z","shell.execute_reply":"2023-10-31T11:24:24.891821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#enmo_dfs = enmo_dfs.reset_index()\n#enmo_dfs.head()","metadata":{"execution":{"iopub.status.busy":"2023-10-31T11:11:38.664273Z","iopub.execute_input":"2023-10-31T11:11:38.664647Z","iopub.status.idle":"2023-10-31T11:11:38.670251Z","shell.execute_reply.started":"2023-10-31T11:11:38.664616Z","shell.execute_reply":"2023-10-31T11:11:38.669019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#enmo_mean = enmo_dfs['enmo'].mean()\n#np.save('enmo_mean.npy', enmo_mean)\n#enmo_std = enmo_dfs['enmo'].std()\n#np.save('enmo_std.npy', enmo_std)\n#print(f'mean:{enmo_mean:.6f}, std:{enmo_std:.6f}')","metadata":{"execution":{"iopub.status.busy":"2023-10-31T11:11:38.671760Z","iopub.execute_input":"2023-10-31T11:11:38.672089Z","iopub.status.idle":"2023-10-31T11:11:38.685710Z","shell.execute_reply.started":"2023-10-31T11:11:38.672060Z","shell.execute_reply":"2023-10-31T11:11:38.684445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}