{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#
TABLE OF CONTENTS
\n \n* [IMPORTS](#1)\n* [INTRODUCTION- OBJECTIVES](#2)\n* [CONFIGURATION](#3)\n* [PREPROCESSING](#4)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"\n#
PACKAGE IMPORTS
","metadata":{}},{"cell_type":"code","source":"%%time \n\n# General library imports:-\nfrom IPython.display import display_html, clear_output, Markdown;\nfrom gc import collect;\n\nfrom copy import deepcopy;\nimport pandas as pd;\nimport polars as pl;\nimport polars.selectors as cs;\nimport numpy as np;\n\nfrom pprint import pprint;\nfrom colorama import Fore, Style, init;\nfrom warnings import filterwarnings;\nfilterwarnings('ignore');\n\nfrom tqdm.notebook import tqdm;\n\nprint();\ncollect();","metadata":{"execution":{"iopub.status.busy":"2023-09-17T20:48:29.038270Z","iopub.execute_input":"2023-09-17T20:48:29.038971Z","iopub.status.idle":"2023-09-17T20:48:29.853989Z","shell.execute_reply.started":"2023-09-17T20:48:29.038921Z","shell.execute_reply":"2023-09-17T20:48:29.852827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\n# Defining global configurations and functions:-\n\n# Color printing \ndef PrintColor(text:str, color = Fore.BLUE, style = Style.BRIGHT):\n \"Prints color outputs using colorama using a text F-string\";\n print(style + color + text + Style.RESET_ALL); \n\n# Making sklearn pipeline outputs as dataframe:-\nfrom sklearn import set_config; \nset_config(transform_output = \"pandas\");\npd.set_option('display.max_columns', 50);\npd.set_option('display.max_rows', 50);\n\n# Setting global configurations for polars:-\npl.Config.activate_decimals(True).set_tbl_hide_column_data_types(True);\npl.Config(**dict(tbl_formatting = 'ASCII_FULL_CONDENSED',\n tbl_hide_column_data_types = True,\n tbl_hide_dataframe_shape = True,\n fmt_float = \"mixed\",\n tbl_cell_alignment = 'CENTER',\n tbl_hide_dtype_separator = True,\n tbl_cols = 100,\n tbl_rows = 100,\n fmt_str_lengths = 100,\n )\n );\n\nPrintColor(f\"\\n---> Defining the global configuration for polars for all further steps\\n\");\npprint(pl.Config.state(), indent = 10, width = 50);\n\nprint();\ncollect();","metadata":{"execution":{"iopub.status.busy":"2023-09-17T20:48:29.855950Z","iopub.execute_input":"2023-09-17T20:48:29.856854Z","iopub.status.idle":"2023-09-17T20:48:31.299955Z","shell.execute_reply.started":"2023-09-17T20:48:29.856817Z","shell.execute_reply":"2023-09-17T20:48:31.298724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#
ASSIGNMENT DEFINITION AND OBJECTIVES
","metadata":{}},{"cell_type":"markdown","source":"
\nNote
\n1. This notebook is adjutant to my previous one in the competition as follows- https://www.kaggle.com/code/ravi20076/sleepstate-eda-baseline
\n2. Herewith, we try and curate a baseline preprocessed dataset from Dr. Carl's initial multi-class work to be used in our future work.
\n3. This notebook starts off with his inputs and curates date features and other relevant secondary columns using polars to create a separate dataset to be used as input for all our baseline models in the next few days. This is likely to save time and computations and help us proceed with the assignment.
\n4. I shall update the dataset with insights and features as deemed suitable going forward.
\n
","metadata":{}},{"cell_type":"markdown","source":"| Version
Number | Version Details | Preparation date|\n| :-: | --- | :-: | \n| **V1** |* Used Dr. Carl's **multiclass** dataset as input
* Secondary features based on date and other primary features
* No scaling|18Sep2023|","metadata":{}},{"cell_type":"markdown","source":"\n#
CONFIGURATION
","metadata":{}},{"cell_type":"code","source":"%%time\n\n# Defining a Configuration class:-\nclass CFG:\n \"This is a Configuration class for the given notebook, preprocessing in this case\";\n \n # Data preparation:- \n version_nb = 1;\n \n # Used during code proto-typing and syntax testing:- \n test_req = \"N\";\n n_samples = 1000;\n gpu_switch = \"OFF\"; \n state = 42;\n target = \"awake\";\n path = f\"/kaggle/input/child-mind-institute-detect-sleep-states/\";\n alt_path = f\"/kaggle/input/zzzs-lightweight-training-dataset-target/Zzzs_train_multi.parquet\";\n dtl_preproc_req = \"Y\";\n \n # Data transforms and scaling:- \n sec_ftre_req = \"Y\";\n roll_nper = 20;\n \n # Global variables for plotting:-\n grid_specs = {'visible': True, 'which': 'both', 'linestyle': '--', \n 'color': 'lightgrey', 'linewidth': 0.75};\n title_specs = {'fontsize': 9, 'fontweight': 'bold', 'color': 'tab:blue'};\n\nprint();\nPrintColor(f\"--> Configuration done!\\n\");\ncollect();","metadata":{"execution":{"iopub.status.busy":"2023-09-17T20:48:31.301328Z","iopub.execute_input":"2023-09-17T20:48:31.301955Z","iopub.status.idle":"2023-09-17T20:48:31.390381Z","shell.execute_reply.started":"2023-09-17T20:48:31.301922Z","shell.execute_reply":"2023-09-17T20:48:31.389007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n#
PREPROCESSING
","metadata":{}},{"cell_type":"code","source":"%time \n\nclass Preprocessor():\n \"\"\"\n This class aims to do the below-\n 1. Read the datasets and change the datatype of the target column in the alternative data\n 2. Curate secondary features based on public work\n \"\"\";\n \n def __init__(self):\n \"\"\"\n We do the below here-\n 1. Read in the train data from the alternative data source\n 2. Read in the test and submission file from the competition source\n 3. We sample the training data if necessary upon testing the code semantics\n \"\"\";\n \n self.test = \\\n pl.scan_parquet(CFG.path + f\"test_series.parquet\").\\\n with_columns(pl.col('timestamp').str.to_datetime().dt.offset_by(\"-4h\"));\n self.sub_fl = pl.read_csv(CFG.path + f\"sample_submission.csv\");\n \n self.target = CFG.target;\n self.test_req = CFG.test_req;\n self.sec_ftre_req = CFG.sec_ftre_req;\n self.nper = CFG.roll_nper;\n \n self.train = \\\n pl.scan_parquet(CFG.alt_path).\\\n with_columns(pl.col(CFG.target).cast(pl.Int8), \n pl.col('timestamp').str.to_datetime().dt.offset_by(\"-4h\")\n );\n \n if self.test_req == \"Y\":\n PrintColor(f\"---> We are testing the code with {CFG.n_samples} data samples per series and target\", \n color = Fore.RED);\n self.train = \\\n self.train.groupby(['series_id', self.target]).\\\n agg(pl.all().sample(CFG.n_samples)).\\\n explode(['step', 'timestamp', 'anglez', 'enmo']);\n \n else:\n PrintColor(f\"---> We are not testing the code- this is an actual and complete run\", color = Fore.RED); \n \n def PreprocessDF(self):\n \"\"\"\n This method ensues basic preprocessing steps for the submission file and collates the starting features\n \"\"\";\n \n PrintColor(f\"\\nSubmission file head\", color = Fore.GREEN);\n display(self.sub_fl.head(5)); \n \n self.strt_ftre = self.train.columns;\n try: self.strt_ftre.remove(CFG.target); \n except: pass;\n \n PrintColor(f\"Starting features\");\n pprint(self.strt_ftre);\n \n return self; \n \n def _XformDF(self, X: pl.DataFrame):\n \"\"\"\n This method prepares secondary features if requested by the user.\n We use the date time column and rolling statistics from public work\n \"\"\"\n \n # Curating date features:- \n df = \\\n X.with_columns([pl.col(\"timestamp\").dt.hour().cast(pl.Int8).alias(\"hour_nb\"),\n pl.col(\"timestamp\").dt.day().cast(pl.Int8).alias(\"day_nb\"),\n pl.col(\"timestamp\").dt.weekday().cast(pl.Int8).alias(\"weekday_nb\"),\n pl.col(\"timestamp\").dt.week().cast(pl.Int8).alias(\"week_nb\"),\n pl.col(\"timestamp\").dt.month().cast(pl.Int8).alias(\"month_nb\"),\n pl.col(\"timestamp\").dt.year().cast(pl.Int16).alias(\"year_nb\"),\n (pl.col(\"anglez\") * pl.col(\"enmo\")).alias(\"anglez_enmo\")\n ]\n ).\\\n with_columns([pl.when(pl.col(\"hour_nb\") <= 12).then(pl.lit(1)).otherwise(pl.lit(0)).cast(pl.Int8).alias(\"is_morning\"),\n pl.when(pl.col(\"weekday_nb\") >= 6).then(pl.lit(1)).otherwise(pl.lit(0)).cast(pl.Int8).alias(\"is_weekend\") \n ]\n );\n \n # Curating rolling features and window results:-\n df = \\\n df.\\\n with_columns([pl.col(\"anglez\").diff(self.nper).over(\"series_id\").\\\n fill_null(strategy = 'backward').alias(\"d_anglez\"),\n pl.col(\"enmo\").diff(self.nper).over(\"series_id\").\\\n fill_null(strategy = 'backward').alias(\"d_enmo\"),\n pl.col(\"anglez\").rolling_mean(self.nper).over(\"series_id\").\\\n fill_null(strategy = 'backward').fill_null(strategy = 'forward').alias('ma_anglez'),\n pl.col(\"enmo\").rolling_mean(self.nper).over(\"series_id\").\\\n fill_null(strategy = 'backward').fill_null(strategy = 'forward').alias('ma_enmo')\n ]\n ).\\\n with_columns([pl.col('d_anglez').rolling_mean(self.nper).over(\"series_id\").\\\n fill_null(strategy = 'backward').fill_null(strategy = 'forward').alias('ma_d_anglez'),\n pl.col('d_enmo').rolling_mean(self.nper).over(\"series_id\").\\\n fill_null(strategy = 'backward').fill_null(strategy = 'forward').alias('ma_d_enmo')\n ]\n );\n \n return df;\n \n def CurateSecFtre(self):\n \"This method curates the dataframe with secondary features and returns a modified dataframe\";\n \n if self.sec_ftre_req == \"Y\":\n PrintColor(f\"Secondary features are required\");\n return self._XformDF(self.train), self._XformDF(self.test);\n else:\n PrintColor(f\"Secondary features are not required\", color = Fore.RED);\n return self.train, self.test;\n \ncollect();\nprint();","metadata":{"execution":{"iopub.status.busy":"2023-09-17T20:48:31.393289Z","iopub.execute_input":"2023-09-17T20:48:31.393756Z","iopub.status.idle":"2023-09-17T20:48:31.506459Z","shell.execute_reply.started":"2023-09-17T20:48:31.393702Z","shell.execute_reply":"2023-09-17T20:48:31.504942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\n# Preprocessinng the data:-\nPrintColor(f\"\\n{'='*30} Data Pre-processing {'='*30}\\n\", Fore.MAGENTA);\npp = Preprocessor();\npp.PreprocessDF();\n\ntrain, test = pp.CurateSecFtre();\nprint();\nPrintColor(f\"\\n--> Train data query plan <---\\n\");\ndisplay(train.show_graph(figsize = (14, 8)));\n\n# Creating the train-test datasets:-\nPrintColor(f\"\\n\\n--> Collating the train-test datasets <---\");\ntrain, test = train.collect(), test.collect();\n\nPrintColor(f\"\\n---> Train set after pre-processing\\n\");\ndisplay(train.head(5));\nPrintColor(f\"\\n---> Test set after pre-processing\\n\");\ndisplay(test.head(5));\n\nPrintColor(f\"\\n---> Train set description after pre-processing\\n\");\ndf = train.describe(percentiles = [0.01, 0.05, 0.90, 0.95, 0.99]).to_pandas();\ncols = df.iloc[:,0].values.tolist();\ndf = df.iloc[:, 1:].transpose();\ndf.columns = cols;\ndisplay(df.drop(columns = ['count']).style.format(precision = 2));\ndel df, cols;\n\n\ncollect();\nprint();","metadata":{"execution":{"iopub.status.busy":"2023-09-17T20:48:31.508461Z","iopub.execute_input":"2023-09-17T20:48:31.509045Z","iopub.status.idle":"2023-09-17T20:49:00.797999Z","shell.execute_reply.started":"2023-09-17T20:48:31.509000Z","shell.execute_reply":"2023-09-17T20:49:00.796743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\n# Saving the datasets for later usage:-\ntrain.drop([\"timestamp\", CFG.target]).write_parquet(f\"Xtrain_V{CFG.version_nb}.parquet\");\ntrain.select(CFG.target).write_parquet(f\"ytrain_V{CFG.version_nb}.parquet\");\n\nprint();\ncollect();","metadata":{"execution":{"iopub.status.busy":"2023-09-17T20:54:07.188472Z","iopub.execute_input":"2023-09-17T20:54:07.188962Z","iopub.status.idle":"2023-09-17T20:54:14.085199Z","shell.execute_reply.started":"2023-09-17T20:54:07.188924Z","shell.execute_reply":"2023-09-17T20:54:14.084003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nReferences
\n1. https://www.kaggle.com/code/carlmcbrideellis/zzzs-random-forest-model-starter
\n2. https://www.kaggle.com/datasets/carlmcbrideellis/zzzs-lightweight-training-dataset-target
\n3. https://www.kaggle.com/code/satheeshbhukya1/detect-sleep-states
\n
","metadata":{}}]}