{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":53666,"databundleVersionId":6589269,"sourceType":"competition"}],"dockerImageVersionId":30587,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Child Mind Institute - Detect Sleep States","metadata":{}},{"cell_type":"markdown","source":"## Importing Libraries","metadata":{}},{"cell_type":"code","source":"!pip install pyspark","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:22.735604Z","iopub.execute_input":"2023-11-16T04:54:22.736531Z","iopub.status.idle":"2023-11-16T04:54:34.974555Z","shell.execute_reply.started":"2023-11-16T04:54:22.736490Z","shell.execute_reply":"2023-11-16T04:54:34.973275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:34.977437Z","iopub.execute_input":"2023-11-16T04:54:34.977859Z","iopub.status.idle":"2023-11-16T04:54:36.095889Z","shell.execute_reply.started":"2023-11-16T04:54:34.977819Z","shell.execute_reply":"2023-11-16T04:54:36.094725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nimport pyspark\nfrom pyspark import SparkContext, SparkConf\nfrom pyspark.sql import functions\nfrom pyspark.sql import Row, SQLContext, SparkSession\nfrom pyspark.sql.functions import *\nfrom pyspark.sql.types import *\nfrom pyspark.sql.functions import col\n\nimport cudf\nimport multiprocessing\nimport os\nimport re\nimport time\nimport pynvml\nimport subprocess\nimport psutil\n\nfrom warnings import filterwarnings;\nfilterwarnings('ignore');","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:36.097285Z","iopub.execute_input":"2023-11-16T04:54:36.097606Z","iopub.status.idle":"2023-11-16T04:54:39.493889Z","shell.execute_reply.started":"2023-11-16T04:54:36.097578Z","shell.execute_reply":"2023-11-16T04:54:39.492811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_time(func):\n    def inner_get_time(*args, **kwargs):\n        start_time = time.time()\n        result = func(*args, **kwargs)\n        end_time = time.time()\n        execution_time = (end_time - start_time) * 1000 \n        print(\"-\" * 80)\n        print(f\"Execution time: {execution_time} ms\")\n        return result, execution_time  \n\n    return inner_get_time","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.496657Z","iopub.execute_input":"2023-11-16T04:54:39.497122Z","iopub.status.idle":"2023-11-16T04:54:39.503442Z","shell.execute_reply.started":"2023-11-16T04:54:39.497094Z","shell.execute_reply":"2023-11-16T04:54:39.502282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(data):\n    for col_name, col_data_type in data.dtypes:\n        if col_data_type.startswith(\"int\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"int\"))\n        elif col_data_type.startswith(\"float\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"float\"))\n        elif col_data_type.startswith(\"decimal\"):\n            # Adjust the scale and precision as needed\n            data = data.withColumn(col_name, col(col_name).cast(\"decimal(10, 2)\"))\n        elif col_data_type.startswith(\"timestamp\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"timestamp\"))\n        elif col_data_type.startswith(\"string\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"string\"))\n        elif col_data_type.startswith(\"boolean\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"boolean\"))\n        elif col_data_type.startswith(\"array\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"array<your_element_type>\"))\n        elif col_data_type.startswith(\"map\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"map<your_key_type, your_value_type>\"))\n        \n\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.504703Z","iopub.execute_input":"2023-11-16T04:54:39.505056Z","iopub.status.idle":"2023-11-16T04:54:39.517488Z","shell.execute_reply.started":"2023-11-16T04:54:39.505022Z","shell.execute_reply":"2023-11-16T04:54:39.516442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_free_gpu_memory():\n    try:\n        pynvml.nvmlInit()\n\n        gpu_count = pynvml.nvmlDeviceGetCount()\n\n        gpu_memory_list = []\n\n        for i in range(gpu_count):\n            handle = pynvml.nvmlDeviceGetHandleByIndex(i)\n            gpu_memory = pynvml.nvmlDeviceGetMemoryInfo(handle)\n            free_memory_mb = gpu_memory.free\n            return free_memory_mb\n\n        pynvml.nvmlShutdown()\n\n        return gpu_memory_list\n\n    except Exception as e:\n        return str(e)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.518358Z","iopub.execute_input":"2023-11-16T04:54:39.518690Z","iopub.status.idle":"2023-11-16T04:54:39.530797Z","shell.execute_reply.started":"2023-11-16T04:54:39.518663Z","shell.execute_reply":"2023-11-16T04:54:39.529689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_bytes(file_size):\n    units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']\n   \n    unit_index = 0\n    while file_size >= 1024 and unit_index < len(units) - 1:\n        file_size /= 1024.0\n        unit_index += 1\n  \n    return f\"{file_size:.2f} {units[unit_index]}\"\n\ndef getSize(file):\n    try:\n        file_size = os.path.getsize(file)\n        file_name = re.search(r'([^/\\\\]+)$', file).group(1)\n\n        print(f\"File {file_name} size is {convert_bytes(file_size)}\")\n    except FileNotFoundError:\n        print(\"File not found.\")\n    except OSError:\n        print(\"OS error occurred.\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.532051Z","iopub.execute_input":"2023-11-16T04:54:39.532349Z","iopub.status.idle":"2023-11-16T04:54:39.546078Z","shell.execute_reply.started":"2023-11-16T04:54:39.532323Z","shell.execute_reply":"2023-11-16T04:54:39.545060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cpu_core_count():\n    try:\n       \n        cpu_core = multiprocessing.cpu_count()\n\n        return cpu_core\n    except Exception as e:\n        return None\n\ndef get_gpu_core_count():\n    try:\n        pynvml.nvmlInit()\n\n        gpu_count = pynvml.nvmlDeviceGetCount()\n\n        pynvml.nvmlShutdown()\n\n        return gpu_count\n\n    except Exception as e:\n        return None","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.547243Z","iopub.execute_input":"2023-11-16T04:54:39.547611Z","iopub.status.idle":"2023-11-16T04:54:39.556235Z","shell.execute_reply.started":"2023-11-16T04:54:39.547586Z","shell.execute_reply":"2023-11-16T04:54:39.555289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@get_time\ndef readParquet(file):\n    data = spark_1.read \\\n        .option(\"header\", \"true\") \\\n        .option(\"inferSchema\", \"true\") \\\n        .parquet(file) \\\n        .cache()\n    print(f\"Number of Partitions -> {data.rdd.getNumPartitions()}\")\n    data.write.format(\"noop\").mode(\"overwrite\").save()\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.557550Z","iopub.execute_input":"2023-11-16T04:54:39.557820Z","iopub.status.idle":"2023-11-16T04:54:39.566236Z","shell.execute_reply.started":"2023-11-16T04:54:39.557797Z","shell.execute_reply":"2023-11-16T04:54:39.565452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.570104Z","iopub.execute_input":"2023-11-16T04:54:39.570473Z","iopub.status.idle":"2023-11-16T04:54:39.576735Z","shell.execute_reply.started":"2023-11-16T04:54:39.570382Z","shell.execute_reply":"2023-11-16T04:54:39.575814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### PySpark Configurations","metadata":{}},{"cell_type":"code","source":"gpu_memory = get_free_gpu_memory()\ncpu_cores = get_cpu_core_count()\ngpu_cores = get_gpu_core_count()\n\nmaster = 'local[*]'\nmaxMemory = '8G'\napp_name = 'Child Mind Institute Detect Sleep States'\nmaxBytes= str(gpu_memory) + \"b\"\n\nconf = {\n    'spark.dynamicAllocation.enabled': 'true',\n    'spark.dynamicAllocation.maxExecutors': 8,\n    \n    'spark.shuffle.service.enabled': 'true',\n    \n    'spark.memory.fraction':'0.6',\n    \n    'spark.executor.memoryOverhead':maxMemory,\n    'spark.executor.memory': maxMemory,\n    'spark.executor.cores': cpu_cores,\n    'spark.executor.resource.gpu.amount': gpu_cores * 3,\n    'spark.executor.resource.gpu.memoryFraction':'0.6',\n    \n    'spark.rapids.memory.pinnedPool.size': maxBytes,\n    'spark.rapids.sql.enabled': 'true',\n    \n    'spark.driver.memory': maxMemory,\n    'spark.driver.maxResultsSize': maxMemory,\n    \n    'spark.sql.repl.eagerEval.enabled': 'true',\n    'spark.sql.files.maxPartitionBytes': maxBytes,\n    'spark.sql.files.maxPartitionNum': cpu_cores * 3,\n    'spark.sql.files.minPartitionNum': cpu_cores * 2,\n    'spark.sql.adaptive.enabled': 'false',\n    'spark.sql.adaptive.localShuffleReader.enabled': 'false',\n    'spark.sql.adaptive.coalescePartitions.enabled': 'false',\n    'spark.sql.sources.partitionOverwriteMode': 'dynamic',\n    'spark.sql.shuffle.partitions': '4',\n}\n\nbuilder = (\n    SparkSession\n    .builder\n    .master(master)\n    .appName(app_name)\n)\n\nfor k, v in conf.items():\n    builder.config(k, v)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.577909Z","iopub.execute_input":"2023-11-16T04:54:39.578257Z","iopub.status.idle":"2023-11-16T04:54:39.596819Z","shell.execute_reply.started":"2023-11-16T04:54:39.578223Z","shell.execute_reply":"2023-11-16T04:54:39.595950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark_1 = builder.getOrCreate()\nspark_1.sparkContext.setLogLevel(\"ERROR\")\nspark_1","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:39.598169Z","iopub.execute_input":"2023-11-16T04:54:39.598491Z","iopub.status.idle":"2023-11-16T04:54:45.293209Z","shell.execute_reply.started":"2023-11-16T04:54:39.598465Z","shell.execute_reply":"2023-11-16T04:54:45.292015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_dataset='../input/child-mind-institute-detect-sleep-states/train_series.parquet'\nsample_submission_dataset='../input/child-mind-institute-detect-sleep-states/sample_submission.csv'\ntrain_events_dataset='../input/child-mind-institute-detect-sleep-states/train_events.csv'\ntest_series_dataset='../input/child-mind-institute-detect-sleep-states/test_series.parquet'","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:45.294820Z","iopub.execute_input":"2023-11-16T04:54:45.295178Z","iopub.status.idle":"2023-11-16T04:54:45.300725Z","shell.execute_reply.started":"2023-11-16T04:54:45.295149Z","shell.execute_reply":"2023-11-16T04:54:45.299332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(getSize(train_series_dataset))\nprint(getSize(sample_submission_dataset))\nprint(getSize(train_events_dataset))\nprint(getSize(test_series_dataset))","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:45.302491Z","iopub.execute_input":"2023-11-16T04:54:45.302786Z","iopub.status.idle":"2023-11-16T04:54:45.321340Z","shell.execute_reply.started":"2023-11-16T04:54:45.302761Z","shell.execute_reply":"2023-11-16T04:54:45.320297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Parquet File: train_series.parquet","metadata":{}},{"cell_type":"code","source":"train_series, execution_time = readParquet(train_series_dataset)\ntrain_series","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:54:45.322775Z","iopub.execute_input":"2023-11-16T04:54:45.323073Z","iopub.status.idle":"2023-11-16T04:57:12.477188Z","shell.execute_reply.started":"2023-11-16T04:54:45.323048Z","shell.execute_reply":"2023-11-16T04:57:12.476105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_train_series = reduce_mem_usage(train_series.na.drop())","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:12.479141Z","iopub.execute_input":"2023-11-16T04:57:12.479585Z","iopub.status.idle":"2023-11-16T04:57:12.622180Z","shell.execute_reply.started":"2023-11-16T04:57:12.479546Z","shell.execute_reply":"2023-11-16T04:57:12.621103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_train_series = cleaned_train_series \\\n        .withColumn(\"date\", date_format(\"timestamp\", \"yyyy-MM-dd\")) \\\n        .withColumn(\"time\", date_format(\"timestamp\", \"hh:mm:ss\")) \\\n        .withColumn(\"hour\", date_format(\"timestamp\", \"hh\")) \\\n        .withColumn(\"hour\", col(\"hour\").cast(\"int\"))","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:12.623282Z","iopub.execute_input":"2023-11-16T04:57:12.623693Z","iopub.status.idle":"2023-11-16T04:57:12.724327Z","shell.execute_reply.started":"2023-11-16T04:57:12.623658Z","shell.execute_reply":"2023-11-16T04:57:12.723122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:12.730626Z","iopub.execute_input":"2023-11-16T04:57:12.731027Z","iopub.status.idle":"2023-11-16T04:57:13.944738Z","shell.execute_reply.started":"2023-11-16T04:57:12.730988Z","shell.execute_reply":"2023-11-16T04:57:13.943804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AnalyseData:\n    def __init__(self, data):\n        self.data = data\n    \n    @get_time\n    def countData(self):\n        return self.data.count()\n\n    @get_time\n    def describeData(self):\n        return self.data.describe()\n    \n    @get_time\n    def printSchemaData(self):\n        return self.data.printSchema()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:13.946165Z","iopub.execute_input":"2023-11-16T04:57:13.946470Z","iopub.status.idle":"2023-11-16T04:57:13.953291Z","shell.execute_reply.started":"2023-11-16T04:57:13.946446Z","shell.execute_reply":"2023-11-16T04:57:13.952219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_analyzer = AnalyseData(cleaned_train_series)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:13.954666Z","iopub.execute_input":"2023-11-16T04:57:13.954938Z","iopub.status.idle":"2023-11-16T04:57:13.963868Z","shell.execute_reply.started":"2023-11-16T04:57:13.954914Z","shell.execute_reply":"2023-11-16T04:57:13.962933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count, count_time = train_series_analyzer.countData()\ndescription, describe_time = train_series_analyzer.describeData()\nprintSchema, printSchema_time = train_series_analyzer.printSchemaData()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:13.964941Z","iopub.execute_input":"2023-11-16T04:57:13.965219Z","iopub.status.idle":"2023-11-16T04:57:24.510327Z","shell.execute_reply.started":"2023-11-16T04:57:13.965196Z","shell.execute_reply":"2023-11-16T04:57:24.509455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Count:\", count)\nprint(\"Execution time for countData+:\", count_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:24.511422Z","iopub.execute_input":"2023-11-16T04:57:24.511768Z","iopub.status.idle":"2023-11-16T04:57:24.517542Z","shell.execute_reply.started":"2023-11-16T04:57:24.511735Z","shell.execute_reply":"2023-11-16T04:57:24.516668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(\"Description:\", description)\n#print(\"Execution time for describeData:\", describe_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T04:57:24.519122Z","iopub.execute_input":"2023-11-16T04:57:24.519496Z","iopub.status.idle":"2023-11-16T05:03:08.661684Z","shell.execute_reply.started":"2023-11-16T04:57:24.519462Z","shell.execute_reply":"2023-11-16T05:03:08.657467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"printSchema:\", printSchema)\nprint(\"Execution time for printSchema:\", printSchema_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:03:20.337280Z","iopub.execute_input":"2023-11-16T05:03:20.337841Z","iopub.status.idle":"2023-11-16T05:03:20.352484Z","shell.execute_reply.started":"2023-11-16T05:03:20.337796Z","shell.execute_reply":"2023-11-16T05:03:20.351216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Taken only 100 samples of 50% data for EDA**","metadata":{}},{"cell_type":"code","source":"sampled_data = cleaned_train_series.sample(0.5).limit(100)","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:03:22.383903Z","iopub.execute_input":"2023-11-16T05:03:22.384534Z","iopub.status.idle":"2023-11-16T05:03:22.425660Z","shell.execute_reply.started":"2023-11-16T05:03:22.384485Z","shell.execute_reply":"2023-11-16T05:03:22.424490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Distribution of steps","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(12, 6))\nsns.histplot(sampled_data.select('step').toPandas(), bins=50, kde=True)\nplt.title('Distribution of Steps')\nplt.xlabel('Step')\nplt.ylabel('Frequency')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:03:22.798904Z","iopub.execute_input":"2023-11-16T05:03:22.799550Z","iopub.status.idle":"2023-11-16T05:15:53.387735Z","shell.execute_reply.started":"2023-11-16T05:03:22.799503Z","shell.execute_reply":"2023-11-16T05:15:53.386504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Time series plot of anglez","metadata":{}},{"cell_type":"code","source":"anglez_data = sampled_data.select(\"timestamp\", \"anglez\").toPandas()\nplt.figure(figsize=(12, 6))\nplt.plot(anglez_data['timestamp'], anglez_data['anglez'])\nplt.title('Time Series Plot of Anglez')\nplt.xlabel('Timestamp')\nplt.ylabel('Anglez')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:15:53.399487Z","iopub.execute_input":"2023-11-16T05:15:53.400282Z","iopub.status.idle":"2023-11-16T05:15:55.267167Z","shell.execute_reply.started":"2023-11-16T05:15:53.400242Z","shell.execute_reply":"2023-11-16T05:15:55.265940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Time series plot of ENMO","metadata":{}},{"cell_type":"code","source":"enmo_data = sampled_data.select(\"timestamp\", \"enmo\").toPandas()\nplt.figure(figsize=(12, 6))\nplt.plot(enmo_data['timestamp'], enmo_data['enmo'])\nplt.title('Time Series Plot of ENMO')\nplt.xlabel('Timestamp')\nplt.ylabel('ENMO')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:15:55.268757Z","iopub.execute_input":"2023-11-16T05:15:55.269265Z","iopub.status.idle":"2023-11-16T05:15:57.286474Z","shell.execute_reply.started":"2023-11-16T05:15:55.269226Z","shell.execute_reply":"2023-11-16T05:15:57.285377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Boxplot of Anglez by Hour","metadata":{}},{"cell_type":"code","source":"boxplot_anglez_hour_data = sampled_data.select(\"hour\", \"anglez\").toPandas()\nplt.figure(figsize=(12, 6))\nsns.boxplot(x='hour', y='anglez', data=boxplot_anglez_hour_data)\nplt.title('Boxplot of Anglez by Hour')\nplt.xlabel('Hour')\nplt.ylabel('Anglez')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:15:57.292728Z","iopub.execute_input":"2023-11-16T05:15:57.296701Z","iopub.status.idle":"2023-11-16T05:15:58.023187Z","shell.execute_reply.started":"2023-11-16T05:15:57.296660Z","shell.execute_reply":"2023-11-16T05:15:58.022114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Boxplot of ENMO by Hour","metadata":{}},{"cell_type":"code","source":"boxplot_enmo_hour_data = sampled_data.select(\"hour\", \"enmo\").toPandas()\nplt.figure(figsize=(12, 6))\nsns.boxplot(x='hour', y='enmo', data=boxplot_enmo_hour_data)\nplt.title('Boxplot of ENMO by Hour')\nplt.xlabel('Hour')\nplt.ylabel('ENMO')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:15:58.029197Z","iopub.execute_input":"2023-11-16T05:15:58.032887Z","iopub.status.idle":"2023-11-16T05:15:58.888825Z","shell.execute_reply.started":"2023-11-16T05:15:58.032848Z","shell.execute_reply":"2023-11-16T05:15:58.887951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Scatter Plot between Anglez and ENMO","metadata":{}},{"cell_type":"code","source":"scatter_data = sampled_data.select(\"anglez\", \"enmo\").toPandas()\nplt.figure(figsize=(12, 6))\nsns.scatterplot(x='anglez', y='enmo', data=scatter_data)\nplt.title('Scatter Plot between Anglez and ENMO')\nplt.xlabel('Anglez')\nplt.ylabel('ENMO')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:15:58.890388Z","iopub.execute_input":"2023-11-16T05:15:58.890827Z","iopub.status.idle":"2023-11-16T05:15:59.770450Z","shell.execute_reply.started":"2023-11-16T05:15:58.890788Z","shell.execute_reply":"2023-11-16T05:15:59.765448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Violin Plot of Anglez","metadata":{}},{"cell_type":"code","source":"violin_anglez_data = sampled_data.select(\"anglez\").toPandas()\nplt.figure(figsize=(12, 6))\nsns.violinplot(x=violin_anglez_data['anglez'])\nplt.title('Violin Plot of Anglez')\nplt.xlabel('Anglez')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:15:59.776900Z","iopub.execute_input":"2023-11-16T05:15:59.780793Z","iopub.status.idle":"2023-11-16T05:16:00.952427Z","shell.execute_reply.started":"2023-11-16T05:15:59.780747Z","shell.execute_reply":"2023-11-16T05:16:00.951472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Violin Plot of ENMO","metadata":{}},{"cell_type":"code","source":"violin_enmo_data = sampled_data.select(\"enmo\").toPandas()\nplt.figure(figsize=(12, 6))\nsns.violinplot(x=violin_enmo_data['enmo'])\nplt.title('Violin Plot of ENMO')\nplt.xlabel('ENMO')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:16:00.958179Z","iopub.execute_input":"2023-11-16T05:16:00.958983Z","iopub.status.idle":"2023-11-16T05:16:01.525406Z","shell.execute_reply.started":"2023-11-16T05:16:00.958943Z","shell.execute_reply":"2023-11-16T05:16:01.524459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pairplot of Anglez, ENMO, and Step","metadata":{}},{"cell_type":"code","source":"pairplot_data = sampled_data.select(\"anglez\", \"enmo\", \"step\").toPandas()\nsns.pairplot(pairplot_data)\nplt.suptitle('Pairplot of Anglez, ENMO, and Step', y=1.02)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:16:01.526702Z","iopub.execute_input":"2023-11-16T05:16:01.527259Z","iopub.status.idle":"2023-11-16T05:16:10.372264Z","shell.execute_reply.started":"2023-11-16T05:16:01.527224Z","shell.execute_reply":"2023-11-16T05:16:10.371138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation Matrix","metadata":{}},{"cell_type":"code","source":"numerical_columns = [\"step\", \"anglez\", \"enmo\", \"hour\"]\ncorrelation_matrix = sampled_data.select(numerical_columns).toPandas().corr()\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(correlation_matrix, annot=True, cmap=\"coolwarm\", fmt=\".2f\", linewidths=.5)\nplt.title('Correlation Matrix')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:16:10.376371Z","iopub.execute_input":"2023-11-16T05:16:10.377637Z","iopub.status.idle":"2023-11-16T05:16:11.391465Z","shell.execute_reply.started":"2023-11-16T05:16:10.377592Z","shell.execute_reply":"2023-11-16T05:16:11.390205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## CSV File: train_events.csv","metadata":{}},{"cell_type":"code","source":"train_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:16:11.396946Z","iopub.execute_input":"2023-11-16T05:16:11.397584Z","iopub.status.idle":"2023-11-16T05:16:11.455531Z","shell.execute_reply.started":"2023-11-16T05:16:11.397545Z","shell.execute_reply":"2023-11-16T05:16:11.454307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrain_events.shape","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:16:11.461653Z","iopub.execute_input":"2023-11-16T05:16:11.462496Z","iopub.status.idle":"2023-11-16T05:16:11.477842Z","shell.execute_reply.started":"2023-11-16T05:16:11.462457Z","shell.execute_reply":"2023-11-16T05:16:11.476853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(4,4))\n\nax = sns.countplot(data=train_events, x='event')\nax.bar_label(ax.containers[0])\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:16:11.481085Z","iopub.execute_input":"2023-11-16T05:16:11.488995Z","iopub.status.idle":"2023-11-16T05:16:11.735930Z","shell.execute_reply.started":"2023-11-16T05:16:11.488957Z","shell.execute_reply":"2023-11-16T05:16:11.734525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6,4))\n\nax = plt.plot(list(train_events['night']))\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-11-16T05:16:11.742175Z","iopub.execute_input":"2023-11-16T05:16:11.742987Z","iopub.status.idle":"2023-11-16T05:16:12.131007Z","shell.execute_reply.started":"2023-11-16T05:16:11.742942Z","shell.execute_reply":"2023-11-16T05:16:12.129871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### **Continue...**","metadata":{}}]}