{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pyspark","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:10:37.772802Z","iopub.execute_input":"2023-09-27T01:10:37.773168Z","iopub.status.idle":"2023-09-27T01:11:29.772839Z","shell.execute_reply.started":"2023-09-27T01:10:37.773126Z","shell.execute_reply":"2023-09-27T01:11:29.771689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:29.776166Z","iopub.execute_input":"2023-09-27T01:11:29.777082Z","iopub.status.idle":"2023-09-27T01:11:30.861108Z","shell.execute_reply.started":"2023-09-27T01:11:29.777042Z","shell.execute_reply":"2023-09-27T01:11:30.859933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\n\nimport pyspark\nfrom pyspark import SparkContext, SparkConf\nfrom pyspark.sql import functions\nfrom pyspark.sql import Row, SQLContext, SparkSession\nfrom pyspark.sql.functions import *\nfrom pyspark.sql.types import *\n\nimport cudf\nimport multiprocessing\nimport os\nimport re\nimport time\nimport pynvml\nimport subprocess\nimport psutil\n\nos.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"   \nos.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\" ","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:30.862651Z","iopub.execute_input":"2023-09-27T01:11:30.863050Z","iopub.status.idle":"2023-09-27T01:11:37.508341Z","shell.execute_reply.started":"2023-09-27T01:11:30.863014Z","shell.execute_reply":"2023-09-27T01:11:37.507345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import socketserver\n\n# class MyRequestHandler(socketserver.BaseRequestHandler):\n#     def handle(self):\n#         try:\n#             # Your request handling code here\n#             data = self.request.recv(1024)\n#             # Process the received data\n            \n#         except EOFError as e:\n#             # Handle the EOFError gracefully (e.g., log the error)\n#             print(f\"EOFError occurred: {e}\")\n#         except Exception as e:\n#             # Handle other exceptions\n#             print(f\"An error occurred: {e}\")\n\n# if __name__ == \"__main__\":\n#     server = socketserver.TCPServer((\"127.0.0.1\", 12345), MyRequestHandler)\n#     server.serve_forever()\n","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.510970Z","iopub.execute_input":"2023-09-27T01:11:37.511698Z","iopub.status.idle":"2023-09-27T01:11:37.517395Z","shell.execute_reply.started":"2023-09-27T01:11:37.511654Z","shell.execute_reply":"2023-09-27T01:11:37.516243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_time(func):\n    def inner_get_time(*args, **kwargs):\n        start_time = time.time()\n        result = func(*args, **kwargs)\n        end_time = time.time()\n        execution_time = (end_time - start_time) * 1000  # Calculate execution time\n        print(\"-\" * 80)\n        print(f\"Execution time: {execution_time} ms\")\n        return result, execution_time  # Return both the result and execution time as a tuple\n\n    return inner_get_time","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.518880Z","iopub.execute_input":"2023-09-27T01:11:37.519463Z","iopub.status.idle":"2023-09-27T01:11:37.527778Z","shell.execute_reply.started":"2023-09-27T01:11:37.519427Z","shell.execute_reply":"2023-09-27T01:11:37.526461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(data):\n    for col_name, col_data_type in data.dtypes:\n        if col_data_type.startswith(\"int\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"int\"))\n        elif col_data_type.startswith(\"float\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"float\"))\n        elif col_data_type.startswith(\"decimal\"):\n            # Adjust the scale and precision as needed\n            data = data.withColumn(col_name, col(col_name).cast(\"decimal(10, 2)\"))\n        elif col_data_type.startswith(\"timestamp\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"timestamp\"))\n        elif col_data_type.startswith(\"string\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"string\"))\n        elif col_data_type.startswith(\"boolean\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"boolean\"))\n        elif col_data_type.startswith(\"array\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"array<your_element_type>\"))\n        elif col_data_type.startswith(\"map\"):\n            data = data.withColumn(col_name, col(col_name).cast(\"map<your_key_type, your_value_type>\"))\n        # Add more data type handling as needed\n\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.529511Z","iopub.execute_input":"2023-09-27T01:11:37.530269Z","iopub.status.idle":"2023-09-27T01:11:37.542404Z","shell.execute_reply.started":"2023-09-27T01:11:37.530231Z","shell.execute_reply":"2023-09-27T01:11:37.541292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_free_gpu_memory():\n    try:\n        # Initialize NVML\n        pynvml.nvmlInit()\n\n        # Get the number of available GPUs\n        gpu_count = pynvml.nvmlDeviceGetCount()\n\n        gpu_memory_list = []\n\n        for i in range(gpu_count):\n            handle = pynvml.nvmlDeviceGetHandleByIndex(i)\n            gpu_memory = pynvml.nvmlDeviceGetMemoryInfo(handle)\n            free_memory_mb = gpu_memory.free\n            return free_memory_mb\n\n        # Shutdown NVML\n        pynvml.nvmlShutdown()\n\n        return gpu_memory_list\n\n    except Exception as e:\n        return str(e)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.544139Z","iopub.execute_input":"2023-09-27T01:11:37.544665Z","iopub.status.idle":"2023-09-27T01:11:37.554215Z","shell.execute_reply.started":"2023-09-27T01:11:37.544632Z","shell.execute_reply":"2023-09-27T01:11:37.553287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_cpu_core_count():\n    try:\n        # Get the number of CPU cores\n        cpu_core = multiprocessing.cpu_count()\n\n        return cpu_core\n    except Exception as e:\n        return None\n\ndef get_gpu_core_count():\n    try:\n        # Initialize NVML\n        pynvml.nvmlInit()\n\n        # Get the number of available GPUs\n        gpu_count = pynvml.nvmlDeviceGetCount()\n\n        # Shutdown NVML\n        pynvml.nvmlShutdown()\n\n        return gpu_count\n\n    except Exception as e:\n        return None","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.556092Z","iopub.execute_input":"2023-09-27T01:11:37.556649Z","iopub.status.idle":"2023-09-27T01:11:37.564315Z","shell.execute_reply.started":"2023-09-27T01:11:37.556614Z","shell.execute_reply":"2023-09-27T01:11:37.563145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_bytes(file_size):\n    # Define the units and their corresponding labels\n    units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']\n    \n    # Determine the appropriate unit to use\n    unit_index = 0\n    while file_size >= 1024 and unit_index < len(units) - 1:\n        file_size /= 1024.0\n        unit_index += 1\n    \n    # Format the file size with the unit label\n    return f\"{file_size:.2f} {units[unit_index]}\"\n\ndef getSize(file):\n    try:\n        file_size = os.path.getsize(file)\n\n        # Extract the file name using regex\n        file_name = re.search(r'([^/\\\\]+)$', file).group(1)\n\n        print(f\"File {file_name} size is {convert_bytes(file_size)}\")\n    except FileNotFoundError:\n        print(\"File not found.\")\n    except OSError:\n        print(\"OS error occurred.\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.566281Z","iopub.execute_input":"2023-09-27T01:11:37.566899Z","iopub.status.idle":"2023-09-27T01:11:37.577184Z","shell.execute_reply.started":"2023-09-27T01:11:37.566864Z","shell.execute_reply":"2023-09-27T01:11:37.575904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@get_time\ndef readCSV(file):\n    # Read sample_submission\n    data = spark_1.read \\\n        .option(\"header\", \"true\") \\\n        .option(\"inferSchema\", \"true\") \\\n        .csv(file) \\\n        .cache()\n    print(f\"Number of Partitions -> {data.rdd.getNumPartitions()}\")\n    data.write.format(\"noop\").mode(\"overwrite\").save()\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.583272Z","iopub.execute_input":"2023-09-27T01:11:37.583706Z","iopub.status.idle":"2023-09-27T01:11:37.590335Z","shell.execute_reply.started":"2023-09-27T01:11:37.583666Z","shell.execute_reply":"2023-09-27T01:11:37.589102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@get_time\ndef readParquet(file):\n    data = spark_2.read \\\n        .option(\"header\", \"true\") \\\n        .option(\"inferSchema\", \"true\") \\\n        .parquet(file) \\\n        .cache()\n    print(f\"Number of Partitions -> {data.rdd.getNumPartitions()}\")\n    data.write.format(\"noop\").mode(\"overwrite\").save()\n    return data","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.592561Z","iopub.execute_input":"2023-09-27T01:11:37.593189Z","iopub.status.idle":"2023-09-27T01:11:37.601406Z","shell.execute_reply.started":"2023-09-27T01:11:37.593151Z","shell.execute_reply":"2023-09-27T01:11:37.600554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class AnalyseData:\n    def __init__(self, data):\n        self.data = data\n    \n    @get_time\n    def countData(self):\n        return self.data.count()\n\n    @get_time\n    def describeData(self):\n        return self.data.describe()\n    \n    @get_time\n    def printSchemaData(self):\n        return self.data.printSchema()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.603320Z","iopub.execute_input":"2023-09-27T01:11:37.604297Z","iopub.status.idle":"2023-09-27T01:11:37.611718Z","shell.execute_reply.started":"2023-09-27T01:11:37.604257Z","shell.execute_reply":"2023-09-27T01:11:37.610705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# List files in the input directory\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.615266Z","iopub.execute_input":"2023-09-27T01:11:37.616606Z","iopub.status.idle":"2023-09-27T01:11:37.624056Z","shell.execute_reply.started":"2023-09-27T01:11:37.616569Z","shell.execute_reply":"2023-09-27T01:11:37.623017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# SparkSession builder\ngpu_memory = get_free_gpu_memory()\ncpu_cores = get_cpu_core_count()\ngpu_cores = get_gpu_core_count()\n\nmaster = 'local[*]'\nmaxMemory = '8G'\napp_name = 'Child Mind Institute Detect Sleep States'\nmaxBytes= str(gpu_memory) + \"b\"\n\nconf = {\n    'spark.dynamicAllocation.enabled': 'true',\n    'spark.dynamicAllocation.maxExecutors': 8,\n    \n    'spark.shuffle.service.enabled': 'true',\n    \n    'spark.memory.fraction':'0.6',\n    \n    'spark.executor.memoryOverhead':maxMemory,\n    'spark.executor.memory': maxMemory,\n    'spark.executor.cores': cpu_cores,\n    'spark.executor.resource.gpu.amount': gpu_cores * 3,\n    'spark.executor.resource.gpu.memoryFraction':'0.6',\n    \n    'spark.rapids.memory.pinnedPool.size': maxBytes,\n    'spark.rapids.sql.enabled': 'true',\n    \n    'spark.driver.memory': maxMemory,\n    'spark.driver.maxResultsSize': maxMemory,\n    \n    'spark.sql.repl.eagerEval.enabled': 'true',\n    'spark.sql.files.maxPartitionBytes': maxBytes,\n    'spark.sql.files.maxPartitionNum': cpu_cores * 3,\n    'spark.sql.files.minPartitionNum': cpu_cores * 2,\n    'spark.sql.adaptive.enabled': 'false',\n    'spark.sql.adaptive.localShuffleReader.enabled': 'false',\n    'spark.sql.adaptive.coalescePartitions.enabled': 'false',\n    'spark.sql.sources.partitionOverwriteMode': 'dynamic',\n    'spark.sql.shuffle.partitions': '4',\n}\n\nbuilder = (\n    SparkSession\n    .builder\n    .master(master)\n    .appName(app_name)\n)\n\n# Apply Spark configurations using a loop\nfor k, v in conf.items():\n    builder.config(k, v)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.626481Z","iopub.execute_input":"2023-09-27T01:11:37.627421Z","iopub.status.idle":"2023-09-27T01:11:37.646709Z","shell.execute_reply.started":"2023-09-27T01:11:37.627392Z","shell.execute_reply":"2023-09-27T01:11:37.645727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark_1 = builder.getOrCreate()\nspark_1.sparkContext.setLogLevel(\"ERROR\")\nspark_1","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:37.648086Z","iopub.execute_input":"2023-09-27T01:11:37.648998Z","iopub.status.idle":"2023-09-27T01:11:45.412310Z","shell.execute_reply.started":"2023-09-27T01:11:37.648963Z","shell.execute_reply":"2023-09-27T01:11:45.411201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark_2 = builder.getOrCreate()\nspark_2.sparkContext.setLogLevel(\"ERROR\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:45.414080Z","iopub.execute_input":"2023-09-27T01:11:45.414786Z","iopub.status.idle":"2023-09-27T01:11:45.438427Z","shell.execute_reply.started":"2023-09-27T01:11:45.414702Z","shell.execute_reply":"2023-09-27T01:11:45.437374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sc = spark.sparkContext\n# sc","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:45.439928Z","iopub.execute_input":"2023-09-27T01:11:45.440590Z","iopub.status.idle":"2023-09-27T01:11:45.445244Z","shell.execute_reply.started":"2023-09-27T01:11:45.440538Z","shell.execute_reply":"2023-09-27T01:11:45.444223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sqlContext = SQLContext(spark.sparkContext)\n# sqlContext","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:45.446676Z","iopub.execute_input":"2023-09-27T01:11:45.447849Z","iopub.status.idle":"2023-09-27T01:11:45.454315Z","shell.execute_reply.started":"2023-09-27T01:11:45.447792Z","shell.execute_reply":"2023-09-27T01:11:45.453192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_dataset='../input/child-mind-institute-detect-sleep-states/train_series.parquet'\nsample_submission_dataset='../input/child-mind-institute-detect-sleep-states/sample_submission.csv'\ntrain_events_dataset='../input/child-mind-institute-detect-sleep-states/train_events.csv'\ntest_series_dataset='../input/child-mind-institute-detect-sleep-states/test_series.parquet'","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:45.455889Z","iopub.execute_input":"2023-09-27T01:11:45.456892Z","iopub.status.idle":"2023-09-27T01:11:45.462529Z","shell.execute_reply.started":"2023-09-27T01:11:45.456855Z","shell.execute_reply":"2023-09-27T01:11:45.461467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(getSize(train_series_dataset))\nprint(getSize(sample_submission_dataset))\nprint(getSize(train_events_dataset))\nprint(getSize(test_series_dataset))","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:45.464101Z","iopub.execute_input":"2023-09-27T01:11:45.464952Z","iopub.status.idle":"2023-09-27T01:11:45.476833Z","shell.execute_reply.started":"2023-09-27T01:11:45.464913Z","shell.execute_reply":"2023-09-27T01:11:45.475734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series, execution_time = readParquet(train_series_dataset)\ntrain_series","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:11:45.479760Z","iopub.execute_input":"2023-09-27T01:11:45.480399Z","iopub.status.idle":"2023-09-27T01:16:36.671387Z","shell.execute_reply.started":"2023-09-27T01:11:45.480357Z","shell.execute_reply":"2023-09-27T01:16:36.670315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sample_submission, execution_time = readCSV(sample_submission_dataset)\n# sample_submission","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:36.674885Z","iopub.execute_input":"2023-09-27T01:16:36.675234Z","iopub.status.idle":"2023-09-27T01:16:36.680384Z","shell.execute_reply.started":"2023-09-27T01:16:36.675203Z","shell.execute_reply":"2023-09-27T01:16:36.679243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events, execution_time = readCSV(train_events_dataset)\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:36.682274Z","iopub.execute_input":"2023-09-27T01:16:36.683044Z","iopub.status.idle":"2023-09-27T01:16:39.798846Z","shell.execute_reply.started":"2023-09-27T01:16:36.683005Z","shell.execute_reply":"2023-09-27T01:16:39.797796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_series, execution_time = readParquet(test_series_dataset)\n# test_series","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:39.800604Z","iopub.execute_input":"2023-09-27T01:16:39.800979Z","iopub.status.idle":"2023-09-27T01:16:39.810631Z","shell.execute_reply.started":"2023-09-27T01:16:39.800948Z","shell.execute_reply":"2023-09-27T01:16:39.809565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Analyse train_series for some data overview","metadata":{}},{"cell_type":"code","source":"cleaned_train_series = reduce_mem_usage(train_series.na.drop())","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:39.812360Z","iopub.execute_input":"2023-09-27T01:16:39.813209Z","iopub.status.idle":"2023-09-27T01:16:39.938792Z","shell.execute_reply.started":"2023-09-27T01:16:39.813174Z","shell.execute_reply":"2023-09-27T01:16:39.937758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_train_series = cleaned_train_series \\\n        .withColumn(\"date\", date_format(\"timestamp\", \"yyyy-MM-dd\")) \\\n        .withColumn(\"time\", date_format(\"timestamp\", \"hh:mm:ss\")) \\\n        .withColumn(\"hour\", date_format(\"timestamp\", \"hh\")) \\\n        .withColumn(\"hour\", col(\"hour\").cast(\"int\"))","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:39.939829Z","iopub.execute_input":"2023-09-27T01:16:39.940163Z","iopub.status.idle":"2023-09-27T01:16:40.039728Z","shell.execute_reply.started":"2023-09-27T01:16:39.940121Z","shell.execute_reply":"2023-09-27T01:16:40.038750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:40.040756Z","iopub.execute_input":"2023-09-27T01:16:40.041070Z","iopub.status.idle":"2023-09-27T01:16:41.339831Z","shell.execute_reply.started":"2023-09-27T01:16:40.041041Z","shell.execute_reply":"2023-09-27T01:16:41.338594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_series_analyzer = AnalyseData(cleaned_train_series)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:41.354514Z","iopub.execute_input":"2023-09-27T01:16:41.357776Z","iopub.status.idle":"2023-09-27T01:16:41.365742Z","shell.execute_reply.started":"2023-09-27T01:16:41.357723Z","shell.execute_reply":"2023-09-27T01:16:41.364293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count, count_time = train_series_analyzer.countData()\ndescription, describe_time = train_series_analyzer.describeData()\nprintSchema, printSchema_time = train_series_analyzer.printSchemaData()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:16:41.371526Z","iopub.execute_input":"2023-09-27T01:16:41.374823Z","iopub.status.idle":"2023-09-27T01:17:03.602406Z","shell.execute_reply.started":"2023-09-27T01:16:41.374779Z","shell.execute_reply":"2023-09-27T01:17:03.601405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Count:\", count)\nprint(\"Execution time for countData+:\", count_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:17:03.604933Z","iopub.execute_input":"2023-09-27T01:17:03.606108Z","iopub.status.idle":"2023-09-27T01:17:03.615863Z","shell.execute_reply.started":"2023-09-27T01:17:03.606067Z","shell.execute_reply":"2023-09-27T01:17:03.614683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Description:\", description)\nprint(\"Execution time for describeData:\", describe_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:17:03.621950Z","iopub.execute_input":"2023-09-27T01:17:03.622846Z","iopub.status.idle":"2023-09-27T01:53:55.754697Z","shell.execute_reply.started":"2023-09-27T01:17:03.622811Z","shell.execute_reply":"2023-09-27T01:53:55.753384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"printSchema:\", printSchema)\nprint(\"Execution time for printSchema:\", printSchema_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:53:55.757156Z","iopub.execute_input":"2023-09-27T01:53:55.757572Z","iopub.status.idle":"2023-09-27T01:53:55.766657Z","shell.execute_reply.started":"2023-09-27T01:53:55.757533Z","shell.execute_reply":"2023-09-27T01:53:55.765659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"string_columns = ['series_id', 'timestamp']\nnumeric_columns = ['step','anglez','enmo']\nmissing_values = {}\n\nfor index, column in enumerate(cleaned_train_series.columns):\n    if column in string_columns:\n        missing_count = cleaned_train_series.filter(col(column).eqNullSafe(None) | col(column).isNull()).count()\n        missing_values.update({column:missing_count})\n    if column in numeric_columns: \n        missing_count = cleaned_train_series.where(col(column).isin([0,None,np.nan])).count()\n        missing_values.update({column:missing_count})\n        \nmissing_df = pd.DataFrame.from_dict([missing_values])\nmissing_df","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:53:55.768824Z","iopub.execute_input":"2023-09-27T01:53:55.769848Z","iopub.status.idle":"2023-09-27T01:54:55.360034Z","shell.execute_reply.started":"2023-09-27T01:53:55.769810Z","shell.execute_reply":"2023-09-27T01:54:55.358916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_anglez = cleaned_train_series.where(cleaned_train_series.anglez < 0)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:54:55.365281Z","iopub.execute_input":"2023-09-27T01:54:55.367956Z","iopub.status.idle":"2023-09-27T01:54:55.389471Z","shell.execute_reply.started":"2023-09-27T01:54:55.367915Z","shell.execute_reply":"2023-09-27T01:54:55.388441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"negative_anglez_summary = negative_anglez.select('anglez').summary()\nprint('Number of anglez that have less than 0 records and statistic summary:')\nnegative_anglez_summary","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:54:55.394597Z","iopub.execute_input":"2023-09-27T01:54:55.395925Z","iopub.status.idle":"2023-09-27T01:57:10.888842Z","shell.execute_reply.started":"2023-09-27T01:54:55.395888Z","shell.execute_reply":"2023-09-27T01:57:10.887803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_anglez = cleaned_train_series.where(cleaned_train_series.anglez == 0)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:57:10.889946Z","iopub.execute_input":"2023-09-27T01:57:10.890445Z","iopub.status.idle":"2023-09-27T01:57:10.922902Z","shell.execute_reply.started":"2023-09-27T01:57:10.890405Z","shell.execute_reply":"2023-09-27T01:57:10.921979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"zero_anglez_summary = zero_anglez.select('anglez').summary()\nprint('Number of anglez that have 0 records and statistic summary:')\nzero_anglez_summary","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:57:10.923864Z","iopub.execute_input":"2023-09-27T01:57:10.924196Z","iopub.status.idle":"2023-09-27T01:57:50.255197Z","shell.execute_reply.started":"2023-09-27T01:57:10.924166Z","shell.execute_reply":"2023-09-27T01:57:50.253666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_anglez = cleaned_train_series.where(cleaned_train_series.anglez > 0)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:57:50.256987Z","iopub.execute_input":"2023-09-27T01:57:50.257363Z","iopub.status.idle":"2023-09-27T01:57:50.276958Z","shell.execute_reply.started":"2023-09-27T01:57:50.257328Z","shell.execute_reply":"2023-09-27T01:57:50.276040Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"positive_anglez_summary = positive_anglez.select('anglez').summary()\nprint('Number of anglez that have more than 0 records and statistic summary:')\npositive_anglez_summary","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:57:50.277995Z","iopub.execute_input":"2023-09-27T01:57:50.278349Z","iopub.status.idle":"2023-09-27T01:59:25.494877Z","shell.execute_reply.started":"2023-09-27T01:57:50.278314Z","shell.execute_reply":"2023-09-27T01:59:25.493858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_train_series","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:25.495874Z","iopub.execute_input":"2023-09-27T01:59:25.496205Z","iopub.status.idle":"2023-09-27T01:59:25.888765Z","shell.execute_reply.started":"2023-09-27T01:59:25.496174Z","shell.execute_reply":"2023-09-27T01:59:25.887852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# hour_values = cleaned_train_series \\\n#                 .select(\"hour\") \\\n#                 .rdd \\\n#                 .flatMap(lambda x: x) \\\n#                 .collect()\n\n# plt.figure(figsize=(6, 6))\n# plt.title('Hours Distribution')\n# sns.kdeplot(hour_values, color='red')\n# plt.xlabel('Hour')\n# plt.ylabel('Density')\n# plt.show()\n# spark_1.stop()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:25.903540Z","iopub.execute_input":"2023-09-27T01:59:25.904205Z","iopub.status.idle":"2023-09-27T01:59:25.918984Z","shell.execute_reply.started":"2023-09-27T01:59:25.904172Z","shell.execute_reply":"2023-09-27T01:59:25.918070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:25.920856Z","iopub.execute_input":"2023-09-27T01:59:25.921574Z","iopub.status.idle":"2023-09-27T01:59:27.148578Z","shell.execute_reply.started":"2023-09-27T01:59:25.921542Z","shell.execute_reply":"2023-09-27T01:59:27.147341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = train_events.withColumn(\"date\", date_format(\"timestamp\", \"yyyy-MM-dd\")).withColumn(\"month\", date_format(\"timestamp\", \"yyyy-MM\")).withColumn(\"time\", date_format(\"timestamp\", \"hh:mm:ss\")).withColumn(\"hour\", date_format(\"timestamp\", \"hh\"))\ntrain_events","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:27.150787Z","iopub.execute_input":"2023-09-27T01:59:27.151233Z","iopub.status.idle":"2023-09-27T01:59:27.801151Z","shell.execute_reply.started":"2023-09-27T01:59:27.151194Z","shell.execute_reply":"2023-09-27T01:59:27.794524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_train_events = train_events.na.drop()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:27.802211Z","iopub.execute_input":"2023-09-27T01:59:27.802572Z","iopub.status.idle":"2023-09-27T01:59:27.841636Z","shell.execute_reply.started":"2023-09-27T01:59:27.802538Z","shell.execute_reply":"2023-09-27T01:59:27.831437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleaned_train_events","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:27.842687Z","iopub.execute_input":"2023-09-27T01:59:27.843035Z","iopub.status.idle":"2023-09-27T01:59:28.503864Z","shell.execute_reply.started":"2023-09-27T01:59:27.843003Z","shell.execute_reply":"2023-09-27T01:59:28.502926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events_analyzer = AnalyseData(cleaned_train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:28.504861Z","iopub.execute_input":"2023-09-27T01:59:28.505196Z","iopub.status.idle":"2023-09-27T01:59:28.513712Z","shell.execute_reply.started":"2023-09-27T01:59:28.505167Z","shell.execute_reply":"2023-09-27T01:59:28.511878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count, count_time = train_events_analyzer.countData()\ndescription, describe_time = train_events_analyzer.describeData()\nprintSchema, printSchema_time = train_events_analyzer.printSchemaData()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:28.515459Z","iopub.execute_input":"2023-09-27T01:59:28.520718Z","iopub.status.idle":"2023-09-27T01:59:28.976779Z","shell.execute_reply.started":"2023-09-27T01:59:28.520683Z","shell.execute_reply":"2023-09-27T01:59:28.975746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Count:\", count)\nprint(\"Execution time for countData+:\", count_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:28.977918Z","iopub.execute_input":"2023-09-27T01:59:28.979859Z","iopub.status.idle":"2023-09-27T01:59:28.989477Z","shell.execute_reply.started":"2023-09-27T01:59:28.979822Z","shell.execute_reply":"2023-09-27T01:59:28.988468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Description:\", description)\nprint(\"Execution time for describeData:\", describe_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:28.991171Z","iopub.execute_input":"2023-09-27T01:59:28.992162Z","iopub.status.idle":"2023-09-27T01:59:30.106282Z","shell.execute_reply.started":"2023-09-27T01:59:28.992127Z","shell.execute_reply":"2023-09-27T01:59:30.105331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"printSchema:\", printSchema)\nprint(\"Execution time for printSchema:\", printSchema_time, \"ms\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:30.107358Z","iopub.execute_input":"2023-09-27T01:59:30.107828Z","iopub.status.idle":"2023-09-27T01:59:30.114418Z","shell.execute_reply.started":"2023-09-27T01:59:30.107794Z","shell.execute_reply":"2023-09-27T01:59:30.113242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"events = [row['event'] for row in cleaned_train_events.select(\"event\").distinct().collect()]","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:30.116309Z","iopub.execute_input":"2023-09-27T01:59:30.117450Z","iopub.status.idle":"2023-09-27T01:59:30.624565Z","shell.execute_reply.started":"2023-09-27T01:59:30.117417Z","shell.execute_reply":"2023-09-27T01:59:30.623510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onset_value = cleaned_train_events.where(cleaned_train_events.event == \"onset\")\nwakeup_value = cleaned_train_events.where(cleaned_train_events.event == \"wakeup\")","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:30.629327Z","iopub.execute_input":"2023-09-27T01:59:30.629705Z","iopub.status.idle":"2023-09-27T01:59:30.646004Z","shell.execute_reply.started":"2023-09-27T01:59:30.629670Z","shell.execute_reply":"2023-09-27T01:59:30.645043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wakeup_value_summary = wakeup_value.select('event').count()\nonset_value_summary = onset_value.select('event').count()\ncounts=[onset_value_summary,wakeup_value_summary]","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:30.651064Z","iopub.execute_input":"2023-09-27T01:59:30.651864Z","iopub.status.idle":"2023-09-27T01:59:31.069788Z","shell.execute_reply.started":"2023-09-27T01:59:30.651824Z","shell.execute_reply":"2023-09-27T01:59:31.067569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Number of onset count:', onset_value_summary)\nprint('Number of wakeup count:', wakeup_value_summary)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:31.072392Z","iopub.execute_input":"2023-09-27T01:59:31.076746Z","iopub.status.idle":"2023-09-27T01:59:31.084840Z","shell.execute_reply.started":"2023-09-27T01:59:31.076709Z","shell.execute_reply":"2023-09-27T01:59:31.083643Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Define custom colors\ncolors = ['#f4c095', '#679289']","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:31.086617Z","iopub.execute_input":"2023-09-27T01:59:31.087046Z","iopub.status.idle":"2023-09-27T01:59:31.095123Z","shell.execute_reply.started":"2023-09-27T01:59:31.086999Z","shell.execute_reply":"2023-09-27T01:59:31.093982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(6, 5))\nplt.bar(events, counts, color=colors)\nplt.xlabel('Event')\nplt.ylabel('Count')\nplt.title('Event Counts')\nplt.grid(axis='y', linestyle='--', alpha=0.7)\n\n# Display the counts on top of the bars\nfor i, count in enumerate(counts):\n    plt.text(i, count + 0.5, str(count), ha='center', va='bottom', fontsize=8)\n\n# Show the plot\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:31.096935Z","iopub.execute_input":"2023-09-27T01:59:31.097306Z","iopub.status.idle":"2023-09-27T01:59:31.499447Z","shell.execute_reply.started":"2023-09-27T01:59:31.097270Z","shell.execute_reply":"2023-09-27T01:59:31.498369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"each_night_count = cleaned_train_events \\\n                    .groupBy(\"night\") \\\n                    .count() \\\n                    .orderBy(asc(\"night\"))\neach_night_count","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:31.500843Z","iopub.execute_input":"2023-09-27T01:59:31.501830Z","iopub.status.idle":"2023-09-27T01:59:32.371396Z","shell.execute_reply.started":"2023-09-27T01:59:31.501791Z","shell.execute_reply":"2023-09-27T01:59:32.370428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = each_night_count \\\n    .select(\"night\") \\\n    .rdd \\\n    .flatMap(lambda x: x) \\\n    .collect()\n\ny = each_night_count \\\n    .select(\"count\") \\\n    .rdd \\\n    .flatMap(lambda x: x) \\\n    .collect()\n\nplt.figure(figsize=(10, 6))\nplt.bar(x,y)\nplt.xlabel(\"Night\")\nplt.ylabel(\"Count\")\nplt.title(\"Night Count Distribution\")\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:32.372942Z","iopub.execute_input":"2023-09-27T01:59:32.375145Z","iopub.status.idle":"2023-09-27T01:59:37.035425Z","shell.execute_reply.started":"2023-09-27T01:59:32.375105Z","shell.execute_reply":"2023-09-27T01:59:37.034439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"night_count = [row['night'] for row in cleaned_train_events \\\n                   .select(\"night\") \\\n                   .distinct() \\\n                   .collect()]\n\n# Sort the list of \"night\" values\nnight_count = sorted(night_count)\n\nprint(night_count)","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:37.037089Z","iopub.execute_input":"2023-09-27T01:59:37.037776Z","iopub.status.idle":"2023-09-27T01:59:37.268453Z","shell.execute_reply.started":"2023-09-27T01:59:37.037741Z","shell.execute_reply":"2023-09-27T01:59:37.267547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"each_month_count = cleaned_train_events \\\n    .groupBy(\"month\") \\\n    .count() \\\n    .orderBy(asc(\"month\"))\neach_month_count","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:37.270157Z","iopub.execute_input":"2023-09-27T01:59:37.270845Z","iopub.status.idle":"2023-09-27T01:59:38.037872Z","shell.execute_reply.started":"2023-09-27T01:59:37.270809Z","shell.execute_reply":"2023-09-27T01:59:38.036936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_date_count, stddev_date_count = each_month_count.select(mean(col('count')), stddev(col('count'))).first()\n\nz_score_threshold = 5\n\n# Filter the DataFrame to keep only rows with z-scores within the threshold\ncleaned_date_df = each_month_count.filter(\n    (col('count') >= mean_date_count - z_score_threshold * stddev_date_count) &\n    (col('count') <= mean_date_count + z_score_threshold * stddev_date_count)\n)\ncleaned_date_df","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:38.038922Z","iopub.execute_input":"2023-09-27T01:59:38.039282Z","iopub.status.idle":"2023-09-27T01:59:39.140344Z","shell.execute_reply.started":"2023-09-27T01:59:38.039248Z","shell.execute_reply":"2023-09-27T01:59:39.139373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = cleaned_date_df \\\n        .select(\"month\") \\\n        .rdd \\\n        .flatMap(lambda x: x) \\\n        .collect()\n\ny = cleaned_date_df \\\n        .select(\"count\") \\\n        .rdd \\\n        .flatMap(lambda x: x) \\\n        .collect()\n\nplt.figure(figsize=(10, 6))\nplt.bar(x,y)\nplt.xlabel(\"Month\")\nplt.ylabel(\"Count\")\nplt.title(\"Month Count Distribution\")\nplt.axhline(mean_date_count, color='red', linestyle='dashed', linewidth=2, label='Mean Count')\nplt.xticks(rotation=50)\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-27T01:59:39.144906Z","iopub.execute_input":"2023-09-27T01:59:39.145293Z","iopub.status.idle":"2023-09-27T01:59:41.967524Z","shell.execute_reply.started":"2023-09-27T01:59:39.145256Z","shell.execute_reply":"2023-09-27T01:59:41.966545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}