{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"
Some quick investigations to the train_events.csv
\n

Will be extended to other data soon

\n\n***","metadata":{}},{"cell_type":"markdown","source":"
\n

\n1. Import needed libraries\n

\n
","metadata":{}},{"cell_type":"code","source":"# Import necessary libraries\nimport numpy as np # Import NumPy for numerical operations\nimport pandas as pd # Import Pandas for data manipulation\nimport matplotlib.pyplot as plt # Import Matplotlib for data visualization\nimport plotly.express as px # Import Plotly Express for interactive plotting\nimport os # Import the OS module for operating system functions\nimport warnings # Import warnings module to suppress warnings\nwarnings.filterwarnings(\"ignore\") # Ignore warning messages\n\n# Set the style for Matplotlib plots\nplt.style.use(\"fivethirtyeight\") # Use the 'fivethirtyeight' style\nplt.rc(\"figure\", autolayout=True) # Set figure autolayout to True\nplt.rc(\n \"axes\",\n labelweight=\"bold\", # Make axis labels bold\n labelsize=\"large\", # Set axis label size to large\n titleweight=\"bold\", # Make axis titles bold\n titlesize=14, # Set axis title size\n titlepad=10, # Set padding for axis titles\n)\n\n# ANSI escape codes for text colors\nRED = \"\\033[91m\" # Red text color\nGREEN = \"\\033[92m\" # Green text color\nYELLOW = \"\\033[93m\" # Yellow text color\nBLUE = \"\\033[94m\" # Blue text color\nRESET = \"\\033[0m\" # Reset text color to default","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:10.243005Z","iopub.execute_input":"2023-09-07T09:50:10.243603Z","iopub.status.idle":"2023-09-07T09:50:11.038816Z","shell.execute_reply.started":"2023-09-07T09:50:10.243567Z","shell.execute_reply":"2023-09-07T09:50:11.037406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Walk through the directory '/kaggle/input' and its subdirectories\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n # Print the full path of each file found\n print(os.path.join(dirname, filename))\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.040608Z","iopub.execute_input":"2023-09-07T09:50:11.041342Z","iopub.status.idle":"2023-09-07T09:50:11.049374Z","shell.execute_reply.started":"2023-09-07T09:50:11.041302Z","shell.execute_reply":"2023-09-07T09:50:11.048131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read the CSV file 'train_events.csv' into a Pandas DataFrame\ntrain_events = pd.read_csv('/kaggle/input/child-mind-institute-detect-sleep-states/train_events.csv')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.051241Z","iopub.execute_input":"2023-09-07T09:50:11.051715Z","iopub.status.idle":"2023-09-07T09:50:11.089202Z","shell.execute_reply.started":"2023-09-07T09:50:11.051680Z","shell.execute_reply":"2023-09-07T09:50:11.087693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.head(15).style.set_properties(**{'background-color': 'black',\n 'color': 'lawngreen',\n 'border': '1.5px white'})","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.092997Z","iopub.execute_input":"2023-09-07T09:50:11.093461Z","iopub.status.idle":"2023-09-07T09:50:11.157959Z","shell.execute_reply.started":"2023-09-07T09:50:11.093397Z","shell.execute_reply":"2023-09-07T09:50:11.156881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 2. Manipulate date-time columns\n

\n
","metadata":{}},{"cell_type":"code","source":"# Convert 'timestamp' column to datetime format, handling errors by setting them to NaN\ntrain_events['timestamp'] = pd.to_datetime(train_events['timestamp'], errors='coerce')\n\n# Extract the date portion from the 'timestamp' column and store it in a new 'date' column\ntrain_events['date'] = train_events['timestamp'].apply(lambda x: np.nan if x is 'NaN' else str(x).split(' ')[0])\n\n# Extract the starting time portion from the 'timestamp' column and store it in a new 'time_starting' column\ntrain_events['time_starting'] = train_events['timestamp'].apply(lambda x: np.nan if x is 'NaN' else str(x).split(' ')[-1].split('-')[0])\n\n# Extract the ending time portion from the 'timestamp' column and store it in a new 'time_ending' column\ntrain_events['time_ending'] = train_events['timestamp'].apply(lambda x: np.nan if x is 'NaN' else str(x).split(' ')[-1].split('-')[-1])","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.159297Z","iopub.execute_input":"2023-09-07T09:50:11.160242Z","iopub.status.idle":"2023-09-07T09:50:11.362522Z","shell.execute_reply.started":"2023-09-07T09:50:11.160205Z","shell.execute_reply":"2023-09-07T09:50:11.361252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.head(20).style.set_properties(**{'background-color': 'black',\n 'color': 'lawngreen',\n 'border': '1.5px white'})","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.364488Z","iopub.execute_input":"2023-09-07T09:50:11.365333Z","iopub.status.idle":"2023-09-07T09:50:11.386959Z","shell.execute_reply.started":"2023-09-07T09:50:11.365280Z","shell.execute_reply":"2023-09-07T09:50:11.385239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 3. Check basic informations of dataframe train_events.csv\n

\n
","metadata":{}},{"cell_type":"code","source":"def information_func(df):\n # Display metadata of the dataset\n print(RED + \"Metadata of the dataset:\\n\" + RESET)\n display(df.info())\n print(\"----\" * 20)\n \n # Display a description of the dataset\n print(RED + \"Description of the dataset:\\n\" + RESET)\n display(df.describe())\n print(\"----\" * 20)\n \n # Display missing values\n print(RED + 'Missing values:\\n' + RESET)\n null = df.isnull().sum()\n print(null)\n print(\"----\" * 20)\n \n # Display value counts for each column\n for col in df.columns:\n print(RED + f'Number of unique values: {df[col].nunique()}' + RESET)\n print('Value counts:\\n')\n print(df[col].value_counts())\n print(\"----\" * 20)\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.388817Z","iopub.execute_input":"2023-09-07T09:50:11.389737Z","iopub.status.idle":"2023-09-07T09:50:11.398338Z","shell.execute_reply.started":"2023-09-07T09:50:11.389679Z","shell.execute_reply":"2023-09-07T09:50:11.397250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"information_func(train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.399860Z","iopub.execute_input":"2023-09-07T09:50:11.402322Z","iopub.status.idle":"2023-09-07T09:50:11.511036Z","shell.execute_reply.started":"2023-09-07T09:50:11.402275Z","shell.execute_reply":"2023-09-07T09:50:11.509763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Group the 'train_events' DataFrame by 'series_id' and count the occurrences in each group\ndf_series_id = train_events.groupby('series_id').count().reset_index().iloc[:, :2]\n\n# Rename the 'night' column to 'counts' for clarity\ndf_series_id = df_series_id.rename(columns={'night': 'counts'})\n\n# Create a bar plot using Plotly Express to visualize counts per 'series_id'\nfig = px.bar(df_series_id, x='series_id', y='counts', title='Counts per series_id')\n\n# Show the plot\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:11.512785Z","iopub.execute_input":"2023-09-07T09:50:11.513199Z","iopub.status.idle":"2023-09-07T09:50:12.086298Z","shell.execute_reply.started":"2023-09-07T09:50:11.513163Z","shell.execute_reply":"2023-09-07T09:50:12.084891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_event_count = train_events.groupby('event').count().reset_index().iloc[:,:2].rename(columns={'series_id': 'counts'})\nfig=px.bar(df_event_count, x='event', y='counts', title='counts per event')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.088083Z","iopub.execute_input":"2023-09-07T09:50:12.088802Z","iopub.status.idle":"2023-09-07T09:50:12.166023Z","shell.execute_reply.started":"2023-09-07T09:50:12.088668Z","shell.execute_reply":"2023-09-07T09:50:12.164516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_time_ending_count = train_events.groupby('time_ending').count().reset_index().iloc[:,:2].rename(columns={'series_id': 'counts'})\nfig=px.bar(df_time_ending_count, x='time_ending', y='counts', title='counts per ending time')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.167619Z","iopub.execute_input":"2023-09-07T09:50:12.168016Z","iopub.status.idle":"2023-09-07T09:50:12.242161Z","shell.execute_reply.started":"2023-09-07T09:50:12.167982Z","shell.execute_reply":"2023-09-07T09:50:12.240796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_date = train_events.groupby('date').count().reset_index().iloc[:,:2].rename(columns={'series_id': 'counts'}).iloc[:-1, :]\nfig=px.bar(df_date, x='date', y='counts', title='counts per date')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.243721Z","iopub.execute_input":"2023-09-07T09:50:12.244887Z","iopub.status.idle":"2023-09-07T09:50:12.321571Z","shell.execute_reply.started":"2023-09-07T09:50:12.244844Z","shell.execute_reply":"2023-09-07T09:50:12.320088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 4. Handle missed data for the Step column, via filling nan with median.\n

\n
","metadata":{}},{"cell_type":"code","source":"# Group the 'train_events' DataFrame by both 'series_id' and 'event', and calculate the median of the 'step' column for each group\ndf_median_step_per_series_id_and_event = train_events.groupby(['series_id', 'event'])['step'].median().to_frame().reset_index()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.326200Z","iopub.execute_input":"2023-09-07T09:50:12.326683Z","iopub.status.idle":"2023-09-07T09:50:12.339978Z","shell.execute_reply.started":"2023-09-07T09:50:12.326648Z","shell.execute_reply":"2023-09-07T09:50:12.338821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_median_step_per_series_id_and_event.head(15).style.set_properties(**{'background-color': 'black',\n 'color': 'lawngreen',\n 'border': '1.5px white'})","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.341598Z","iopub.execute_input":"2023-09-07T09:50:12.342851Z","iopub.status.idle":"2023-09-07T09:50:12.362156Z","shell.execute_reply.started":"2023-09-07T09:50:12.342804Z","shell.execute_reply":"2023-09-07T09:50:12.360528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"information_func(df_median_step_per_series_id_and_event)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.363904Z","iopub.execute_input":"2023-09-07T09:50:12.364550Z","iopub.status.idle":"2023-09-07T09:50:12.408228Z","shell.execute_reply.started":"2023-09-07T09:50:12.364502Z","shell.execute_reply":"2023-09-07T09:50:12.406895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter rows in the DataFrame where the 'step' column has null (NaN) values\nfiltered_df = df_median_step_per_series_id_and_event[df_median_step_per_series_id_and_event['step'].isnull()]\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.409711Z","iopub.execute_input":"2023-09-07T09:50:12.410966Z","iopub.status.idle":"2023-09-07T09:50:12.418979Z","shell.execute_reply.started":"2023-09-07T09:50:12.410925Z","shell.execute_reply":"2023-09-07T09:50:12.417315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter rows in the DataFrame where 'series_id' is \"038441c925bb\" and 'event' is \"onset\"\nfiltered_df = df_median_step_per_series_id_and_event.query('series_id == \"038441c925bb\" and event == \"onset\"')\n","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.420495Z","iopub.execute_input":"2023-09-07T09:50:12.420895Z","iopub.status.idle":"2023-09-07T09:50:12.440104Z","shell.execute_reply.started":"2023-09-07T09:50:12.420863Z","shell.execute_reply":"2023-09-07T09:50:12.438520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fill_na_step(df):\n # Check for null values in the 'step' column\n isnull_step = df['step'].isnull()\n \n # Iterate through the DataFrame to fill null 'step' values\n for idx, item in enumerate(isnull_step):\n if item:\n # Get the 'series_id' and 'event' values for the current row\n series_id_temp, event_temp = df.iloc[idx, 0], df.iloc[idx, 2]\n \n # Query the 'df_madien_step_per_series_id_and_event' DataFrame to find the median 'step' value\n median_step = df_median_step_per_series_id_and_event.query('series_id == @series_id_temp and event == @event_temp')['step']\n \n # Update the 'step' value in the original DataFrame\n df.iloc[idx, 3] = median_step\n \n # Return the DataFrame with filled 'step' values\n return df","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.441281Z","iopub.execute_input":"2023-09-07T09:50:12.442343Z","iopub.status.idle":"2023-09-07T09:50:12.458092Z","shell.execute_reply.started":"2023-09-07T09:50:12.442284Z","shell.execute_reply":"2023-09-07T09:50:12.456576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events = fill_na_step(train_events)","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:12.459700Z","iopub.execute_input":"2023-09-07T09:50:12.460133Z","iopub.status.idle":"2023-09-07T09:50:31.517311Z","shell.execute_reply.started":"2023-09-07T09:50:12.460082Z","shell.execute_reply":"2023-09-07T09:50:31.515980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_events.head(20).style.set_properties(**{'background-color': 'black',\n 'color': 'lawngreen',\n 'border': '1.5px white'})","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:50:31.518883Z","iopub.execute_input":"2023-09-07T09:50:31.519609Z","iopub.status.idle":"2023-09-07T09:50:31.542638Z","shell.execute_reply.started":"2023-09-07T09:50:31.519563Z","shell.execute_reply":"2023-09-07T09:50:31.541403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\n

\n 5. Distribution of step per series_id, event, date and night\n

\n
","metadata":{}},{"cell_type":"code","source":"def plot_distribution_step(df, l: list, m: str):\n for item in l:\n fig = px.histogram(data_frame=df, x='step', color=item, marginal=m, title=f'Distribution of step with various {item}')\n fig.show()","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:51:56.637482Z","iopub.execute_input":"2023-09-07T09:51:56.638046Z","iopub.status.idle":"2023-09-07T09:51:56.645091Z","shell.execute_reply.started":"2023-09-07T09:51:56.638003Z","shell.execute_reply":"2023-09-07T09:51:56.643528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list_distribution_step = ['series_id', 'night', 'event', 'date']\nplot_distribution_step(df=train_events, l=list_distribution_step, m='rug')","metadata":{"execution":{"iopub.status.busy":"2023-09-07T09:51:58.836161Z","iopub.execute_input":"2023-09-07T09:51:58.836663Z","iopub.status.idle":"2023-09-07T09:52:08.338126Z","shell.execute_reply.started":"2023-09-07T09:51:58.836623Z","shell.execute_reply":"2023-09-07T09:52:08.336779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}],"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}}