{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87331,"databundleVersionId":9932741,"sourceType":"competition"}],"dockerImageVersionId":30715,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Prep","metadata":{}},{"cell_type":"code","source":"%%capture\n!pip install autogluon.eda","metadata":{"execution":{"iopub.status.busy":"2024-10-24T12:50:54.685999Z","iopub.execute_input":"2024-10-24T12:50:54.686399Z","iopub.status.idle":"2024-10-24T12:51:58.877384Z","shell.execute_reply.started":"2024-10-24T12:50:54.686367Z","shell.execute_reply":"2024-10-24T12:51:58.876041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install --upgrade pillow==10.3.0","metadata":{"execution":{"iopub.status.busy":"2024-10-24T12:51:58.879406Z","iopub.execute_input":"2024-10-24T12:51:58.879750Z","iopub.status.idle":"2024-10-24T12:52:13.311738Z","shell.execute_reply.started":"2024-10-24T12:51:58.879718Z","shell.execute_reply":"2024-10-24T12:52:13.310369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Loading","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport librosa\nimport os\n\ndf_train = pd.read_csv('/kaggle/input/airs-ai-in-respiratory-sounds/train.csv')\ndf_test = pd.read_csv('/kaggle/input/airs-ai-in-respiratory-sounds/test.csv')\ntarget_col = 'disease'","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:06:40.408719Z","iopub.execute_input":"2024-10-24T13:06:40.409658Z","iopub.status.idle":"2024-10-24T13:06:40.425456Z","shell.execute_reply.started":"2024-10-24T13:06:40.409617Z","shell.execute_reply":"2024-10-24T13:06:40.424417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Path to the folder with the audio data\ndata_path = '/kaggle/input/airs-ai-in-respiratory-sounds/sounds/sounds/'\n\n# Function to extract audio features from .wav files\ndef extract_audio_features(file_path):\n    # Load the audio file\n    y, sr = librosa.load(file_path, sr=None)\n    \n    # Check if the audio signal is empty\n    if len(y) == 0:\n        return None, None, None, None  # Return None for all features if signal is empty\n    \n    # Extract features\n    duration = librosa.get_duration(y=y, sr=sr)  # Duration in seconds\n    spectral_centroid = librosa.feature.spectral_centroid(y=y, sr=sr).mean()  # Average frequency\n    zero_crossing_rate = librosa.feature.zero_crossing_rate(y=y).mean()  # Zero-crossing rate\n    rms = librosa.feature.rms(y=y).mean()  # Root Mean Square energy\n    \n    return duration, spectral_centroid, zero_crossing_rate, rms\n\n# List to store data\ndata = []\n\n# Iterate through all subfolders\nfor candidate_id in os.listdir(data_path):\n    candidate_folder = os.path.join(data_path, candidate_id)\n    \n    if os.path.isdir(candidate_folder):\n        # Initialize variables for cough and vowel features\n        cough_features = [None, None, None, None]\n        vowel_features = [None, None, None, None]\n        \n        # Loop through files in the patient's folder\n        for file in os.listdir(candidate_folder):\n            if file.endswith('.wav'):\n                file_path = os.path.join(candidate_folder, file)\n                \n                # Extract features\n                features = extract_audio_features(file_path)\n                \n                if features[0] is None:  # Check if features are None\n                    continue  # Skip to the next file if the audio is empty\n                \n                duration, spectral_centroid, zero_crossing_rate, rms = features\n                \n                # Store features based on the type of sound\n                if 'cough' in file:\n                    cough_features = [duration, spectral_centroid, zero_crossing_rate, rms]\n                elif 'vowel' in file:\n                    vowel_features = [duration, spectral_centroid, zero_crossing_rate, rms]\n        \n        # Append the data to the list\n        data.append([candidate_id] + cough_features + vowel_features)\n\n# Create a DataFrame\ncolumns = ['candidateID', 'cough_duration', 'cough_spectral_centroid', 'cough_zero_crossing_rate', 'cough_rms',\n           'vowel_duration', 'vowel_spectral_centroid', 'vowel_zero_crossing_rate', 'vowel_rms']\nsounds_df = pd.DataFrame(data, columns=columns)\n\n# Display the first few rows of the DataFrame\nsounds_df.head()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:10:21.680673Z","iopub.execute_input":"2024-10-24T13:10:21.681187Z","iopub.status.idle":"2024-10-24T13:13:04.101835Z","shell.execute_reply.started":"2024-10-24T13:10:21.681146Z","shell.execute_reply":"2024-10-24T13:13:04.100712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-24T12:52:13.721949Z","iopub.execute_input":"2024-10-24T12:52:13.722366Z","iopub.status.idle":"2024-10-24T12:52:13.745360Z","shell.execute_reply.started":"2024-10-24T12:52:13.722331Z","shell.execute_reply":"2024-10-24T12:52:13.744095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sounds_df","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:13:39.477968Z","iopub.execute_input":"2024-10-24T13:13:39.478385Z","iopub.status.idle":"2024-10-24T13:13:39.499027Z","shell.execute_reply.started":"2024-10-24T13:13:39.478354Z","shell.execute_reply":"2024-10-24T13:13:39.497990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape, df_test.shape","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:14:02.716075Z","iopub.execute_input":"2024-10-24T13:14:02.717046Z","iopub.status.idle":"2024-10-24T13:14:02.723422Z","shell.execute_reply.started":"2024-10-24T13:14:02.717006Z","shell.execute_reply":"2024-10-24T13:14:02.722399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.merge(sounds_df, on='candidateID', how='left')","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:16:43.567804Z","iopub.execute_input":"2024-10-24T13:16:43.568600Z","iopub.status.idle":"2024-10-24T13:16:43.578969Z","shell.execute_reply.started":"2024-10-24T13:16:43.568562Z","shell.execute_reply":"2024-10-24T13:16:43.577921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = df_test.merge(sounds_df, on='candidateID', how='left')","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:16:47.282252Z","iopub.execute_input":"2024-10-24T13:16:47.283054Z","iopub.status.idle":"2024-10-24T13:16:47.292440Z","shell.execute_reply.started":"2024-10-24T13:16:47.283020Z","shell.execute_reply":"2024-10-24T13:16:47.291337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def transform_data_types(df):\n    \"\"\"\n    Transforms data types in the DataFrame according to specified conditions.\n    \n    Parameters:\n    df (pd.DataFrame): Input DataFrame containing patient data.\n    \n    Returns:\n    pd.DataFrame: DataFrame with updated data types.\n    \"\"\"\n    \n    # Convert candidateID to string\n    df['candidateID'] = df['candidateID'].astype(str)  \n    \n    # Convert age to integer\n    df['age'] = df['age'].astype(int)  \n    \n    # Convert gender to boolean\n    df['gender'] = df['gender'].astype(bool)  \n    \n    # Convert tbContactHistory to boolean\n    df['tbContactHistory'] = df['tbContactHistory'].astype(bool)  \n    \n    # Convert wheezingHistory to boolean\n    df['wheezingHistory'] = df['wheezingHistory'].astype(bool)  \n    \n    # Convert phlegmCough to boolean\n    df['phlegmCough'] = df['phlegmCough'].astype(bool)  \n    \n    # Convert familyAsthmaHistory to boolean\n    df['familyAsthmaHistory'] = df['familyAsthmaHistory'].astype(bool)  \n    \n    # Convert feverHistory to boolean\n    df['feverHistory'] = df['feverHistory'].astype(bool)  \n    \n    # Convert coldPresent to boolean\n    df['coldPresent'] = df['coldPresent'].astype(bool)  \n    \n    # Convert packYears to integer\n    df['packYears'] = df['packYears'].astype(int)  \n    \n    # Check if the disease column exists before converting\n    if 'disease' in df.columns:\n        df['disease'] = df['disease'].astype(int)  # Convert disease to integer (target variable)\n\n    return df\n\ndf_train = transform_data_types(df_train)\ndf_test = transform_data_types(df_test)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:40:54.252802Z","iopub.execute_input":"2024-10-24T13:40:54.253225Z","iopub.status.idle":"2024-10-24T13:40:54.269794Z","shell.execute_reply.started":"2024-10-24T13:40:54.253192Z","shell.execute_reply":"2024-10-24T13:40:54.268453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Overview","metadata":{}},{"cell_type":"code","source":"import autogluon.eda.auto as auto\n\nauto.dataset_overview(train_data=df_train, test_data=df_test, label=target_col)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:41:17.127368Z","iopub.execute_input":"2024-10-24T13:41:17.128082Z","iopub.status.idle":"2024-10-24T13:41:17.733210Z","shell.execute_reply.started":"2024-10-24T13:41:17.128048Z","shell.execute_reply":"2024-10-24T13:41:17.732174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Target Analysis","metadata":{}},{"cell_type":"code","source":"auto.target_analysis(\n    train_data=df_train, label=target_col, \n    # Optional; default will try to fit all available distributions\n    fit_distributions=['laplace_asymmetric', 'johnsonsu', 'exponnorm']  \n)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:41:25.976556Z","iopub.execute_input":"2024-10-24T13:41:25.976955Z","iopub.status.idle":"2024-10-24T13:41:26.892738Z","shell.execute_reply.started":"2024-10-24T13:41:25.976922Z","shell.execute_reply":"2024-10-24T13:41:26.891535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Quick Fit","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain, val = train_test_split(df_train, test_size=0.2, random_state=42)\n\nstate = auto.quick_fit(train, target_col, fit_bagging_folds=3, return_state=True, show_feature_importance_barplots=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:41:34.548738Z","iopub.execute_input":"2024-10-24T13:41:34.549183Z","iopub.status.idle":"2024-10-24T13:41:38.359296Z","shell.execute_reply.started":"2024-10-24T13:41:34.549146Z","shell.execute_reply":"2024-10-24T13:41:38.358182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Explain Rows","metadata":{}},{"cell_type":"code","source":"auto.explain_rows(\n    train_data=df_train,\n    model=state.model,\n    display_rows=True,\n    rows=state.model_evaluation.highest_error[:1]\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:41:51.356687Z","iopub.execute_input":"2024-10-24T13:41:51.357527Z","iopub.status.idle":"2024-10-24T13:41:59.034702Z","shell.execute_reply.started":"2024-10-24T13:41:51.357486Z","shell.execute_reply":"2024-10-24T13:41:59.033461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# auto.explain_rows(\n#     train_data=df_train,\n#     model=state.model,\n#     display_rows=True,\n#     plot=\"waterfall\",\n#     rows=state.model_evaluation.undecided[:1],\n# )","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:42:26.843724Z","iopub.execute_input":"2024-10-24T13:42:26.844711Z","iopub.status.idle":"2024-10-24T13:42:26.849498Z","shell.execute_reply.started":"2024-10-24T13:42:26.844669Z","shell.execute_reply":"2024-10-24T13:42:26.848101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Covariate Shift Analysis","metadata":{}},{"cell_type":"code","source":"auto.covariate_shift_detection(train_data=df_train, test_data=df_test, label=target_col)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:42:32.491566Z","iopub.execute_input":"2024-10-24T13:42:32.492032Z","iopub.status.idle":"2024-10-24T13:42:33.940423Z","shell.execute_reply.started":"2024-10-24T13:42:32.491996Z","shell.execute_reply":"2024-10-24T13:42:33.939329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Missing Values","metadata":{}},{"cell_type":"code","source":"auto.missing_values_analysis(train_data=df_train)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:42:42.766327Z","iopub.execute_input":"2024-10-24T13:42:42.767407Z","iopub.status.idle":"2024-10-24T13:42:43.340987Z","shell.execute_reply.started":"2024-10-24T13:42:42.767362Z","shell.execute_reply":"2024-10-24T13:42:43.339959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"auto.missing_values_analysis(train_data=df_test)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:42:48.669535Z","iopub.execute_input":"2024-10-24T13:42:48.670521Z","iopub.status.idle":"2024-10-24T13:42:49.344207Z","shell.execute_reply.started":"2024-10-24T13:42:48.670486Z","shell.execute_reply":"2024-10-24T13:42:49.342974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features Interaction\n\n* [more to know ](https://auto.gluon.ai/0.8.0/tutorials/eda/eda-auto-analyze-interaction.html)","metadata":{}},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:42:54.729529Z","iopub.execute_input":"2024-10-24T13:42:54.729972Z","iopub.status.idle":"2024-10-24T13:42:54.737527Z","shell.execute_reply.started":"2024-10-24T13:42:54.729935Z","shell.execute_reply":"2024-10-24T13:42:54.736419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.dtypes","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:42:58.164831Z","iopub.execute_input":"2024-10-24T13:42:58.165579Z","iopub.status.idle":"2024-10-24T13:42:58.173696Z","shell.execute_reply.started":"2024-10-24T13:42:58.165542Z","shell.execute_reply":"2024-10-24T13:42:58.172663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_columns = df_train.columns.drop(['disease', 'candidateID'])\n\ny_columns = df_train.select_dtypes(include=['number', 'datetime']).columns\ny_columns = y_columns.drop('disease')\n\nfor x in x_columns:\n    for y in y_columns:\n        if x != y:     \n            if df_train[y].nunique() > 1:\n                try:\n                    auto.analyze_interaction(train_data=df_train, x=x, y=y, hue='disease')\n                except ValueError as e:\n                    # Catch an errors\n                    print(f\"Error to analyze {x} and {y} interaction: {e}\")","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:43:58.611728Z","iopub.execute_input":"2024-10-24T13:43:58.612394Z","iopub.status.idle":"2024-10-24T13:45:04.556452Z","shell.execute_reply.started":"2024-10-24T13:43:58.612362Z","shell.execute_reply":"2024-10-24T13:45:04.555271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#state = auto.partial_dependence_plots(df_train.dropna(), target='disease', return_state=True, label=target_col)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:45:51.836572Z","iopub.execute_input":"2024-10-24T13:45:51.836970Z","iopub.status.idle":"2024-10-24T13:45:51.841743Z","shell.execute_reply.started":"2024-10-24T13:45:51.836939Z","shell.execute_reply":"2024-10-24T13:45:51.840629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Anomaly detection","metadata":{}},{"cell_type":"code","source":"train_noid = df_train.drop(columns=['candidateID'])\ntest_noid = df_test.drop(columns=['candidateID'])","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:45:59.321647Z","iopub.execute_input":"2024-10-24T13:45:59.322044Z","iopub.status.idle":"2024-10-24T13:45:59.329484Z","shell.execute_reply.started":"2024-10-24T13:45:59.322015Z","shell.execute_reply":"2024-10-24T13:45:59.328339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This parameter specifies how many standard deviations above mean anomaly score are considered\n# to be anomalies (only needed for visualization, does not affect scores calculation).\nthreshold_stds = 3\n\nauto.detect_anomalies(\n    train_data=train_noid.dropna(),\n    test_data=test_noid.dropna(),\n    label=target_col,\n    threshold_stds=threshold_stds,\n    show_top_n_anomalies=5,\n    fig_args={\n        'figsize': (6, 4)\n    },\n    chart_args={\n        'normal.color': 'lightgrey',\n        'anomaly.color': 'orange',\n    }\n)","metadata":{"execution":{"iopub.status.busy":"2024-10-24T13:46:03.702302Z","iopub.execute_input":"2024-10-24T13:46:03.702680Z","iopub.status.idle":"2024-10-24T13:46:44.255525Z","shell.execute_reply.started":"2024-10-24T13:46:03.702652Z","shell.execute_reply":"2024-10-24T13:46:44.254113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## That's all folks! Thank you 🙏 \n\nCreated by [Samvel Kocharyan](https://www.kaggle.com/samvelkoch), 2024","metadata":{}}]}