{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87331,"databundleVersionId":10191418,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Simple ensembling model: propagating error from one input to another","metadata":{}},{"cell_type":"code","source":"## defining the path for parent directory\nDIR = '/kaggle/input/airs-ai-in-respiratory-sounds/'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-17T10:51:42.078293Z","iopub.status.idle":"2024-11-17T10:51:42.078791Z","shell.execute_reply.started":"2024-11-17T10:51:42.078556Z","shell.execute_reply":"2024-11-17T10:51:42.078585Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Import necessary libraries","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\nfrom sklearn.metrics import f1_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.preprocessing import StandardScaler\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.layers import Dense, Concatenate, Input\nfrom tensorflow.keras import Model\nfrom glob import glob\nimport os\nimport json\nfrom catboost import CatBoostClassifier\n\npd.set_option('future.no_silent_downcasting', True)\n\nfor filenames in glob(DIR+'*'):\n    print(filenames)\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:51:42.080786Z","iopub.status.idle":"2024-11-17T10:51:42.081255Z","shell.execute_reply.started":"2024-11-17T10:51:42.081029Z","shell.execute_reply":"2024-11-17T10:51:42.081052Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# defining all paths\ntr_path = '/kaggle/input/airs-ai-in-respiratory-sounds/train.csv'\nts_path = '/kaggle/input/airs-ai-in-respiratory-sounds/test.csv'\nfiles_path = '/kaggle/input/airs-ai-in-respiratory-sounds/sounds/sounds/'","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:51:57.753897Z","iopub.execute_input":"2024-11-17T10:51:57.754378Z","iopub.status.idle":"2024-11-17T10:51:57.763305Z","shell.execute_reply.started":"2024-11-17T10:51:57.754333Z","shell.execute_reply":"2024-11-17T10:51:57.760526Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.impute import SimpleImputer\n# Initialize imputer, using 'mean' as the strategy (can also use 'median' or 'most_frequent')\nimputer = SimpleImputer(strategy='most_frequent')\n\ndef load_table_data(file_path, target_column):\n    \"\"\"Load and prepare table data.\"\"\"\n    table_data = pd.read_csv(file_path)\n    target = table_data[target_column]\n    table_data = table_data.drop(columns=['candidateID'])\n    X = table_data.drop(columns=[target_column])\n    X = imputer.fit_transform(X)\n    return X, target\n\ndef load_embeddings(files_path, disease_map, emb_type='cough'):\n    \"\"\"Load embeddings, ensuring only candidates with embeddings are used.\"\"\"\n    X = []\n    y = []\n    valid_ids = {}\n\n    for _id in glob(files_path + '*'):\n        # emb_path = os.path.join(_id, f'emb_{emb_type}.json')\n        emb_path = os.path.join(_id, f'{emb_type}-opera.npy')\n        cand_id = os.path.basename(_id)\n        if os.path.exists(emb_path) and disease_map.get(cand_id) is not None:\n            # embs = json.load(open(emb_path, 'r'))\n            embs = np.load(emb_path)\n            for emb in embs[:1]:\n                valid_ids[cand_id] = len(y)\n                X.append(emb)\n                y.append(disease_map[cand_id])\n    print(f\"Loaded {emb_type.capitalize()} Embeddings:\", len(X))\n    return np.array(X), np.array(y), valid_ids","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:51:59.688463Z","iopub.execute_input":"2024-11-17T10:51:59.688919Z","iopub.status.idle":"2024-11-17T10:51:59.707226Z","shell.execute_reply.started":"2024-11-17T10:51:59.688873Z","shell.execute_reply":"2024-11-17T10:51:59.705895Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tdf = pd.read_csv(tr_path)\n\n# 1. Load Table Data\nX_table, y = load_table_data(tr_path, 'disease')\n\ndisease_map = {d['candidateID']: d['disease'] for d in tdf.to_dict('records')}\nx_id = {d['candidateID']: idx for idx, d in enumerate(tdf.to_dict('records'))}\n\n# 5. Load Vowel and Cough Embeddings (If Available)\nX_train_vowel, y_train_vowel, valid_vowel_ids = load_embeddings(files_path, disease_map, emb_type='vowel')\nX_train_cough, y_train_cough, valid_cough_ids = load_embeddings(files_path, disease_map, emb_type='cough')\n","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:52:02.757506Z","iopub.execute_input":"2024-11-17T10:52:02.757982Z","iopub.status.idle":"2024-11-17T10:52:11.059374Z","shell.execute_reply.started":"2024-11-17T10:52:02.757935Z","shell.execute_reply":"2024-11-17T10:52:11.057961Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def initial_prediction_with_table(X_train, y_train, X_val, y_val):\n    \"\"\"Make initial predictions using table data.\"\"\"\n    clf = LogisticRegression(max_iter=200)\n    clf.fit(X_train, y_train)\n    y_pred = clf.predict(X_val)\n    # Calculate F1 score for each category\n    f1_scores_per_class = f1_score(y_val, y_pred, average=None)\n    print(\"F1 Score for each category:\", f1_scores_per_class)\n    \n    score = f1_score(y_val, y_pred, average='macro')\n    print(\"Initial F1 Score with Table Data:\", score)\n    return clf","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:52:11.061786Z","iopub.execute_input":"2024-11-17T10:52:11.062291Z","iopub.status.idle":"2024-11-17T10:52:11.070294Z","shell.execute_reply.started":"2024-11-17T10:52:11.062237Z","shell.execute_reply":"2024-11-17T10:52:11.068925Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Get initial results from csv","metadata":{}},{"cell_type":"code","source":"# 3. Split Table Data\nX_train_table, X_val_table, y_train, y_val = train_test_split(X_table, y, test_size=0.2, random_state=42)\n\n# 4. Initial Prediction with Table Data\nt_clf = initial_prediction_with_table(X_train_table, y_train, X_val_table, y_val)\ntable_pred = t_clf.predict(X_table)\n","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:53:37.937440Z","iopub.execute_input":"2024-11-17T10:53:37.937928Z","iopub.status.idle":"2024-11-17T10:53:38.022618Z","shell.execute_reply.started":"2024-11-17T10:53:37.937876Z","shell.execute_reply":"2024-11-17T10:53:38.021454Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def combine_with_predictions(X, predictions):\n    \"\"\"Combine features with predictions from previous models.\"\"\"\n    return np.hstack((X, predictions.reshape(-1, 1)))\n\ndef boosting_with_embeddings(X_train_boosted, y_train, X_val_boosted, y_val):\n    \"\"\"Boosting with embeddings by combining previous model predictions.\"\"\"\n    clf = CatBoostClassifier(\n                iterations=100,\n                depth=7,\n                loss_function='MultiClassOneVsAll',\n                random_seed=42,\n                verbose=False,\n                class_weights={0:1.15, 1:0.75, 2:0.65}\n            )\n    # clf = GradientBoostingClassifier(n_estimators=100, random_state=42)\n    \n    clf.fit(X_train_boosted, y_train)\n    y_pred = clf.predict(X_val_boosted)\n    y_orig_pred = clf.predict(X_train_boosted)\n    # Calculate F1 score for each category\n    f1_scores_per_class = f1_score(y_val, y_pred, average=None)\n    print(\"F1 Score for each category:\", f1_scores_per_class)\n    \n    score = f1_score(y_val, y_pred, average='macro')\n    print(\"Boosted F1 Score with Table + Embeddings:\", score)\n    return clf","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:53:40.850564Z","iopub.execute_input":"2024-11-17T10:53:40.851596Z","iopub.status.idle":"2024-11-17T10:53:40.861334Z","shell.execute_reply.started":"2024-11-17T10:53:40.851534Z","shell.execute_reply":"2024-11-17T10:53:40.859810Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Improve results using csv output and vowel embeddings","metadata":{}},{"cell_type":"code","source":"X_combined, y_combined, valid_indices = [], [], []\n\nfor id_, idx in x_id.items():\n    if id_ in valid_vowel_ids:\n        X_combined.append(np.hstack([table_pred[idx], X_train_vowel[valid_vowel_ids[id_]]]))\n        y_combined.append(y[idx])\n        valid_indices.append(idx)\n\n\nX_combined = np.array(X_combined)\ny_combined = np.array(y_combined)\nv_idx_id = {idx: i for i, idx in enumerate(valid_indices)}\n\nX_train_v, X_val_v, y_train_v, y_val_v = train_test_split(X_combined, y_combined, test_size=0.2, random_state=42)\n\nv_clf = boosting_with_embeddings(X_train_v, y_train_v, X_val_v, y_val_v)\n\nvowel_pred = v_clf.predict(X_combined)\n\n","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:53:42.353444Z","iopub.execute_input":"2024-11-17T10:53:42.353969Z","iopub.status.idle":"2024-11-17T10:54:13.253491Z","shell.execute_reply.started":"2024-11-17T10:53:42.353922Z","shell.execute_reply":"2024-11-17T10:54:13.252216Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Getting result from combined output of vowel, csv and cough embeddings","metadata":{}},{"cell_type":"code","source":"# Here we have only picked one cough bout sample and used single cough embedding per sample \nX_combined, y_combined = [], []\n\nfor id_, idx in x_id.items():\n    if id_ in valid_cough_ids and idx in valid_indices:\n        X_combined.append(np.hstack([vowel_pred[v_idx_id[idx]], X_train_cough[valid_cough_ids[id_]]]))\n        y_combined.append(y[idx])\n\nX_combined = np.array(X_combined)\ny_combined = np.array(y_combined)\n\nX_train_v, X_val_v, y_train_v, y_val_v = train_test_split(X_combined, y_combined, test_size=0.2, random_state=42)\n\nf_clf = boosting_with_embeddings(X_train_v, y_train_v, X_val_v, y_val_v)\n\n\n","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:54:13.255751Z","iopub.execute_input":"2024-11-17T10:54:13.256151Z","iopub.status.idle":"2024-11-17T10:54:43.941989Z","shell.execute_reply.started":"2024-11-17T10:54:13.256110Z","shell.execute_reply":"2024-11-17T10:54:43.940448Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Trying the other sequence\nX_combined, y_combined, valid_indices = [], [], []\n\nfor id_, idx in x_id.items():\n    if id_ in valid_cough_ids:\n        X_combined.append(np.hstack([table_pred[idx], X_train_cough[valid_cough_ids[id_]]]))\n        y_combined.append(y[idx])\n        valid_indices.append(idx)\n\n\nX_combined = np.array(X_combined)\ny_combined = np.array(y_combined)\nc_idx_id = {idx: i for i, idx in enumerate(valid_indices)}\n\nX_train_c, X_val_c, y_train_c, y_val_c = train_test_split(X_combined, y_combined, test_size=0.2, random_state=42)\n\nc_clf = boosting_with_embeddings(X_train_c, y_train_c, X_val_c, y_val_c)\n\ncough_pred = c_clf.predict(X_combined)","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:54:43.943522Z","iopub.execute_input":"2024-11-17T10:54:43.943928Z","iopub.status.idle":"2024-11-17T10:55:14.596874Z","shell.execute_reply.started":"2024-11-17T10:54:43.943881Z","shell.execute_reply":"2024-11-17T10:55:14.595348Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# This combination yields lower results than the other sequence hence using the sequence csv -> vowel -> cough\nX_combined, y_combined = [], []\n\nfor id_, idx in x_id.items():\n    if id_ in valid_vowel_ids and idx in valid_indices:\n        X_combined.append(np.hstack([cough_pred[c_idx_id[idx]], X_train_vowel[valid_vowel_ids[id_]]]))\n        y_combined.append(y[idx])\n\nX_combined = np.array(X_combined)\ny_combined = np.array(y_combined)\n\nX_train_v, X_val_v, y_train_v, y_val_v = train_test_split(X_combined, y_combined, test_size=0.2, random_state=42)\n\nfc_clf = boosting_with_embeddings(X_train_v, y_train_v, X_val_v, y_val_v)","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:55:45.141367Z","iopub.execute_input":"2024-11-17T10:55:45.141898Z","iopub.status.idle":"2024-11-17T10:56:15.311022Z","shell.execute_reply.started":"2024-11-17T10:55:45.141843Z","shell.execute_reply":"2024-11-17T10:56:15.309544Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Predicting output given the output of embedding in different cases\n\ndef final_prediction(df, id_):\n    r = df[df['candidateID']==id_]\n    r = r.drop(columns = ['candidateID'])\n    if 'disease' in r.columns:\n        r = r.drop(columns = ['disease'])\n    r = imputer.transform(r)\n    # c_path = os.path.join(files_path, id_, f'emb_cough.json')\n    c_path = os.path.join(files_path, id_, f'cough-opera.npy')\n    # v_path = os.path.join(files_path, id_, f'emb_vowel.json')\n    v_path = os.path.join(files_path, id_, f'vowel-opera.npy')\n    c_ex = os.path.exists(c_path)\n    v_ex = os.path.exists(v_path)\n    _p = t_clf.predict(r)\n    \n    if v_ex:\n        # v = np.array(json.load(open(v_path, 'r')))[0]\n        v = np.load(v_path)[0]\n        n = np.hstack([_p, v]).reshape((-1, 1)).T\n        _p = v_clf.predict(n).flatten()\n        if c_ex:\n            c = np.load(c_path, 'r')[0]\n            n = np.hstack([_p, c]).reshape((-1, 1)).T\n            _p = f_clf.predict(n)\n            \n    elif c_ex:\n        c = np.load(c_path, 'r')[0]\n        n = np.hstack([_p, c]).reshape((-1, 1)).T\n        _p = c_clf.predict(n)\n        \n    return _p.item()\n        ","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:55:45.081409Z","iopub.execute_input":"2024-11-17T10:55:45.081777Z","iopub.status.idle":"2024-11-17T10:55:45.095282Z","shell.execute_reply.started":"2024-11-17T10:55:45.081738Z","shell.execute_reply":"2024-11-17T10:55:45.094064Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tdf","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:55:45.097176Z","iopub.execute_input":"2024-11-17T10:55:45.097826Z","iopub.status.idle":"2024-11-17T10:55:45.138599Z","shell.execute_reply.started":"2024-11-17T10:55:45.097773Z","shell.execute_reply":"2024-11-17T10:55:45.137245Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"final_prediction(tdf, '90ebadb136321')","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:57:18.376671Z","iopub.execute_input":"2024-11-17T10:57:18.377161Z","iopub.status.idle":"2024-11-17T10:57:18.565232Z","shell.execute_reply.started":"2024-11-17T10:57:18.377114Z","shell.execute_reply":"2024-11-17T10:57:18.563201Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tsdf = pd.read_csv(ts_path)\ntsdf","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:57:33.995320Z","iopub.execute_input":"2024-11-17T10:57:33.995783Z","iopub.status.idle":"2024-11-17T10:57:34.023764Z","shell.execute_reply.started":"2024-11-17T10:57:33.995737Z","shell.execute_reply":"2024-11-17T10:57:34.022469Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Getting output prediction","metadata":{}},{"cell_type":"code","source":"tsdf['disease'] = tsdf['candidateID'].apply(lambda x: final_prediction(tsdf, x))\ntsdf","metadata":{"execution":{"iopub.status.busy":"2024-11-17T10:57:37.848847Z","iopub.execute_input":"2024-11-17T10:57:37.849292Z","iopub.status.idle":"2024-11-17T10:57:39.091129Z","shell.execute_reply.started":"2024-11-17T10:57:37.849251Z","shell.execute_reply":"2024-11-17T10:57:39.089451Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tsdf[['candidateID', 'disease']].to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-26T02:39:14.732068Z","iopub.execute_input":"2024-10-26T02:39:14.732393Z","iopub.status.idle":"2024-10-26T02:39:14.742707Z","shell.execute_reply.started":"2024-10-26T02:39:14.732359Z","shell.execute_reply":"2024-10-26T02:39:14.741591Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Feel free to try different models and try out many other approaches 🫡","metadata":{}},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}