{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":87331,"databundleVersionId":9932741,"sourceType":"competition"},{"sourceId":203852739,"sourceType":"kernelVersion"}],"dockerImageVersionId":30786,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport os\nimport json\nfrom scipy.stats import skew, kurtosis","metadata":{"execution":{"iopub.status.busy":"2024-10-29T10:59:06.672021Z","iopub.execute_input":"2024-10-29T10:59:06.672399Z","iopub.status.idle":"2024-10-29T10:59:09.169903Z","shell.execute_reply.started":"2024-10-29T10:59:06.672364Z","shell.execute_reply":"2024-10-29T10:59:09.168922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/airs-ai-in-respiratory-sounds/train.csv')\ntest = pd.read_csv('/kaggle/input/airs-ai-in-respiratory-sounds/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:00:41.063502Z","iopub.execute_input":"2024-10-29T11:00:41.064384Z","iopub.status.idle":"2024-10-29T11:00:41.095837Z","shell.execute_reply.started":"2024-10-29T11:00:41.064341Z","shell.execute_reply":"2024-10-29T11:00:41.094953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Stat Prep\n\nWe'll create dataframes with embeddings and embeddings stat","metadata":{}},{"cell_type":"code","source":"def preprocess(df):\n    \"\"\"\n    Transforms data types in the DataFrame according to specified conditions\n    and adds new derived features.\n    \n    Parameters:\n    df (pd.DataFrame): Input DataFrame containing patient data.\n    \n    Returns:\n    pd.DataFrame: DataFrame with updated data types and new features.\n    \"\"\"\n    \n    # Преобразование типов данных\n    df['candidateID'] = df['candidateID'].astype(str)  \n    df['age'] = df['age'].astype(int)  \n    df['gender'] = df['gender'].astype(bool)  \n    df['tbContactHistory'] = df['tbContactHistory'].astype(bool)  \n    df['wheezingHistory'] = df['wheezingHistory'].astype(bool)  \n    df['phlegmCough'] = df['phlegmCough'].astype(bool)  \n    df['familyAsthmaHistory'] = df['familyAsthmaHistory'].astype(bool)  \n    df['feverHistory'] = df['feverHistory'].astype(bool)  \n    df['coldPresent'] = df['coldPresent'].astype(bool)  \n    df['packYears'] = df['packYears'].astype(int)  \n    if 'disease' in df.columns:\n        df['disease'] = df['disease'].astype(int)  # Convert disease to integer (target variable)\n    \n    return df\n\n\ntrain = preprocess(train)\ntest = preprocess(test)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:00:53.191605Z","iopub.execute_input":"2024-10-29T11:00:53.192620Z","iopub.status.idle":"2024-10-29T11:00:53.213333Z","shell.execute_reply.started":"2024-10-29T11:00:53.192573Z","shell.execute_reply":"2024-10-29T11:00:53.212224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"root_dir = \"/kaggle/input/airs-ai-in-respiratory-sounds\"\nsounds_dir = os.path.join(root_dir, \"sounds\", \"sounds\")","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:01:07.180398Z","iopub.execute_input":"2024-10-29T11:01:07.180790Z","iopub.status.idle":"2024-10-29T11:01:07.185598Z","shell.execute_reply.started":"2024-10-29T11:01:07.180753Z","shell.execute_reply":"2024-10-29T11:01:07.184436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vowel_embs = list()\n\nfor dir_name in os.listdir(sounds_dir):\n        \n    if os.path.exists(os.path.join(sounds_dir, dir_name, \"emb_vowel.json\")):\n        with open(os.path.join(sounds_dir, dir_name, \"emb_vowel.json\")) as f:\n            emb_vowel = json.load(f)\n            vowel_embs.append([dir_name] + emb_vowel[0])\n    else:\n        vowel_embs.append([dir_name] + [np.NaN] * 512)\n\n# Задаем имена колонок\ncol_names = ['candidateID'] + [f'emb_vowel_{i}' for i in range(512)]\nvowel_embs = pd.DataFrame(vowel_embs, columns=col_names)\n\nprint(vowel_embs.shape)\nvowel_embs.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:01:08.273125Z","iopub.execute_input":"2024-10-29T11:01:08.273990Z","iopub.status.idle":"2024-10-29T11:01:19.061676Z","shell.execute_reply.started":"2024-10-29T11:01:08.273947Z","shell.execute_reply":"2024-10-29T11:01:19.060689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cough_embs = list()\n\nfor dir_name in os.listdir(sounds_dir):\n        \n    if os.path.exists(os.path.join(sounds_dir, dir_name, \"emb_cough.json\")):\n        with open(os.path.join(sounds_dir, dir_name, \"emb_cough.json\")) as f:\n            emb_cough = json.load(f)\n            # Берем первые две строки эмбеддинга (или меньше, если их меньше двух)\n            first_row = emb_cough[0] if len(emb_cough) > 0 else [np.NaN] * 512\n            #second_row = emb_cough[1] if len(emb_cough) > 1 else [np.NaN] * 512\n            cough_embs.append([dir_name] + first_row)\n    else:\n        # Если файла нет, заполняем пропусками\n        cough_embs.append([dir_name] + [np.NaN] * 512)  # 512 для первой строки и 512 для второй строки\n\n# Создаем DataFrame, добавляя соответствующие названия колонок\ncol_names = ['candidateID'] + [f'emb_cough_{i}_bout_0' for i in range(512)]\ncough_embs = pd.DataFrame(cough_embs, columns=col_names)\n\ncough_embs.head()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:01:19.063286Z","iopub.execute_input":"2024-10-29T11:01:19.063621Z","iopub.status.idle":"2024-10-29T11:01:27.172790Z","shell.execute_reply.started":"2024-10-29T11:01:19.063586Z","shell.execute_reply":"2024-10-29T11:01:27.171757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Отделим колонки bout_0 и bout_1\nbout_0_cols = [col for col in cough_embs.columns if \"bout_0\" in col]\nbout_1_cols = [col for col in cough_embs.columns if \"bout_1\" in col]\n\n# Функция для вычисления статистик\ndef calculate_statistics(data, prefix):\n    stats = pd.DataFrame({\n        f'{prefix}_mean': data.mean(axis=1),\n        f'{prefix}_std': data.std(axis=1),\n        f'{prefix}_min': data.min(axis=1),\n        f'{prefix}_max': data.max(axis=1),\n        f'{prefix}_sum': data.sum(axis=1),\n        f'{prefix}_median': data.median(axis=1),\n        f'{prefix}_25th_percentile': data.quantile(0.25, axis=1),\n        f'{prefix}_75th_percentile': data.quantile(0.75, axis=1),\n        f'{prefix}_skew': data.apply(lambda x: skew(x, nan_policy='omit'), axis=1),\n        f'{prefix}_kurtosis': data.apply(lambda x: kurtosis(x, nan_policy='omit'), axis=1),\n        f'{prefix}_zero_count': (data == 0).sum(axis=1),  # Количество нулевых значений\n        f'{prefix}_mode': data.mode(axis=1)[0],  # Мода (если есть несколько, возьмём первую)\n        f'{prefix}_range': data.max(axis=1) - data.min(axis=1),  # Размах\n        f'{prefix}_iqr': data.quantile(0.75, axis=1) - data.quantile(0.25, axis=1)  # Интерквартильный размах\n    })\n    return stats\n\n# Вычислим статистики для каждой группы колонок\nbout_0_stats = calculate_statistics(cough_embs[bout_0_cols], \"bout_0\")\n#bout_1_stats = calculate_statistics(cough_embs[bout_1_cols], \"bout_1\")\n\n# Объединим статистики с колонкой candidateID\ncough_emb_stat = pd.concat([cough_embs[['candidateID']], bout_0_stats], axis=1)\n\ncough_emb_stat","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:01:27.174122Z","iopub.execute_input":"2024-10-29T11:01:27.174447Z","iopub.status.idle":"2024-10-29T11:01:29.269588Z","shell.execute_reply.started":"2024-10-29T11:01:27.174414Z","shell.execute_reply":"2024-10-29T11:01:29.268502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Отделим колонки с эмбеддингами\nvowel_cols = [col for col in vowel_embs.columns if \"emb_vowel\" in col]\n\n# Вычислим статистики для эмбеддингов vowel\nvowel_stats = calculate_statistics(vowel_embs[vowel_cols], \"vowel\")\n\n# Объединим статистики с колонкой candidateID\nvowel_emb_stat = pd.concat([vowel_embs[['candidateID']], vowel_stats], axis=1)\n\nvowel_emb_stat","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:01:29.271678Z","iopub.execute_input":"2024-10-29T11:01:29.272051Z","iopub.status.idle":"2024-10-29T11:01:31.301743Z","shell.execute_reply.started":"2024-10-29T11:01:29.272008Z","shell.execute_reply":"2024-10-29T11:01:31.300629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = train.merge(cough_emb_stat, on='candidateID', how='left')\ntrain_emb = _.merge(vowel_emb_stat, on='candidateID', how='left')\ntrain_emb","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:01:31.302934Z","iopub.execute_input":"2024-10-29T11:01:31.303253Z","iopub.status.idle":"2024-10-29T11:01:31.344369Z","shell.execute_reply.started":"2024-10-29T11:01:31.303220Z","shell.execute_reply":"2024-10-29T11:01:31.343271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_ = test.merge(cough_emb_stat, on='candidateID', how='left')\ntest_emb = _.merge(vowel_emb_stat, on='candidateID', how='left')\ntest_emb","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:01:31.345742Z","iopub.execute_input":"2024-10-29T11:01:31.346083Z","iopub.status.idle":"2024-10-29T11:01:31.380928Z","shell.execute_reply.started":"2024-10-29T11:01:31.346048Z","shell.execute_reply":"2024-10-29T11:01:31.379781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb.to_csv('train_emb.csv', index=False)\ntest_emb.to_csv('test_emb.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:02:01.469425Z","iopub.execute_input":"2024-10-29T11:02:01.469858Z","iopub.status.idle":"2024-10-29T11:02:01.517488Z","shell.execute_reply.started":"2024-10-29T11:02:01.469815Z","shell.execute_reply":"2024-10-29T11:02:01.516389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Embeddings Viz","metadata":{}},{"cell_type":"code","source":"%%capture\n!pip install -U umap-learn","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:02:16.642905Z","iopub.execute_input":"2024-10-29T11:02:16.643271Z","iopub.status.idle":"2024-10-29T11:02:29.838076Z","shell.execute_reply.started":"2024-10-29T11:02:16.643238Z","shell.execute_reply":"2024-10-29T11:02:29.836638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.preprocessing import StandardScaler\nimport umap.umap_ as umap\nimport umap.plot","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:02:54.927962Z","iopub.execute_input":"2024-10-29T11:02:54.928399Z","iopub.status.idle":"2024-10-29T11:03:36.756686Z","shell.execute_reply.started":"2024-10-29T11:02:54.928359Z","shell.execute_reply":"2024-10-29T11:03:36.755519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cough_pre = train.merge(cough_embs, on='candidateID', how='left') \ncough_pre = cough_pre.drop(columns=['age', 'gender', 'tbContactHistory', 'wheezingHistory',\n       'phlegmCough', 'familyAsthmaHistory', 'feverHistory', 'coldPresent',\n       'packYears'])\ncough_pre","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:36.758445Z","iopub.execute_input":"2024-10-29T11:03:36.759038Z","iopub.status.idle":"2024-10-29T11:03:36.802331Z","shell.execute_reply.started":"2024-10-29T11:03:36.758998Z","shell.execute_reply":"2024-10-29T11:03:36.801169Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_nan_dropped = cough_pre.dropna()\ntrain_numeric = train_nan_dropped.select_dtypes(include='number')\ntrain_numeric.info()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:36.803949Z","iopub.execute_input":"2024-10-29T11:03:36.804418Z","iopub.status.idle":"2024-10-29T11:03:36.848390Z","shell.execute_reply.started":"2024-10-29T11:03:36.804369Z","shell.execute_reply":"2024-10-29T11:03:36.847179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Standardize the data\ntrain_umap = StandardScaler().fit_transform(train_numeric.drop(columns=['disease']))\nmapper = umap.UMAP(n_neighbors=5, n_components=2).fit(train_umap)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:36.850593Z","iopub.execute_input":"2024-10-29T11:03:36.850987Z","iopub.status.idle":"2024-10-29T11:03:47.643005Z","shell.execute_reply.started":"2024-10-29T11:03:36.850949Z","shell.execute_reply":"2024-10-29T11:03:47.641999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"umap.plot.points(mapper, labels=train_numeric.disease, background='black', color_key_cmap='Wistia');","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:47.644298Z","iopub.execute_input":"2024-10-29T11:03:47.644632Z","iopub.status.idle":"2024-10-29T11:03:47.906956Z","shell.execute_reply.started":"2024-10-29T11:03:47.644598Z","shell.execute_reply":"2024-10-29T11:03:47.905910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:47.908738Z","iopub.execute_input":"2024-10-29T11:03:47.909196Z","iopub.status.idle":"2024-10-29T11:03:48.322905Z","shell.execute_reply.started":"2024-10-29T11:03:47.909147Z","shell.execute_reply":"2024-10-29T11:03:48.321899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cough_tsne = cough_pre.drop(columns=['candidateID']).dropna()\ncough_tsne['disease'] = cough_tsne['disease'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:48.324129Z","iopub.execute_input":"2024-10-29T11:03:48.324468Z","iopub.status.idle":"2024-10-29T11:03:48.335065Z","shell.execute_reply.started":"2024-10-29T11:03:48.324433Z","shell.execute_reply":"2024-10-29T11:03:48.334105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.manifold import TSNE\nimport plotly.express as px\n\ntsne = TSNE(n_components=2, random_state=0)\nprojections = tsne.fit_transform(cough_tsne)\n\nfig = px.scatter(\n    projections, x=0, y=1,\n    color=cough_tsne.disease, labels={'color': 'disease'}\n)\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:48.336355Z","iopub.execute_input":"2024-10-29T11:03:48.336668Z","iopub.status.idle":"2024-10-29T11:03:54.120808Z","shell.execute_reply.started":"2024-10-29T11:03:48.336637Z","shell.execute_reply":"2024-10-29T11:03:54.119692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Какое-то говно. Эмбеддинги пока визуально бесполезны","metadata":{}},{"cell_type":"markdown","source":"## DBSCAN и Kmeans на статистиках эмбеддингов","metadata":{}},{"cell_type":"code","source":"from sklearn.cluster import KMeans\nfrom sklearn.preprocessing import StandardScaler\nimport pandas as pd\n\n# Убираем ненужные колонки и строки с пропусками\ndf_cluster = train_emb.drop(columns=['candidateID', 'disease']).dropna()\n\n# Масштабируем данные для улучшения качества кластеризации\nscaler = StandardScaler()\ndf_scaled = scaler.fit_transform(df_cluster)\n\n# Применим KMeans для поиска кластеров\nkmeans = KMeans(n_clusters=3, random_state=0)  # Попробуем сначала с 3 кластерами\nclusters = kmeans.fit_predict(df_scaled)\n\n# Добавим результаты кластеризации к данным\ndf_cluster['cluster'] = clusters\n\n# Посмотрим на распределение данных по кластерам\nprint(df_cluster['cluster'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:54.122335Z","iopub.execute_input":"2024-10-29T11:03:54.122856Z","iopub.status.idle":"2024-10-29T11:03:54.988782Z","shell.execute_reply.started":"2024-10-29T11:03:54.122804Z","shell.execute_reply":"2024-10-29T11:03:54.987755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb.disease.value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:54.992372Z","iopub.execute_input":"2024-10-29T11:03:54.992832Z","iopub.status.idle":"2024-10-29T11:03:55.000522Z","shell.execute_reply.started":"2024-10-29T11:03:54.992793Z","shell.execute_reply":"2024-10-29T11:03:54.999463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.decomposition import PCA\n\n# Применим PCA для понижения размерности до 2 компонент\npca = PCA(n_components=2)\ndf_pca = pca.fit_transform(df_scaled)\n\n# Создаем DataFrame для удобства визуализации\ndf_cluster['pca_1'] = df_pca[:, 0]\ndf_cluster['pca_2'] = df_pca[:, 1]\n\n# Визуализируем кластеры\nplt.figure(figsize=(10, 7))\nscatter = plt.scatter(df_cluster['pca_1'], df_cluster['pca_2'], c=df_cluster['cluster'], cmap='viridis', alpha=0.7)\nplt.colorbar(scatter, label='Cluster')\nplt.xlabel('PCA Component 1')\nplt.ylabel('PCA Component 2')\nplt.title('Cluster Visualization using PCA')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:55.001950Z","iopub.execute_input":"2024-10-29T11:03:55.002281Z","iopub.status.idle":"2024-10-29T11:03:55.414654Z","shell.execute_reply.started":"2024-10-29T11:03:55.002247Z","shell.execute_reply":"2024-10-29T11:03:55.413591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.manifold import TSNE\n\n# Применим t-SNE для понижения размерности до 2 компонент\ntsne = TSNE(n_components=2, random_state=0)\ndf_tsne = tsne.fit_transform(df_scaled)\n\n# Создаем DataFrame для визуализации\ndf_cluster['tsne_1'] = df_tsne[:, 0]\ndf_cluster['tsne_2'] = df_tsne[:, 1]\n\n# Визуализируем кластеры\nplt.figure(figsize=(10, 7))\nscatter = plt.scatter(df_cluster['tsne_1'], df_cluster['tsne_2'], c=df_cluster['cluster'], cmap='viridis', alpha=0.7)\nplt.colorbar(scatter, label='Cluster')\nplt.xlabel('t-SNE Component 1')\nplt.ylabel('t-SNE Component 2')\nplt.title('Cluster Visualization using t-SNE')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:55.415803Z","iopub.execute_input":"2024-10-29T11:03:55.416116Z","iopub.status.idle":"2024-10-29T11:03:58.439600Z","shell.execute_reply.started":"2024-10-29T11:03:55.416084Z","shell.execute_reply":"2024-10-29T11:03:58.438475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\n# Создаем DataFrame для удобства визуализации с PCA и истинными метками\ndf_cluster['true_label'] = train_emb.loc[df_cluster.index, 'disease']\n\n# Визуализация с использованием истинных меток для цветовой дифференциации\nplt.figure(figsize=(12, 6))\n\n# Визуализация с PCA\nplt.subplot(1, 2, 1)\nsns.scatterplot(x='pca_1', y='pca_2', hue='true_label', style='cluster', data=df_cluster, palette='tab10', alpha=0.7)\nplt.title('PCA: Clusters and True Labels')\nplt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.xlabel('PCA Component 1')\nplt.ylabel('PCA Component 2')\n\n# Визуализация с t-SNE (если он уже был выполнен)\nplt.subplot(1, 2, 2)\nsns.scatterplot(x='tsne_1', y='tsne_2', hue='true_label', style='cluster', data=df_cluster, palette='tab10', alpha=0.7)\nplt.title('t-SNE: Clusters and True Labels')\nplt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.xlabel('t-SNE Component 1')\nplt.ylabel('t-SNE Component 2')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:58.441011Z","iopub.execute_input":"2024-10-29T11:03:58.441369Z","iopub.status.idle":"2024-10-29T11:03:59.410885Z","shell.execute_reply.started":"2024-10-29T11:03:58.441332Z","shell.execute_reply":"2024-10-29T11:03:59.409574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Создаем DataFrame для удобства визуализации с PCA и истинными метками\ndf_cluster['true_label'] = train_emb.loc[df_cluster.index, 'disease']\n\n# Визуализация с использованием истинных меток для цветовой дифференциации\nfig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10, 12))\n\n# Визуализация с PCA\nsns.scatterplot(\n    x='pca_1', y='pca_2', hue='true_label', style='cluster', data=df_cluster,\n    palette='tab10', alpha=0.7, ax=axes[0]\n)\naxes[0].set_title('PCA: Clusters and True Labels')\naxes[0].set_xlabel('PCA Component 1')\naxes[0].set_ylabel('PCA Component 2')\naxes[0].legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n\n# Визуализация с t-SNE (если t-SNE уже был выполнен)\nsns.scatterplot(\n    x='tsne_1', y='tsne_2', hue='true_label', style='cluster', data=df_cluster,\n    palette='tab10', alpha=0.7, ax=axes[1]\n)\naxes[1].set_title('t-SNE: Clusters and True Labels')\naxes[1].set_xlabel('t-SNE Component 1')\naxes[1].set_ylabel('t-SNE Component 2')\naxes[1].legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:03:59.412339Z","iopub.execute_input":"2024-10-29T11:03:59.412720Z","iopub.status.idle":"2024-10-29T11:04:00.431222Z","shell.execute_reply.started":"2024-10-29T11:03:59.412665Z","shell.execute_reply":"2024-10-29T11:04:00.429992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\n# Настройки для улучшенной видимости легенд\nlegend_settings = dict(\n    title_font_size=16,\n    font_size=14,\n    orientation=\"h\",      # Горизонтальное расположение\n    x=0.5, xanchor=\"center\", # Центровка по горизонтали\n    y=1.15,                # Над графиком\n    bgcolor=\"rgba(255, 255, 255, 0.5)\"  # Полупрозрачный фон для легенды\n)\n\n# Визуализация с PCA\nfig_pca = px.scatter(\n    df_cluster,\n    x='pca_1', y='pca_2',\n    color='true_label',\n    symbol='cluster',\n    title='PCA: Clusters and True Labels',\n    labels={'pca_1': 'PCA Component 1', 'pca_2': 'PCA Component 2'}\n)\nfig_pca.update_layout(legend=legend_settings)\nfig_pca.show()\n\n# Визуализация с t-SNE (если t-SNE уже был выполнен)\nfig_tsne = px.scatter(\n    df_cluster,\n    x='tsne_1', y='tsne_2',\n    color='true_label',\n    symbol='cluster',\n    title='t-SNE: Clusters and True Labels',\n    labels={'tsne_1': 't-SNE Component 1', 'tsne_2': 't-SNE Component 2'}\n)\nfig_tsne.update_layout(legend=legend_settings)\nfig_tsne.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:04:00.432626Z","iopub.execute_input":"2024-10-29T11:04:00.433006Z","iopub.status.idle":"2024-10-29T11:04:00.640986Z","shell.execute_reply.started":"2024-10-29T11:04:00.432968Z","shell.execute_reply":"2024-10-29T11:04:00.639962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Подсчет количества элементов каждого класса в каждом кластере\ncluster_distribution = df_cluster.groupby(['cluster', 'true_label']).size().unstack(fill_value=0)\n\n# Подсчет процентов\ncluster_percentage = cluster_distribution.div(cluster_distribution.sum(axis=1), axis=0) * 100\n\n# Вывод таблицы с процентами\nprint(\"Процент классов в каждом кластере:\")\ncluster_percentage","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:04:00.642218Z","iopub.execute_input":"2024-10-29T11:04:00.642549Z","iopub.status.idle":"2024-10-29T11:04:00.662968Z","shell.execute_reply.started":"2024-10-29T11:04:00.642515Z","shell.execute_reply":"2024-10-29T11:04:00.661754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n# Построение столбчатой диаграммы\ncluster_percentage.plot(kind='bar', stacked=True, figsize=(10, 6), colormap='tab10')\nplt.title(\"Percentage of Each True Label in Clusters\")\nplt.xlabel(\"Cluster\")\nplt.ylabel(\"Percentage of True Labels\")\nplt.legend(title=\"True Label\", bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:04:00.664721Z","iopub.execute_input":"2024-10-29T11:04:00.665135Z","iopub.status.idle":"2024-10-29T11:04:00.955186Z","shell.execute_reply.started":"2024-10-29T11:04:00.665098Z","shell.execute_reply":"2024-10-29T11:04:00.954145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Тепловая карта с процентами\nplt.figure(figsize=(10, 8))\nsns.heatmap(cluster_percentage, annot=True, fmt=\".1f\", cmap=\"YlGnBu\", cbar_kws={'label': 'Percentage'})\nplt.title(\"Heatmap of True Label Percentages per Cluster\")\nplt.xlabel(\"True Label\")\nplt.ylabel(\"Cluster\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:04:56.203549Z","iopub.execute_input":"2024-10-29T11:04:56.204594Z","iopub.status.idle":"2024-10-29T11:04:56.527893Z","shell.execute_reply.started":"2024-10-29T11:04:56.204544Z","shell.execute_reply":"2024-10-29T11:04:56.526791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.express as px\n\n# Подготовка данных для Sunburst диаграммы\ndf_sunburst = cluster_percentage.reset_index().melt(id_vars=\"cluster\", var_name=\"true_label\", value_name=\"percentage\")\n\nfig = px.sunburst(\n    df_sunburst,\n    path=['cluster', 'true_label'],\n    values='percentage',\n    title=\"Sunburst Diagram of True Label Percentages per Cluster\",\n    color='true_label',\n    color_discrete_sequence=px.colors.qualitative.Pastel\n)\nfig.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:05:04.590246Z","iopub.execute_input":"2024-10-29T11:05:04.590677Z","iopub.status.idle":"2024-10-29T11:05:04.721381Z","shell.execute_reply.started":"2024-10-29T11:05:04.590629Z","shell.execute_reply":"2024-10-29T11:05:04.720081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 6))\nfor i, label in enumerate(cluster_percentage.columns):\n    plt.plot(cluster_percentage.index, cluster_percentage[label], 'o-', label=label)\n\nplt.title(\"Dot Plot of True Label Percentages per Cluster\")\nplt.xlabel(\"Cluster\")\nplt.ylabel(\"Percentage\")\nplt.legend(title=\"True Label\", bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:05:08.731235Z","iopub.execute_input":"2024-10-29T11:05:08.731659Z","iopub.status.idle":"2024-10-29T11:05:09.135025Z","shell.execute_reply.started":"2024-10-29T11:05:08.731604Z","shell.execute_reply":"2024-10-29T11:05:09.133800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Features creation by clusters info","metadata":{}},{"cell_type":"markdown","source":"Да, можно создать новые признаки на основе информации о распределении классов в кластерах, которые затем можно использовать для тестовых данных. Например, можно рассчитать вероятности того, что каждый объект принадлежит определенному классу, основываясь на кластере, в который он попадает.\n\nШаги для создания новых признаков\nСформируем вероятности классов для каждого кластера: Используем полученное распределение классов по кластерам в тренировочных данных как вероятностное распределение для новых данных.\nДобавим вероятностные признаки: Для каждого объекта в тестовых данных, зная его кластер, добавим вероятности каждого класса в качестве новых признаков.","metadata":{"execution":{"iopub.status.busy":"2024-10-28T12:19:45.761296Z","iopub.execute_input":"2024-10-28T12:19:45.761914Z","iopub.status.idle":"2024-10-28T12:19:45.768563Z","shell.execute_reply.started":"2024-10-28T12:19:45.761867Z","shell.execute_reply":"2024-10-28T12:19:45.766911Z"}}},{"cell_type":"code","source":"# # Расчет распределения классов по кластерам на тренировочных данных\n# class_probs_per_cluster = cluster_distribution.div(cluster_distribution.sum(axis=1), axis=0)\n# # Добавим вероятности классов как новые признаки в тренировочный датасет\n# for label in class_probs_per_cluster.columns:\n#     df_cluster[f'class_{label}_prob'] = df_cluster['cluster'].map(class_probs_per_cluster[label])\n# # Предположим, test_df содержит тестовые данные с уже определенными кластерами\n# for label in class_probs_per_cluster.columns:\n#     test_df[f'class_{label}_prob'] = test_df['cluster'].map(class_probs_per_cluster[label])\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:05:32.417912Z","iopub.execute_input":"2024-10-29T11:05:32.418331Z","iopub.status.idle":"2024-10-29T11:05:32.423659Z","shell.execute_reply.started":"2024-10-29T11:05:32.418292Z","shell.execute_reply":"2024-10-29T11:05:32.422385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Adversarial Validation. Понять как отилчаются train и test ","metadata":{}},{"cell_type":"code","source":"#thanks to @carlmcbrideellis\n#https://www.kaggle.com/code/carlmcbrideellis/what-is-adversarial-validation\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import roc_curve\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ndef adversarial_validation(dataset_1 = train_emb, dataset_2 = test_emb, plot=False, label = 'Train-Test'):\n\n    adv_train = train_emb.drop(['candidateID', 'disease'], axis = 1)\n    adv_test = test_emb.drop(['candidateID'], axis = 1)\n\n    adv_train['is_test'] = 0\n    adv_test['is_test'] = 1\n\n    adv = pd.concat([adv_train, adv_test], ignore_index = True)\n\n    adv_shuffled = adv.sample(frac = 1)\n\n    adv_X = adv_shuffled.drop('is_test', axis = 1)\n    adv_y = adv_shuffled.is_test\n\n    skf = StratifiedKFold(n_splits = 5, random_state = 42, shuffle = True)\n\n    val_scores = []\n    predictions = np.zeros(len(adv))\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(adv_X, adv_y)):\n    \n        adv_lr = XGBClassifier(random_state = 42)\n        adv_lr.fit(adv_X.iloc[train_idx], adv_y.iloc[train_idx])\n        \n        val_preds = adv_lr.predict_proba(adv_X.iloc[val_idx])[:,1]\n        predictions[val_idx] = val_preds\n        val_score = roc_auc_score(adv_y.iloc[val_idx], val_preds)\n        val_scores.append(val_score)\n    \n    fpr, tpr, _ = roc_curve(adv['is_test'], predictions)\n    \n    if plot: \n        plt.figure(figsize = (6, 6), dpi = 100)\n        sns.lineplot(x=[0, 1], y=[0, 1], linestyle=\"--\", label=\"Indistinguishable Datasets\")\n        sns.lineplot(x=fpr, y=tpr, label=\"Adversarial Validation Classifier\")\n        plt.title(f'{label} Validation = {np.mean(val_scores):.5f}', weight = 'bold', size = 17)\n        plt.xlabel('False Positive Rate')\n        plt.ylabel('True Positive Rate')\n        plt.show()\n    else: \n        print(f'{label} Validation = {np.mean(val_scores):.5f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:05:36.662299Z","iopub.execute_input":"2024-10-29T11:05:36.663628Z","iopub.status.idle":"2024-10-29T11:05:36.900496Z","shell.execute_reply.started":"2024-10-29T11:05:36.663578Z","shell.execute_reply":"2024-10-29T11:05:36.899509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adversarial_validation(dataset_1 = train_emb, dataset_2 = test_emb, label = 'Train-Test', plot=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:08:05.700915Z","iopub.execute_input":"2024-10-29T11:08:05.701342Z","iopub.status.idle":"2024-10-29T11:08:11.916577Z","shell.execute_reply.started":"2024-10-29T11:08:05.701305Z","shell.execute_reply":"2024-10-29T11:08:11.915423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:08:17.682592Z","iopub.execute_input":"2024-10-29T11:08:17.683327Z","iopub.status.idle":"2024-10-29T11:08:17.765778Z","shell.execute_reply.started":"2024-10-29T11:08:17.683282Z","shell.execute_reply":"2024-10-29T11:08:17.764780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_emb.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:08:20.808612Z","iopub.execute_input":"2024-10-29T11:08:20.809410Z","iopub.status.idle":"2024-10-29T11:08:20.885583Z","shell.execute_reply.started":"2024-10-29T11:08:20.809364Z","shell.execute_reply":"2024-10-29T11:08:20.884575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\n\n# Игнорировать все предупреждения\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:08:44.313887Z","iopub.execute_input":"2024-10-29T11:08:44.314922Z","iopub.status.idle":"2024-10-29T11:08:44.319719Z","shell.execute_reply.started":"2024-10-29T11:08:44.314874Z","shell.execute_reply":"2024-10-29T11:08:44.318552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"columns_to_compare = ['age', 'gender', 'tbContactHistory', 'wheezingHistory',\n       'phlegmCough', 'familyAsthmaHistory', 'feverHistory', 'coldPresent',\n       'packYears', 'bout_0_mean', 'bout_0_std', 'bout_0_min',\n       'bout_0_max', 'bout_0_sum', 'bout_0_median', 'bout_0_25th_percentile',\n       'bout_0_75th_percentile', 'bout_0_skew', 'bout_0_kurtosis',\n       'bout_0_zero_count', 'bout_0_mode', 'bout_0_range', 'bout_0_iqr',\n       'vowel_mean', 'vowel_std', 'vowel_min', 'vowel_max', 'vowel_sum',\n       'vowel_median', 'vowel_25th_percentile', 'vowel_75th_percentile',\n       'vowel_skew', 'vowel_kurtosis', 'vowel_zero_count', 'vowel_mode',\n       'vowel_range', 'vowel_iqr']","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:08:51.971038Z","iopub.execute_input":"2024-10-29T11:08:51.971448Z","iopub.status.idle":"2024-10-29T11:08:51.977612Z","shell.execute_reply.started":"2024-10-29T11:08:51.971408Z","shell.execute_reply":"2024-10-29T11:08:51.976508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for column in columns_to_compare:\n    plt.figure(figsize=(8, 4))\n    sns.histplot(train_emb[column], color='blue', label='Train', kde=True, stat='density')\n    sns.histplot(test_emb[column], color='red', label='Test', kde=True, stat='density')\n    plt.legend()\n    plt.title(f'Distribution of {column}')\n    plt.xlabel(column)\n    plt.ylabel('Density')\n    plt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T07:52:27.093876Z","iopub.execute_input":"2024-10-29T07:52:27.094375Z","iopub.status.idle":"2024-10-29T07:52:43.461990Z","shell.execute_reply.started":"2024-10-29T07:52:27.094328Z","shell.execute_reply":"2024-10-29T07:52:43.460801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## DBSCAN и Kmean на сырых эмбеллингах","metadata":{}},{"cell_type":"code","source":"cough_embs_wo_nan = train[['candidateID', 'disease']].merge(cough_embs, on='candidateID', how='left').dropna()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:05.575520Z","iopub.execute_input":"2024-10-29T11:09:05.576408Z","iopub.status.idle":"2024-10-29T11:09:05.596761Z","shell.execute_reply.started":"2024-10-29T11:09:05.576357Z","shell.execute_reply":"2024-10-29T11:09:05.595489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.cluster import KMeans\nfrom sklearn.preprocessing import StandardScaler\nimport pandas as pd\n\n# Убираем ненужные колонки и строки с пропусками\ndf_cluster = cough_embs_wo_nan.drop(columns=['candidateID', 'disease'])\n\n# Масштабируем данные для улучшения качества кластеризации\nscaler = StandardScaler()\ndf_scaled = scaler.fit_transform(df_cluster)\n\n# Применим KMeans для поиска кластеров\nkmeans = KMeans(n_clusters=3, random_state=0)  # Попробуем сначала с 3 кластерами\nclusters = kmeans.fit_predict(df_scaled)\n\n# Добавим результаты кластеризации к данным\ndf_cluster['cluster'] = clusters\n\n# Посмотрим на распределение данных по кластерам\nprint(df_cluster['cluster'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:07.023567Z","iopub.execute_input":"2024-10-29T11:09:07.024684Z","iopub.status.idle":"2024-10-29T11:09:08.218914Z","shell.execute_reply.started":"2024-10-29T11:09:07.024632Z","shell.execute_reply":"2024-10-29T11:09:08.217801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.decomposition import PCA\n\n# Применим PCA для понижения размерности до 2 компонент\npca = PCA(n_components=2)\ndf_pca = pca.fit_transform(df_scaled)\n\n# Создаем DataFrame для удобства визуализации\ndf_cluster['pca_1'] = df_pca[:, 0]\ndf_cluster['pca_2'] = df_pca[:, 1]\n\n# Визуализируем кластеры\nplt.figure(figsize=(10, 7))\nscatter = plt.scatter(df_cluster['pca_1'], df_cluster['pca_2'], c=df_cluster['cluster'], cmap='viridis', alpha=0.7)\nplt.colorbar(scatter, label='Cluster')\nplt.xlabel('PCA Component 1')\nplt.ylabel('PCA Component 2')\nplt.title('Cluster Visualization using PCA')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:10.525281Z","iopub.execute_input":"2024-10-29T11:09:10.525716Z","iopub.status.idle":"2024-10-29T11:09:10.954437Z","shell.execute_reply.started":"2024-10-29T11:09:10.525661Z","shell.execute_reply":"2024-10-29T11:09:10.953355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.manifold import TSNE\n\n# Применим t-SNE для понижения размерности до 2 компонент\ntsne = TSNE(n_components=2, random_state=0)\ndf_tsne = tsne.fit_transform(df_scaled)\n\n# Создаем DataFrame для визуализации\ndf_cluster['tsne_1'] = df_tsne[:, 0]\ndf_cluster['tsne_2'] = df_tsne[:, 1]\n\n# Визуализируем кластеры\nplt.figure(figsize=(10, 7))\nscatter = plt.scatter(df_cluster['tsne_1'], df_cluster['tsne_2'], c=df_cluster['cluster'], cmap='viridis', alpha=0.7)\nplt.colorbar(scatter, label='Cluster')\nplt.xlabel('t-SNE Component 1')\nplt.ylabel('t-SNE Component 2')\nplt.title('Cluster Visualization using t-SNE')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:14.216283Z","iopub.execute_input":"2024-10-29T11:09:14.216693Z","iopub.status.idle":"2024-10-29T11:09:17.087182Z","shell.execute_reply.started":"2024-10-29T11:09:14.216654Z","shell.execute_reply":"2024-10-29T11:09:17.085983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\n# Создаем DataFrame для удобства визуализации с PCA и истинными метками\ndf_cluster['true_label'] = cough_embs_wo_nan.loc[df_cluster.index, 'disease']\n\n# Визуализация с использованием истинных меток для цветовой дифференциации\nplt.figure(figsize=(12, 6))\n\n# Визуализация с PCA\nplt.subplot(1, 2, 1)\nsns.scatterplot(x='pca_1', y='pca_2', hue='true_label', style='cluster', data=df_cluster, palette='tab10', alpha=0.7)\nplt.title('PCA: Clusters and True Labels')\nplt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.xlabel('PCA Component 1')\nplt.ylabel('PCA Component 2')\n\n# Визуализация с t-SNE (если он уже был выполнен)\nplt.subplot(1, 2, 2)\nsns.scatterplot(x='tsne_1', y='tsne_2', hue='true_label', style='cluster', data=df_cluster, palette='tab10', alpha=0.7)\nplt.title('t-SNE: Clusters and True Labels')\nplt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')\nplt.xlabel('t-SNE Component 1')\nplt.ylabel('t-SNE Component 2')\n\nplt.tight_layout()\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:18.696456Z","iopub.execute_input":"2024-10-29T11:09:18.696871Z","iopub.status.idle":"2024-10-29T11:09:19.593395Z","shell.execute_reply.started":"2024-10-29T11:09:18.696833Z","shell.execute_reply":"2024-10-29T11:09:19.591971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### DBSCAN на статистиках эмбедингов без других колонок","metadata":{}},{"cell_type":"code","source":"train_emb.columns","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:22.767853Z","iopub.execute_input":"2024-10-29T11:09:22.768280Z","iopub.status.idle":"2024-10-29T11:09:22.775638Z","shell.execute_reply.started":"2024-10-29T11:09:22.768240Z","shell.execute_reply":"2024-10-29T11:09:22.774557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.cluster import KMeans\nfrom sklearn.preprocessing import StandardScaler\nimport pandas as pd\n\n# Убираем ненужные колонки и строки с пропусками\ndf_cluster = train_emb.drop(columns=['candidateID', 'age', 'gender', 'tbContactHistory', 'wheezingHistory',\n       'phlegmCough', 'familyAsthmaHistory', 'feverHistory', 'coldPresent',\n       'packYears', 'disease',]).dropna()\n\n# Масштабируем данные для улучшения качества кластеризации\nscaler = StandardScaler()\ndf_scaled = scaler.fit_transform(df_cluster)\n\n# Применим KMeans для поиска кластеров\nkmeans = KMeans(n_clusters=3, random_state=0)  # Попробуем сначала с 3 кластерами\nclusters = kmeans.fit_predict(df_scaled)\n\n# Добавим результаты кластеризации к данным\ndf_cluster['cluster'] = clusters\n\n# Посмотрим на распределение данных по кластерам\nprint(df_cluster['cluster'].value_counts())","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:26.014398Z","iopub.execute_input":"2024-10-29T11:09:26.014838Z","iopub.status.idle":"2024-10-29T11:09:26.771083Z","shell.execute_reply.started":"2024-10-29T11:09:26.014791Z","shell.execute_reply":"2024-10-29T11:09:26.770012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.decomposition import PCA\n\n# Применим PCA для понижения размерности до 2 компонент\npca = PCA(n_components=2)\ndf_pca = pca.fit_transform(df_scaled)\n\n# Создаем DataFrame для удобства визуализации\ndf_cluster['pca_1'] = df_pca[:, 0]\ndf_cluster['pca_2'] = df_pca[:, 1]\n\n# Визуализируем кластеры\nplt.figure(figsize=(10, 7))\nscatter = plt.scatter(df_cluster['pca_1'], df_cluster['pca_2'], c=df_cluster['cluster'], cmap='viridis', alpha=0.7)\nplt.colorbar(scatter, label='Cluster')\nplt.xlabel('PCA Component 1')\nplt.ylabel('PCA Component 2')\nplt.title('Cluster Visualization using PCA')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:29.104682Z","iopub.execute_input":"2024-10-29T11:09:29.105123Z","iopub.status.idle":"2024-10-29T11:09:29.511442Z","shell.execute_reply.started":"2024-10-29T11:09:29.105072Z","shell.execute_reply":"2024-10-29T11:09:29.510301Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.manifold import TSNE\n\n# Применим t-SNE для понижения размерности до 2 компонент\ntsne = TSNE(n_components=2, random_state=0)\ndf_tsne = tsne.fit_transform(df_scaled)\n\n# Создаем DataFrame для визуализации\ndf_cluster['tsne_1'] = df_tsne[:, 0]\ndf_cluster['tsne_2'] = df_tsne[:, 1]\n\n# Визуализируем кластеры\nplt.figure(figsize=(10, 7))\nscatter = plt.scatter(df_cluster['tsne_1'], df_cluster['tsne_2'], c=df_cluster['cluster'], cmap='viridis', alpha=0.7)\nplt.colorbar(scatter, label='Cluster')\nplt.xlabel('t-SNE Component 1')\nplt.ylabel('t-SNE Component 2')\nplt.title('Cluster Visualization using t-SNE')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:31.243584Z","iopub.execute_input":"2024-10-29T11:09:31.244067Z","iopub.status.idle":"2024-10-29T11:09:34.758924Z","shell.execute_reply.started":"2024-10-29T11:09:31.244022Z","shell.execute_reply":"2024-10-29T11:09:34.757758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Создаем DataFrame для удобства визуализации с PCA и истинными метками\ndf_cluster['true_label'] = train_emb.loc[df_cluster.index, 'disease']\n\n# Визуализация с использованием истинных меток для цветовой дифференциации\nfig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10, 12))\n\n# Визуализация с PCA\nsns.scatterplot(\n    x='pca_1', y='pca_2', hue='true_label', style='cluster', data=df_cluster,\n    palette='tab10', alpha=0.7, ax=axes[0]\n)\naxes[0].set_title('PCA: Clusters and True Labels')\naxes[0].set_xlabel('PCA Component 1')\naxes[0].set_ylabel('PCA Component 2')\naxes[0].legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n\n# Визуализация с t-SNE (если t-SNE уже был выполнен)\nsns.scatterplot(\n    x='tsne_1', y='tsne_2', hue='true_label', style='cluster', data=df_cluster,\n    palette='tab10', alpha=0.7, ax=axes[1]\n)\naxes[1].set_title('t-SNE: Clusters and True Labels')\naxes[1].set_xlabel('t-SNE Component 1')\naxes[1].set_ylabel('t-SNE Component 2')\naxes[1].legend(bbox_to_anchor=(1.05, 1), loc='upper left')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:09:34.761063Z","iopub.execute_input":"2024-10-29T11:09:34.761509Z","iopub.status.idle":"2024-10-29T11:09:35.756883Z","shell.execute_reply.started":"2024-10-29T11:09:34.761459Z","shell.execute_reply":"2024-10-29T11:09:35.755641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import DBSCAN\nfrom sklearn.preprocessing import StandardScaler\n\n\n# Применим DBSCAN\ndbscan = DBSCAN(eps=0.5, min_samples=5)  # Настройте параметры eps и min_samples по мере необходимости\ndf_cluster['cluster'] = dbscan.fit_predict(df_scaled)\n\n# Применим PCA для понижения размерности до 2 компонент\npca = PCA(n_components=2)\ndf_pca = pca.fit_transform(df_scaled)\n\n# Создаем DataFrame для удобства визуализации\ndf_cluster['pca_1'] = df_pca[:, 0]\ndf_cluster['pca_2'] = df_pca[:, 1]\n\n# Визуализируем кластеры\nplt.figure(figsize=(10, 7))\nscatter = plt.scatter(df_cluster['pca_1'], df_cluster['pca_2'], c=df_cluster['cluster'], cmap='viridis', alpha=0.7)\nplt.colorbar(scatter, label='Cluster')\nplt.xlabel('PCA Component 1')\nplt.ylabel('PCA Component 2')\nplt.title('Cluster Visualization using DBSCAN and PCA')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:11:41.323468Z","iopub.execute_input":"2024-10-29T11:11:41.323962Z","iopub.status.idle":"2024-10-29T11:11:41.761413Z","shell.execute_reply.started":"2024-10-29T11:11:41.323915Z","shell.execute_reply":"2024-10-29T11:11:41.759996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Сравним датасеты train и test\n\nДобавим к статистикам эмбедингов еще эвристики из MSCC Heuristic Extraction","metadata":{}},{"cell_type":"code","source":"train_heuristics = pd.read_csv('/kaggle/input/mscc-heuristics-extraction/train_heuristics.csv')\ntest_heuristics = pd.read_csv('/kaggle/input/mscc-heuristics-extraction/test_heuristics.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:11:57.632037Z","iopub.execute_input":"2024-10-29T11:11:57.632927Z","iopub.status.idle":"2024-10-29T11:11:57.663732Z","shell.execute_reply.started":"2024-10-29T11:11:57.632876Z","shell.execute_reply":"2024-10-29T11:11:57.662737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_emb","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:12:26.830027Z","iopub.execute_input":"2024-10-29T11:12:26.830447Z","iopub.status.idle":"2024-10-29T11:12:26.860096Z","shell.execute_reply.started":"2024-10-29T11:12:26.830408Z","shell.execute_reply":"2024-10-29T11:12:26.858992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = ['candidateID','H_D_mean',\n 'H_average_interval',\n 'H_freq',\n 'H_peak_count',\n 'H_snr',\n 'H_volume_db',\n 'H_whistle_freqs',\n 'mfcc_1',\n 'mfcc_10',\n 'mfcc_11',\n 'mfcc_12',\n 'mfcc_13',\n 'mfcc_2',\n 'mfcc_3',\n 'mfcc_4',\n 'mfcc_5',\n 'mfcc_6',\n 'mfcc_7',\n 'mfcc_8',\n 'mfcc_9']","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:12:48.928324Z","iopub.execute_input":"2024-10-29T11:12:48.929112Z","iopub.status.idle":"2024-10-29T11:12:48.934406Z","shell.execute_reply.started":"2024-10-29T11:12:48.929065Z","shell.execute_reply":"2024-10-29T11:12:48.933247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb_heur = train_emb.merge(train_heuristics[features], on='candidateID', how='left').drop_duplicates(subset=['candidateID'])\ntest_emb_heur = test_emb.merge(test_heuristics[features], on='candidateID', how='left').drop_duplicates(subset=['candidateID'])","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:12:50.798639Z","iopub.execute_input":"2024-10-29T11:12:50.799150Z","iopub.status.idle":"2024-10-29T11:12:50.819119Z","shell.execute_reply.started":"2024-10-29T11:12:50.799102Z","shell.execute_reply":"2024-10-29T11:12:50.818048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_emb_heur","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:13:12.487168Z","iopub.execute_input":"2024-10-29T11:13:12.488159Z","iopub.status.idle":"2024-10-29T11:13:12.516593Z","shell.execute_reply.started":"2024-10-29T11:13:12.488113Z","shell.execute_reply":"2024-10-29T11:13:12.515315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb_heur.to_csv('train_emb_heur.csv', index=False)\ntest_emb_heur.to_csv('test_emb_heur.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:13:17.339795Z","iopub.execute_input":"2024-10-29T11:13:17.340247Z","iopub.status.idle":"2024-10-29T11:13:17.408969Z","shell.execute_reply.started":"2024-10-29T11:13:17.340205Z","shell.execute_reply":"2024-10-29T11:13:17.407941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\nimport sys\n!{sys.executable} -m pip install -U ydata-profiling[notebook]\n!pip install jupyter-contrib-nbextensions","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:13:34.240096Z","iopub.execute_input":"2024-10-29T11:13:34.240520Z","iopub.status.idle":"2024-10-29T11:14:18.465383Z","shell.execute_reply.started":"2024-10-29T11:13:34.240480Z","shell.execute_reply":"2024-10-29T11:14:18.463835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%capture\n!pip install --upgrade numba visions ydata_profiling","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:14:18.467789Z","iopub.execute_input":"2024-10-29T11:14:18.468191Z","iopub.status.idle":"2024-10-29T11:14:31.329081Z","shell.execute_reply.started":"2024-10-29T11:14:18.468150Z","shell.execute_reply":"2024-10-29T11:14:31.327693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from ydata_profiling import ProfileReport, compare","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:14:31.330676Z","iopub.execute_input":"2024-10-29T11:14:31.331079Z","iopub.status.idle":"2024-10-29T11:14:32.985869Z","shell.execute_reply.started":"2024-10-29T11:14:31.331032Z","shell.execute_reply":"2024-10-29T11:14:32.984629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_report = ProfileReport(train_emb_heur, title=\"Train\", minimal=True)\n\ntest_report = ProfileReport(test_emb_heur, title=\"Test\", minimal=True)\n\ncomparison_report = train_report.compare(test_report)\n\n# Obtain merged statistics\nstatistics = comparison_report.get_description()\n\ncomparison_report.to_file(\"comparison_heuremb.html\")","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:14:32.988279Z","iopub.execute_input":"2024-10-29T11:14:32.990012Z","iopub.status.idle":"2024-10-29T11:15:37.575350Z","shell.execute_reply.started":"2024-10-29T11:14:32.989965Z","shell.execute_reply":"2024-10-29T11:15:37.574276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_report = ProfileReport(train_emb_heur, title=\"Train\", minimal=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-28T14:59:59.935114Z","iopub.execute_input":"2024-10-28T14:59:59.935564Z","iopub.status.idle":"2024-10-28T15:00:00.130891Z","shell.execute_reply.started":"2024-10-28T14:59:59.935519Z","shell.execute_reply":"2024-10-28T15:00:00.129219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_report.to_file('train_report.html')","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:15:37.577342Z","iopub.execute_input":"2024-10-29T11:15:37.577683Z","iopub.status.idle":"2024-10-29T11:15:37.582329Z","shell.execute_reply.started":"2024-10-29T11:15:37.577648Z","shell.execute_reply":"2024-10-29T11:15:37.581163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlations","metadata":{}},{"cell_type":"code","source":"# Исключение ненужных столбцов и выбор только числовых данных\nnumeric_df = train_emb_heur.select_dtypes(include=['number']).drop(columns=['candidateID'], errors='ignore')\n\n# Вычисление корреляций\ncorrelation_matrix = numeric_df.corr()\n\n# Настройка размера графика\nplt.figure(figsize=(12, 8))\n\n# Создание тепловой карты для 'disease'\nsns.heatmap(correlation_matrix[['disease']], annot=True, cmap='coolwarm', fmt='.2f', vmin=-1, vmax=1)\n\n# Настройка заголовка\nplt.title('Correlation Heatmap with Target Variable \"disease\"')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:15:37.583608Z","iopub.execute_input":"2024-10-29T11:15:37.584035Z","iopub.status.idle":"2024-10-29T11:15:38.179039Z","shell.execute_reply.started":"2024-10-29T11:15:37.583998Z","shell.execute_reply":"2024-10-29T11:15:38.177958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Quick Modeling","metadata":{}},{"cell_type":"code","source":"# import pandas as pd\n# import optuna\n# from catboost import CatBoostClassifier\n# from sklearn.model_selection import train_test_split\n# from sklearn.metrics import f1_score\n# from sklearn.ensemble import VotingClassifier\n\n\n# # Выделение признаков и целевой переменной\n# X = train_emb_heur.drop(columns=['candidateID', 'disease'])\n# y = train_emb_heur['disease']\n\n# # Разделение на обучающую и тестовую выборки\n# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n\n# # Определение функции для оптимизации с помощью Optuna\n# def objective(trial, class_label):\n#     # Определение диапазонов гиперпараметров\n#     iterations = trial.suggest_int('iterations', 500, 2000)  # Увеличено до 2000\n#     learning_rate = trial.suggest_loguniform('learning_rate', 0.001, 0.3)  # Увеличен диапазон\n#     depth = trial.suggest_int('depth', 3, 10)  # Увеличен диапазон\n\n#     # Создание и обучение модели\n#     model = CatBoostClassifier(iterations=iterations, learning_rate=learning_rate, depth=depth, verbose=0)\n#     model.fit(X_train, (y_train == class_label).astype(int), eval_set=(X_test, (y_test == class_label).astype(int)), early_stopping_rounds=50)\n\n#     # Возвращаем метрику F1 Macro\n#     y_pred = model.predict(X_test)\n#     return f1_score(y_test == class_label, y_pred, average='macro')\n\n# # Оптимизация гиперпараметров для каждой модели класса\n# best_models = {}\n# for class_label in range(3):\n#     study = optuna.create_study(direction='maximize')\n#     study.optimize(lambda trial: objective(trial, class_label), n_trials=100)  # Увеличено до 100 испытаний\n    \n#     # Сохраняем лучшую модель\n#     best_params = study.best_params\n#     best_model = CatBoostClassifier(iterations=best_params['iterations'], \n#                                      learning_rate=best_params['learning_rate'], \n#                                      depth=best_params['depth'], \n#                                      verbose=0)\n#     best_model.fit(X_train, (y_train == class_label).astype(int))\n#     best_models[class_label] = best_model\n\n# # Обучение модели для всех классов с оптимальными гиперпараметрами\n# study_all = optuna.create_study(direction='maximize')\n# study_all.optimize(lambda trial: objective(trial, None), n_trials=100)  # Увеличено до 100 испытаний\n\n# best_params_all = study_all.best_params\n# model_all_classes = CatBoostClassifier(iterations=best_params_all['iterations'], \n#                                         learning_rate=best_params_all['learning_rate'], \n#                                         depth=best_params_all['depth'], \n#                                         verbose=0)\n# model_all_classes.fit(X_train, y_train)\n\n# # Создаем VotingClassifier с лучшими моделями\n# voting_model = VotingClassifier(\n#     estimators=[(f'model_class_{i}', best_models[i]) for i in range(3)] + [('model_all', model_all_classes)],\n#     voting='soft'\n# )\n\n# # Обучаем VotingClassifier\n# voting_model.fit(X_train, y_train)\n\n# # Предсказания и оценка итоговой модели\n# voting_predictions = voting_model.predict(X_test)\n# voting_f1_macro = f1_score(y_test, voting_predictions, average='macro')\n# print(f'Voting Classifier F1 Macro: {voting_f1_macro:.4f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-28T17:09:32.368704Z","iopub.execute_input":"2024-10-28T17:09:32.369303Z","iopub.status.idle":"2024-10-28T17:09:32.382399Z","shell.execute_reply.started":"2024-10-28T17:09:32.369246Z","shell.execute_reply":"2024-10-28T17:09:32.380660Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import pandas as pd\n# from catboost import CatBoostClassifier\n# from sklearn.model_selection import KFold\n# from sklearn.metrics import f1_score\n\n# # Предобработка данных\n# # Пропуски будут обработаны автоматически моделями\n\n# # Выделение признаков и целевой переменной\n# X = train_emb_heur.drop(columns=['candidateID', 'disease'])\n# y = train_emb_heur['disease']\n\n# # Задайте оптимальные параметры, найденные ранее\n# best_params = {\n#     'iterations': 1338,  # пример\n#     'learning_rate': 0.1488664469304496,  # пример\n#     'depth': 4  # пример\n# }\n\n# #'iterations': 1338, 'learning_rate': 0.1488664469304496, 'depth': 4}. Best is trial 46 with value: 0.8931372549019607\n\n# # Создание модели с лучшими параметрами\n# best_model = CatBoostClassifier(iterations=best_params['iterations'], \n#                                  learning_rate=best_params['learning_rate'], \n#                                  depth=best_params['depth'], \n#                                  verbose=0)\n\n# # Функция для оценки модели с помощью кросс-валидации\n# def cross_validate_model(X, y, model, n_splits):\n#     kf = KFold(n_splits=n_splits, shuffle=True, random_state=42)\n#     f1_scores = []\n\n#     for train_index, test_index in kf.split(X):\n#         X_train_fold, X_test_fold = X.iloc[train_index], X.iloc[test_index]\n#         y_train_fold, y_test_fold = y.iloc[train_index], y.iloc[test_index]\n\n#         model.fit(X_train_fold, y_train_fold)\n#         y_pred = model.predict(X_test_fold)\n#         f1_scores.append(f1_score(y_test_fold, y_pred, average='macro'))\n\n#     return sum(f1_scores) / len(f1_scores)\n\n# # Проверка на 5, 7 и 10 фолдах\n# for n_folds in [5]:\n#     f1_macro_score = cross_validate_model(X, y, best_model, n_folds)\n#     print(f'F1 Macro Score with {n_folds} folds: {f1_macro_score:.4f}')\n","metadata":{"execution":{"iopub.status.busy":"2024-10-28T17:07:52.771576Z","iopub.execute_input":"2024-10-28T17:07:52.772184Z","iopub.status.idle":"2024-10-28T17:08:28.632222Z","shell.execute_reply.started":"2024-10-28T17:07:52.772141Z","shell.execute_reply":"2024-10-28T17:08:28.630694Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Adversarial Validation не heuremb","metadata":{}},{"cell_type":"code","source":"#thanks to @carlmcbrideellis\n#https://www.kaggle.com/code/carlmcbrideellis/what-is-adversarial-validation\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import roc_auc_score, roc_curve\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\n\ndef adversarial_validation(dataset_1 = train_emb_heur, dataset_2 = test_emb_heur, plot=False, label = 'Train-Test'):\n\n    adv_train = dataset_1.drop(['candidateID', 'disease'], axis=1, errors='ignore')\n    adv_test = dataset_2.drop(['candidateID'], axis=1, errors='ignore')\n\n    adv_train['is_test'] = 0\n    adv_test['is_test'] = 1\n\n    adv = pd.concat([adv_train, adv_test], ignore_index=True)\n    adv_shuffled = adv.sample(frac=1, random_state=42)\n\n    adv_X = adv_shuffled.drop('is_test', axis=1)\n    adv_y = adv_shuffled['is_test']\n\n    skf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)\n    val_scores = []\n    predictions = np.zeros(len(adv))\n    feature_importances = np.zeros(adv_X.shape[1])\n\n    for fold, (train_idx, val_idx) in enumerate(skf.split(adv_X, adv_y)):\n        adv_lr = XGBClassifier(random_state=42)\n        adv_lr.fit(adv_X.iloc[train_idx], adv_y.iloc[train_idx])\n        \n        # Получение фичаимпортансов и накопление их значений\n        feature_importances += adv_lr.feature_importances_ / skf.n_splits\n\n        val_preds = adv_lr.predict_proba(adv_X.iloc[val_idx])[:, 1]\n        predictions[val_idx] = val_preds\n        val_score = roc_auc_score(adv_y.iloc[val_idx], val_preds)\n        val_scores.append(val_score)\n\n    fpr, tpr, _ = roc_curve(adv_y, predictions)\n\n    # Построение графика ROC-кривой\n    if plot:\n        plt.figure(figsize=(6, 6), dpi=100)\n        sns.lineplot(x=[0, 1], y=[0, 1], linestyle=\"--\", label=\"Indistinguishable Datasets\")\n        sns.lineplot(x=fpr, y=tpr, label=\"Adversarial Validation Classifier\")\n        plt.title(f'{label} Validation = {np.mean(val_scores):.5f}', weight='bold', size=17)\n        plt.xlabel('False Positive Rate')\n        plt.ylabel('True Positive Rate')\n        plt.show()\n    else:\n        print(f'{label} Validation = {np.mean(val_scores):.5f}')\n\n    # Построение графика важности признаков\n    feature_importances_df = pd.DataFrame({\n        'Feature': adv_X.columns,\n        'Importance': feature_importances\n    }).sort_values(by='Importance', ascending=False)\n\n    plt.figure(figsize=(10, 6))\n    sns.barplot(data=feature_importances_df.head(20), x='Importance', y='Feature')\n    plt.title('Top 20 Features by Importance')\n    plt.show()\n\n    return feature_importances_df","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:29:43.260511Z","iopub.execute_input":"2024-10-29T11:29:43.260995Z","iopub.status.idle":"2024-10-29T11:29:43.277505Z","shell.execute_reply.started":"2024-10-29T11:29:43.260952Z","shell.execute_reply":"2024-10-29T11:29:43.276134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adversarial_validation(dataset_1 = train_emb_heur, dataset_2 = test_emb_heur, label = 'Train-Test', plot=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:29:50.128994Z","iopub.execute_input":"2024-10-29T11:29:50.129942Z","iopub.status.idle":"2024-10-29T11:29:56.314505Z","shell.execute_reply.started":"2024-10-29T11:29:50.129889Z","shell.execute_reply":"2024-10-29T11:29:56.313391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Уберем vowel_min, vowel_sum","metadata":{}},{"cell_type":"code","source":"train_emb_heur.vowel_min.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:34:29.496508Z","iopub.execute_input":"2024-10-29T11:34:29.497667Z","iopub.status.idle":"2024-10-29T11:34:29.508849Z","shell.execute_reply.started":"2024-10-29T11:34:29.497622Z","shell.execute_reply":"2024-10-29T11:34:29.507769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_emb_heur.vowel_min.describe()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:34:40.607079Z","iopub.execute_input":"2024-10-29T11:34:40.607521Z","iopub.status.idle":"2024-10-29T11:34:40.620321Z","shell.execute_reply.started":"2024-10-29T11:34:40.607481Z","shell.execute_reply":"2024-10-29T11:34:40.618897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb_heur.columns","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:39:34.534754Z","iopub.execute_input":"2024-10-29T11:39:34.535184Z","iopub.status.idle":"2024-10-29T11:39:34.543246Z","shell.execute_reply.started":"2024-10-29T11:39:34.535147Z","shell.execute_reply":"2024-10-29T11:39:34.541918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove_cols = ['vowel_mean', 'vowel_std', 'vowel_min', 'vowel_max', 'vowel_sum',\n#        'vowel_median', 'vowel_25th_percentile', 'vowel_75th_percentile',\n#        'vowel_skew', 'vowel_kurtosis', 'vowel_zero_count', 'vowel_mode',\n#        'vowel_range', 'vowel_iqr', 'coldPresent', 'tbContactHistory', 'familyAsthmaHistory', \n#                'packYears', 'wheezingHistory', 'mfcc_3', 'H_whistle_freqs', 'mfcc_2', \n#                'H_average_interval', 'mfcc_6', 'bout_0_75th_percentile', 'bout_0_iqr', \n#                'bout_0_sum', 'H_freq','bout_0_kurtosis', 'H_peak_count', 'bout_0_max', \n#                'mfcc_4', 'mfcc_9', 'mfcc_11', 'mfcc_10', 'mfcc_8', 'mfcc_7','bout_0_skew', 'mfcc_1']","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:06:52.976666Z","iopub.execute_input":"2024-10-29T12:06:52.977565Z","iopub.status.idle":"2024-10-29T12:06:52.984772Z","shell.execute_reply.started":"2024-10-29T12:06:52.977506Z","shell.execute_reply":"2024-10-29T12:06:52.983377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_original = pd.read_csv('/kaggle/input/airs-ai-in-respiratory-sounds/train.csv')\ntest_original = pd.read_csv('/kaggle/input/airs-ai-in-respiratory-sounds/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:24:10.204608Z","iopub.execute_input":"2024-10-29T12:24:10.205070Z","iopub.status.idle":"2024-10-29T12:24:10.219828Z","shell.execute_reply.started":"2024-10-29T12:24:10.205028Z","shell.execute_reply":"2024-10-29T12:24:10.218810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remove_cols_original = ['familyAsthmaHistory','wheezingHistory']","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:33:23.103014Z","iopub.execute_input":"2024-10-29T12:33:23.103457Z","iopub.status.idle":"2024-10-29T12:33:23.109077Z","shell.execute_reply.started":"2024-10-29T12:33:23.103415Z","shell.execute_reply":"2024-10-29T12:33:23.107853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adversarial_validation(dataset_1 = train_original.drop(columns=remove_cols_original), \n                       dataset_2 = test_original.drop(columns=remove_cols_original), \n                       label = 'Train-Test', plot=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:33:27.654365Z","iopub.execute_input":"2024-10-29T12:33:27.655221Z","iopub.status.idle":"2024-10-29T12:33:31.962894Z","shell.execute_reply.started":"2024-10-29T12:33:27.655175Z","shell.execute_reply":"2024-10-29T12:33:31.961786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb_heur.drop(columns=remove_cols).columns","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:01:45.497322Z","iopub.execute_input":"2024-10-29T12:01:45.498278Z","iopub.status.idle":"2024-10-29T12:01:45.506064Z","shell.execute_reply.started":"2024-10-29T12:01:45.498232Z","shell.execute_reply":"2024-10-29T12:01:45.504945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adversarial_validation(dataset_1 = train_emb_heur.drop(columns=remove_cols), \n                       dataset_2 = test_emb_heur.drop(columns=remove_cols), \n                       label = 'Train-Test', plot=True)","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:06:55.641836Z","iopub.execute_input":"2024-10-29T12:06:55.642679Z","iopub.status.idle":"2024-10-29T12:07:01.668320Z","shell.execute_reply.started":"2024-10-29T12:06:55.642637Z","shell.execute_reply":"2024-10-29T12:07:01.667214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_emb_heur.coldPresent.value_counts(normalize=True))\ndisplay(test_emb_heur.coldPresent.value_counts(normalize=True))","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:41:25.770192Z","iopub.execute_input":"2024-10-29T11:41:25.770757Z","iopub.status.idle":"2024-10-29T11:41:25.787309Z","shell.execute_reply.started":"2024-10-29T11:41:25.770678Z","shell.execute_reply":"2024-10-29T11:41:25.785990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_emb_heur.tbContactHistory.value_counts(normalize=True))\ndisplay(test_emb_heur.tbContactHistory.value_counts(normalize=True))","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:42:39.008574Z","iopub.execute_input":"2024-10-29T11:42:39.009033Z","iopub.status.idle":"2024-10-29T11:42:39.022351Z","shell.execute_reply.started":"2024-10-29T11:42:39.008992Z","shell.execute_reply":"2024-10-29T11:42:39.021283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_emb_heur.familyAsthmaHistory.value_counts(normalize=True))\ndisplay(test_emb_heur.familyAsthmaHistory.value_counts(normalize=True))","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:43:30.778651Z","iopub.execute_input":"2024-10-29T11:43:30.779455Z","iopub.status.idle":"2024-10-29T11:43:30.792427Z","shell.execute_reply.started":"2024-10-29T11:43:30.779407Z","shell.execute_reply":"2024-10-29T11:43:30.791253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_emb_heur.packYears.value_counts(normalize=True))\ndisplay(test_emb_heur.packYears.value_counts(normalize=True))","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:44:52.414011Z","iopub.execute_input":"2024-10-29T11:44:52.414415Z","iopub.status.idle":"2024-10-29T11:44:52.428351Z","shell.execute_reply.started":"2024-10-29T11:44:52.414379Z","shell.execute_reply":"2024-10-29T11:44:52.427304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_emb_heur.wheezingHistory.value_counts(normalize=True))\ndisplay(test_emb_heur.wheezingHistory.value_counts(normalize=True))","metadata":{"execution":{"iopub.status.busy":"2024-10-29T11:48:57.708442Z","iopub.execute_input":"2024-10-29T11:48:57.708901Z","iopub.status.idle":"2024-10-29T11:48:57.722693Z","shell.execute_reply.started":"2024-10-29T11:48:57.708860Z","shell.execute_reply":"2024-10-29T11:48:57.721599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_emb_heur.candidateID.sort_values().head()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:17:28.651012Z","iopub.execute_input":"2024-10-29T12:17:28.651946Z","iopub.status.idle":"2024-10-29T12:17:28.662169Z","shell.execute_reply.started":"2024-10-29T12:17:28.651892Z","shell.execute_reply":"2024-10-29T12:17:28.660980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_emb_heur.candidateID.sort_values().head()","metadata":{"execution":{"iopub.status.busy":"2024-10-29T12:17:49.621506Z","iopub.execute_input":"2024-10-29T12:17:49.622544Z","iopub.status.idle":"2024-10-29T12:17:49.632866Z","shell.execute_reply.started":"2024-10-29T12:17:49.622489Z","shell.execute_reply":"2024-10-29T12:17:49.631763Z"},"trusted":true},"execution_count":null,"outputs":[]}]}