{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":115104,"databundleVersionId":13748315,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":13486736,"sourceType":"datasetVersion","datasetId":8562661}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Импорт библиотек","metadata":{}},{"cell_type":"code","source":"!pip3 install rapidfuzz","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:39:55.257895Z","iopub.execute_input":"2025-10-24T10:39:55.258080Z","iopub.status.idle":"2025-10-24T10:40:02.065098Z","shell.execute_reply.started":"2025-10-24T10:39:55.258062Z","shell.execute_reply":"2025-10-24T10:40:02.064181Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nimport re\nimport sys\nimport time\nimport shap\nimport torch\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nfrom pathlib import Path\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nfrom rapidfuzz import fuzz, process\nfrom catboost import CatBoostRanker, Pool\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.model_selection import GroupShuffleSplit\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom sklearn.metrics import accuracy_score, jaccard_score\nfrom sklearn.preprocessing import LabelEncoder, normalize\nfrom sklearn.feature_extraction.text import CountVectorizer","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:40:02.066283Z","iopub.execute_input":"2025-10-24T10:40:02.066574Z","iopub.status.idle":"2025-10-24T10:40:39.967596Z","shell.execute_reply.started":"2025-10-24T10:40:02.066546Z","shell.execute_reply":"2025-10-24T10:40:39.966729Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Загрузка датасетов","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/21-vek-by-searched-products-ranking/train.csv')\ntrain_df.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:40:39.969244Z","iopub.execute_input":"2025-10-24T10:40:39.969861Z","iopub.status.idle":"2025-10-24T10:41:05.380204Z","shell.execute_reply.started":"2025-10-24T10:40:39.969836Z","shell.execute_reply":"2025-10-24T10:41:05.379489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/21-vek-by-searched-products-ranking/test.csv')\ntest_df.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:05.381181Z","iopub.execute_input":"2025-10-24T10:41:05.381530Z","iopub.status.idle":"2025-10-24T10:41:06.269188Z","shell.execute_reply.started":"2025-10-24T10:41:05.381502Z","shell.execute_reply":"2025-10-24T10:41:06.268418Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"product_df = pd.read_csv('/kaggle/input/21-vek-by-searched-products-ranking/products.csv')\nproduct_df.head(3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:06.270055Z","iopub.execute_input":"2025-10-24T10:41:06.270291Z","iopub.status.idle":"2025-10-24T10:41:06.522939Z","shell.execute_reply.started":"2025-10-24T10:41:06.270254Z","shell.execute_reply":"2025-10-24T10:41:06.522238Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Очистка данных","metadata":{}},{"cell_type":"code","source":"def data_validation(df, dataset_name):\n    print(f\"{dataset_name.upper()}\")\n    print(f\"Размеры данных: {df.shape[0]:_} строк × {df.shape[1]} колонок\")\n    print(f\"Колличество пропусков в данных: {df.isnull().sum().sum()}\")\n    print(f\"Колличество дубликатов в данных: {df.duplicated().sum()}\")\n\ndata_validation(train_df, 'train')\ndata_validation(test_df, 'test')\ndata_validation(product_df, 'product')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:06.523732Z","iopub.execute_input":"2025-10-24T10:41:06.524022Z","iopub.status.idle":"2025-10-24T10:41:13.493609Z","shell.execute_reply.started":"2025-10-24T10:41:06.523999Z","shell.execute_reply":"2025-10-24T10:41:13.492889Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"product_df = product_df.drop_duplicates()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:13.494419Z","iopub.execute_input":"2025-10-24T10:41:13.494660Z","iopub.status.idle":"2025-10-24T10:41:13.541226Z","shell.execute_reply.started":"2025-10-24T10:41:13.494640Z","shell.execute_reply":"2025-10-24T10:41:13.540423Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:13.542052Z","iopub.execute_input":"2025-10-24T10:41:13.542304Z","iopub.status.idle":"2025-10-24T10:41:13.558869Z","shell.execute_reply.started":"2025-10-24T10:41:13.542283Z","shell.execute_reply":"2025-10-24T10:41:13.557979Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Анализируем распределение ключевых числовых фич\nnumeric_cols = ['price', 'price_discount', 'browse_time', 'response_time', 'response_products', 'product_position']\n\nfig, axes = plt.subplots(2, 3, figsize=(15, 10))\naxes = axes.ravel()\n\nfor i, col in enumerate(numeric_cols):\n    if col in train_df.columns:\n        axes[i].boxplot(train_df[col].dropna())\n        axes[i].set_title(f\"Boxplot {col}\")\n        axes[i].set_ylabel(col)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:13.561827Z","iopub.execute_input":"2025-10-24T10:41:13.562115Z","iopub.status.idle":"2025-10-24T10:41:18.986647Z","shell.execute_reply.started":"2025-10-24T10:41:13.562089Z","shell.execute_reply":"2025-10-24T10:41:18.985516Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Время ответа (response_time)\n- **4560 микросекунд = 4.56 миллисекунд** - быстрый поиск, тут всё ок\n\n### Время просмотра (browse_time) \n- **11 часов** - это выбросы (слишком не реалистично), поэтому удаляем их\n- Оставляем сессии, время которых **ниже 4.8 часа**","metadata":{}},{"cell_type":"code","source":"browse_upper = train_df['browse_time'].quantile(0.9999)\ntrain_df.drop(train_df[train_df['browse_time'] > browse_upper].index, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:18.987621Z","iopub.execute_input":"2025-10-24T10:41:18.988462Z","iopub.status.idle":"2025-10-24T10:41:19.760192Z","shell.execute_reply.started":"2025-10-24T10:41:18.988429Z","shell.execute_reply":"2025-10-24T10:41:19.759293Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Цена (price)\n- Есть **три товара** с ценой более **20k BYN** (три стиральных порошка), их удаляем\n- Остальные высокие цены выглядят реалистично","metadata":{}},{"cell_type":"code","source":"price_upper = 20_000_00\ntrain_df.drop(train_df[train_df['price'] > price_upper].index, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:19.761149Z","iopub.execute_input":"2025-10-24T10:41:19.761451Z","iopub.status.idle":"2025-10-24T10:41:21.365334Z","shell.execute_reply.started":"2025-10-24T10:41:19.761429Z","shell.execute_reply":"2025-10-24T10:41:21.364626Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Позиция товара (product_position)\n- Слишком длинные выдачи могут исказить результат, т.к. мы прогнозируем только 20 позиций\n- Поэтому удаляем такие сессии, оставляем такие, где **279 и меньше товаров**\n- Оставляем **99%** что бы много не удалить","metadata":{}},{"cell_type":"code","source":"product_position_upper = train_df['product_position'].quantile(0.99)\ntrain_df.drop(train_df[train_df['product_position'] > product_position_upper].index, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:21.366061Z","iopub.execute_input":"2025-10-24T10:41:21.366339Z","iopub.status.idle":"2025-10-24T10:41:22.841412Z","shell.execute_reply.started":"2025-10-24T10:41:21.366314Z","shell.execute_reply":"2025-10-24T10:41:22.840489Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Объединение таблиц","metadata":{}},{"cell_type":"code","source":"train_df = train_df.merge(product_df, on='product_id', how='left')\ntest_df = test_df.merge(product_df, on='product_id', how='left')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:22.842233Z","iopub.execute_input":"2025-10-24T10:41:22.842488Z","iopub.status.idle":"2025-10-24T10:41:25.210233Z","shell.execute_reply.started":"2025-10-24T10:41:22.842469Z","shell.execute_reply":"2025-10-24T10:41:25.209402Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Добавляем фичи","metadata":{}},{"cell_type":"markdown","source":"Добавляем сессионные признаки, такие как средняя цена в сессии, количество товаров в поисковой выдаче, количество уникальных запросов для продукта.\n\nЭти фичи позволяют модели учитывать контекст конкретной поисковой сессии.","metadata":{}},{"cell_type":"code","source":"def add_session_features(df):\n\n    df = df.merge(df.groupby('search_number')\n          .agg(session_price_mean=('price', 'mean'),\n               session_price_std=('price', 'std'),\n               session_size=('price', 'count'))\n          .round(4).reset_index()\n    )\n\n    df = df.merge(df.groupby('product_id')\n          .agg(product_search_count=('search_number', 'nunique'))\n          .reset_index(), on='product_id'\n    )\n    \n    return df\n\ntrain_df = add_session_features(train_df)\ntest_df = add_session_features(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:25.211222Z","iopub.execute_input":"2025-10-24T10:41:25.211489Z","iopub.status.idle":"2025-10-24T10:41:34.837102Z","shell.execute_reply.started":"2025-10-24T10:41:25.211469Z","shell.execute_reply":"2025-10-24T10:41:34.836420Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Вычислим признаки, отражающие поведение пользователя и особенности товаров.","metadata":{}},{"cell_type":"code","source":"def add_general_features(df):\n    avg_session_size = train_df['session_size'].mean()\n    avg_session_time = train_df['browse_time'].mean()\n    \n    # время на товар\n    df['time_per_product'] = df['browse_time'] / df['session_size']\n\n    # процентный ранг позиции товара внутри каждой поисковой сессии (чем меньше, тем товар выше)\n    df['position_rank'] = df.groupby('search_number')['product_position'].rank(pct=True)\n\n    # возраст продукта\n    df['product_age_days'] = (pd.to_datetime('today') - pd.to_datetime(df['date_of_create'])).dt.days.astype(int)\n\n    # насколько товар дороже/дешевле средней цены в сессии\n    df['price_vs_session_mean'] = df['price'] / df['session_price_mean']\n\n    # зависимость позиции от времени\n    df['rank_weighted_engagement'] = df['browse_time'] * df['position_rank']\n    \n    # коэффициент вариации для цен в сессии\n    df['price_cv'] = df['session_price_std'] / df['session_price_mean']\n\n    # продуктивность\n    df['composite_score'] = (df['browse_time'] / avg_session_time) * (df['session_size'] / avg_session_size)\n\n    # популярность товаров\n    df = df.merge(df.groupby('product_id').size().reset_index(name='product_click_count'), on='product_id')\n    \n    # логарифмированная популярность товара (сглаживает выбросы)\n    df['product_popularity_log'] = np.log1p(df['product_click_count'])\n\n    # показатель кликабельности\n    df['product_ctr'] = df['product_click_count'] / df['product_search_count']\n        \n    for col in ['price', 'price_discount', 'browse_time', 'response_time']:\n        df[col] = np.log1p(df[col])\n\n    return df\n\ntrain_df = add_general_features(train_df)\ntest_df = add_general_features(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:34.837841Z","iopub.execute_input":"2025-10-24T10:41:34.838049Z","iopub.status.idle":"2025-10-24T10:41:42.496885Z","shell.execute_reply.started":"2025-10-24T10:41:34.838033Z","shell.execute_reply":"2025-10-24T10:41:42.496079Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Считаем метрику коэффициент Жаккара между запросом и моделью продукта. Пробовал и для других текстовых столбцов, но они себя плохо показали.","metadata":{}},{"cell_type":"code","source":"def add_query_text_features(df):\n    tqdm.pandas()\n    \n    # очистка текста от лишнего\n    def clean_text(text):\n        if pd.isna(text):\n            return ''\n        text = text.lower()\n        text = re.sub(r'[^a-zа-я0-9\\s]', ' ', text)\n        text = re.sub(r'\\s+', ' ', text).strip()\n        return text\n\n    # коэффициент Жаккара (мера схожести множеств слов)\n    def jaccard_similarity(q, n):\n        q_words = set(q.split())\n        n_words = set(n.split())\n        if not q_words or not n_words:\n            return 0\n        return len(q_words & n_words) / len(q_words | n_words)\n        \n    df['query_clean'] = df['query'].progress_apply(clean_text)\n    df['model_clean'] = df['model'].progress_apply(clean_text)\n\n    # длина и количество слов\n    df['query_len'] = df['query_clean'].str.len()\n\n    df['model_token_jaccard'] = df.progress_apply(lambda x: jaccard_similarity(x['query_clean'], x['model_clean']), axis=1)\n\n    return df\n\ntrain_df = add_query_text_features(train_df)\ntest_df = add_query_text_features(test_df)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:41:42.497913Z","iopub.execute_input":"2025-10-24T10:41:42.498201Z","iopub.status.idle":"2025-10-24T10:44:25.708660Z","shell.execute_reply.started":"2025-10-24T10:41:42.498162Z","shell.execute_reply":"2025-10-24T10:44:25.707428Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Генерация эмбеддингов и вычисление семантического сходства, которое должно улавливать смысловую близость текста запроса и описания товара.","metadata":{}},{"cell_type":"code","source":"# проверяем наличие GPU\ndevice = 'cuda' if torch.cuda.is_available() else 'cpu'\nprint(f\"Используется устройство: {device}\")\ncache_dir = Path('/kaggle/input/embeddings-cache/')\ncache_dir.mkdir(exist_ok=True)\n\n# инициализация модели\nmodel_sbert = SentenceTransformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2', device=device)\n\n# Функция для батчевой векторизации\ndef get_embeddings(texts, batch_size=256):\n    vectors = []\n    texts_list = texts.fillna('').tolist()\n    \n    for i in tqdm(range(0, len(texts_list), batch_size), desc='Генерация эмбеддингов'):\n        batch = texts_list[i:i + batch_size]\n        vecs = model_sbert.encode(batch, show_progress_bar=False, normalize_embeddings=True)\n        vectors.append(vecs)\n    \n    return np.vstack(vectors)\n\n# загрузка или вычисление эмбеддингов\ndef load_or_compute_embeddings(df, column_name, dataset_type):\n    file_path = cache_dir / f\"{dataset_type}_{column_name}.npy\"\n    \n    if file_path.exists():\n        print(f\"Загружаем кэш: {file_path.name}\")\n        return np.load(file_path)\n    \n    print(f\"Генерируем эмбеддинги для {column_name}...\")\n    vectors = get_embeddings(df[column_name])\n    np.save(file_path, vectors.astype(np.float16))\n    print(f\"Сохранено: {file_path.name}\")\n    \n    return vectors\n\n# вычисление косинусного сходства\ndef compute_cosine_similarity(a, b, batch_size=100000):\n\n    assert len(a) == len(b), 'Массивы должны быть одинаковой длины!'\n    \n    n = len(a)\n    results = np.empty(n, dtype=np.float32)\n    \n    for i in tqdm(range(0, n, batch_size), desc='Косинусное сходство'):\n        end = min(i + batch_size, n)\n        \n        # конвертируем в тензоры\n        a_t = torch.tensor(a[i:end], dtype=torch.float32, device=device)\n        b_t = torch.tensor(b[i:end], dtype=torch.float32, device=device)\n        \n        # вычисляем сходство\n        with torch.no_grad():\n            batch_sim = torch.nn.functional.cosine_similarity(a_t, b_t)\n            results[i:end] = batch_sim.cpu().numpy()\n        \n        # очистка памяти\n        del a_t, b_t, batch_sim\n        if device == 'cuda':\n            torch.cuda.empty_cache()\n    \n    return results\n\n# обработка датафрейма\ndef process_dataframe(df, dataset_type):\n    print(f\"\\nОбрабатка {dataset_type.upper()}\")\n    \n    query_emb = load_or_compute_embeddings(df, 'query', dataset_type)\n    \n    for column in ['brand', 'model', 'type']:\n        print(f\"Вычисление сходства query-{column}...\")\n        \n        col_emb = load_or_compute_embeddings(df, column, dataset_type)\n        similarity = compute_cosine_similarity(query_emb, col_emb)\n        \n        df[f\"query_{column}_cosine\"] = similarity\n        \n    return df\n\ntrain_df = process_dataframe(train_df, 'train')\ntest_df = process_dataframe(test_df, 'test')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:44:25.710075Z","iopub.execute_input":"2025-10-24T10:44:25.710424Z","iopub.status.idle":"2025-10-24T10:48:17.612987Z","shell.execute_reply.started":"2025-10-24T10:44:25.710367Z","shell.execute_reply":"2025-10-24T10:48:17.611991Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Дополняет эмбеддинговые признаки простыми символьными совпадениями","metadata":{}},{"cell_type":"code","source":"def add_fuzzy_features(df, text_columns=['brand', 'model', 'type'], query_col='query'):\n    tqdm.pandas()\n\n    # схожесть между двумя строками\n    def fuzzy_similarity(a, b):\n        if not isinstance(a, str) or not isinstance(b, str):\n            return 0\n        return fuzz.partial_ratio(a.lower(), b.lower())\n    \n    # фичи для каждой указанной колонки\n    for col in text_columns:\n        feature_name = f\"{col}_fuzzy\"\n        df[feature_name] = df.progress_apply(\n            lambda x: fuzzy_similarity(x[query_col], x[col]), axis=1\n        )\n        # нормализация до [0, 1]\n        df[feature_name] = df[feature_name] / 100\n    \n    return df\n\ntrain_df = add_fuzzy_features(train_df, ['brand', 'model', 'type'])\ntest_df = add_fuzzy_features(test_df, ['brand', 'model', 'type'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:48:17.614090Z","iopub.execute_input":"2025-10-24T10:48:17.614518Z","iopub.status.idle":"2025-10-24T10:52:51.353678Z","shell.execute_reply.started":"2025-10-24T10:48:17.614491Z","shell.execute_reply":"2025-10-24T10:52:51.352657Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Модель","metadata":{}},{"cell_type":"code","source":"features = [\n    'response_products', 'browse_time', 'price', 'price_discount',\n\n    'device', 'selected_category_id', 'filters_applied', 'available',\n\n    'session_price_mean', 'session_size', 'product_search_count', 'time_per_product', 'position_rank', 'product_age_days', 'price_vs_session_mean',\n    'rank_weighted_engagement', 'price_cv', 'product_popularity_log',\n\n    'query_len', 'model_token_jaccard',\n\n    'query_brand_cosine', 'query_model_cosine', 'query_type_cosine', 'product_ctr', 'composite_score' ,'brand_fuzzy', 'model_fuzzy', 'type_fuzzy'\n]\n\ncat_features = ['device', 'selected_category_id', 'filters_applied', 'available']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:52:51.354838Z","iopub.execute_input":"2025-10-24T10:52:51.355197Z","iopub.status.idle":"2025-10-24T10:52:51.361832Z","shell.execute_reply.started":"2025-10-24T10:52:51.355174Z","shell.execute_reply":"2025-10-24T10:52:51.360872Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = train_df.sort_values('search_number').reset_index(drop=True)\ntest_df = test_df.sort_values('search_number').reset_index(drop=True)\n\nsplitter = GroupShuffleSplit(test_size=0.2, n_splits=1, random_state=42)\ntrain_idx, valid_idx = next(splitter.split(train_df, groups=train_df['search_number']))\n\ntrain_data = train_df.iloc[train_idx]\nvalid_data = train_df.iloc[valid_idx]\n\ntrain_pool = Pool(\n    data=train_data[features],\n    label=train_data['relevance'],\n    cat_features=cat_features,\n    group_id=train_data['search_number']\n)\n\nvalid_pool = Pool(\n    data=valid_data[features],\n    label=valid_data['relevance'],\n    cat_features=cat_features,\n    group_id=valid_data['search_number']\n)\n\ntest_pool = Pool(\n    data=test_df[features],\n    cat_features=cat_features,\n    group_id=test_df['search_number']\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:52:51.363488Z","iopub.execute_input":"2025-10-24T10:52:51.363821Z","iopub.status.idle":"2025-10-24T10:53:19.167690Z","shell.execute_reply.started":"2025-10-24T10:52:51.363794Z","shell.execute_reply":"2025-10-24T10:53:19.166709Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model = CatBoostRanker(\n    # Основные параметры\n    iterations=7_000,\n    learning_rate=0.05,\n    depth=8,\n    \n    # Регуляризация\n    l2_leaf_reg=5,\n    \n    # Функция потерь и метрика\n    loss_function='YetiRank',\n    eval_metric='NDCG:top=20',\n    \n    # Бутстрап для GPU\n    bootstrap_type='Bernoulli',\n    \n    # Мониторинг и логи\n    metric_period=50,\n    verbose=100,\n    \n    # Ранняя остановка\n    od_type='Iter',\n    od_wait=100,\n    \n    # Воспроизводимость и ускорение\n    random_seed=42,\n    #task_type='GPU'\n)\n\nmodel.fit(train_pool, eval_set=valid_pool, verbose = 100, use_best_model=True, plot = True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:53:44.301062Z","iopub.execute_input":"2025-10-24T10:53:44.301405Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# важность признаков\nfeature_importance_values = model.get_feature_importance(train_pool)\n\nfeature_importance = pd.DataFrame({\n    'feature': model.feature_names_,\n    'importance': feature_importance_values\n}).sort_values(by='importance', ascending=False)\n\nprint('Топ признаков по важности:')\nprint(feature_importance.head(55))\n\nplt.figure(figsize=(10,6))\nplt.barh(feature_importance['feature'].head(15)[::-1],\n          feature_importance['importance'].head(15)[::-1])\nplt.title('Feature Importance (CatBoostRanker)')\nplt.xlabel('Importance')\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:53:19.389843Z","iopub.status.idle":"2025-10-24T10:53:19.390098Z","shell.execute_reply.started":"2025-10-24T10:53:19.389980Z","shell.execute_reply":"2025-10-24T10:53:19.389991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"top_features = feature_importance.head(20)['feature'].tolist()\n\ncorr_df = train_df[top_features].copy()\n\ncorr_df = corr_df.select_dtypes(include=['int', 'float'])\n\ncorr_matrix = corr_df.corr()\n\nplt.figure(figsize=(12, 10))\nsns.heatmap(\n    corr_matrix,\n    cmap='coolwarm',\n    annot=True,\n    fmt='.2f',\n    linewidths=0.5\n)\nplt.title('Корреляции между топ-20 признаками', fontsize=14)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:53:19.391222Z","iopub.status.idle":"2025-10-24T10:53:19.391565Z","shell.execute_reply.started":"2025-10-24T10:53:19.391366Z","shell.execute_reply":"2025-10-24T10:53:19.391401Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sample_df = train_df.sample(500, random_state=42)  # берем 500 наблюдений\n\nsample_df = sample_df.sort_values('search_number').reset_index(drop=True)\n\nsample_pool = Pool(\n    data=sample_df[features],\n    cat_features=cat_features,\n    label=sample_df['relevance'],\n    group_id=sample_df['search_number']\n)\n\nshap_values = model.get_feature_importance(sample_pool, type='ShapValues')[:, :-1]\n\nshap_df = pd.DataFrame(shap_values, columns=features)\n\nshap.summary_plot(shap_values, sample_df[features])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:53:19.392800Z","iopub.status.idle":"2025-10-24T10:53:19.393118Z","shell.execute_reply.started":"2025-10-24T10:53:19.392956Z","shell.execute_reply":"2025-10-24T10:53:19.392972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preds = model.predict(test_pool)\n\nsubmission = pd.DataFrame({\n    'session_product_id': test_df['session_product_id'].astype(str),\n    'relevance': preds.astype(float)\n})\n\nprint(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:53:19.394568Z","iopub.status.idle":"2025-10-24T10:53:19.394830Z","shell.execute_reply.started":"2025-10-24T10:53:19.394712Z","shell.execute_reply":"2025-10-24T10:53:19.394723Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv('my_test_47.csv', index=False)\nprint('Файл my_test_47.csv сохранён!')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-24T10:53:19.396271Z","iopub.status.idle":"2025-10-24T10:53:19.396590Z","shell.execute_reply.started":"2025-10-24T10:53:19.396449Z","shell.execute_reply":"2025-10-24T10:53:19.396462Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}