{"metadata":{"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceType":"competition","sourceId":115104}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Установка библиотек. Скачивание данных ","metadata":{}},{"cell_type":"code","source":"! pip install numpy pandas catboost kaggle lightgbm scikit-learn ipywidgets 'catboost[widget]'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:15:28.990497Z","iopub.execute_input":"2026-10-20T12:15:28.990912Z","iopub.status.idle":"2026-10-20T12:15:34.303715Z","shell.execute_reply.started":"2026-10-20T12:15:28.990874Z","shell.execute_reply":"2026-10-20T12:15:34.302936Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Загрузка данных в память","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom catboost import CatBoostRanker, Pool","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:15:34.305294Z","iopub.execute_input":"2026-10-20T12:15:34.305671Z","iopub.status.idle":"2026-10-20T12:15:34.309829Z","shell.execute_reply.started":"2026-10-20T12:15:34.305648Z","shell.execute_reply":"2026-10-20T12:15:34.309291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_dir = \"/kaggle/input/competitions/21-vek-by-searched-products-ranking/\"\n# data_dir = \"data\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:15:34.31084Z","iopub.execute_input":"2026-10-20T12:15:34.311206Z","iopub.status.idle":"2026-10-20T12:15:34.327852Z","shell.execute_reply.started":"2026-10-20T12:15:34.311164Z","shell.execute_reply":"2026-10-20T12:15:34.327061Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df = pd.read_csv(data_dir + \"train.csv\")\ntest_df = pd.read_csv(data_dir + \"test.csv\")\nproducts_df = pd.read_csv(data_dir + \"products.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:15:34.32883Z","iopub.execute_input":"2026-10-20T12:15:34.329173Z","iopub.status.idle":"2026-10-20T12:15:55.410736Z","shell.execute_reply.started":"2026-10-20T12:15:34.329152Z","shell.execute_reply":"2026-10-20T12:15:55.410057Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\n    f\"\"\"\n    Размер train:      {len(train_df)}\n    Размер test:       {len(test_df)}\n    Размер products:   {len(products_df)}\n    \"\"\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:15:55.413396Z","iopub.execute_input":"2026-10-20T12:15:55.413738Z","iopub.status.idle":"2026-10-20T12:15:55.418977Z","shell.execute_reply.started":"2026-10-20T12:15:55.413714Z","shell.execute_reply":"2026-10-20T12:15:55.418277Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_df.isna().all().all() | products_df.isna().all().all() | test_df.isna().all().all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:15:55.42004Z","iopub.execute_input":"2026-10-20T12:15:55.42035Z","iopub.status.idle":"2026-10-20T12:15:56.516327Z","shell.execute_reply.started":"2026-10-20T12:15:55.420321Z","shell.execute_reply":"2026-10-20T12:15:56.515294Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Первичная подготовка данных","metadata":{}},{"cell_type":"code","source":"train_source = train_df.merge(products_df.drop_duplicates(), on=\"product_id\", how='left')\ntrain_source = train_source.sort_values(by=[\"date\", \"search_number\"]).reset_index(drop=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:15:56.517623Z","iopub.execute_input":"2026-10-20T12:15:56.51797Z","iopub.status.idle":"2026-10-20T12:16:07.210382Z","shell.execute_reply.started":"2026-10-20T12:15:56.517946Z","shell.execute_reply":"2026-10-20T12:16:07.209378Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"split_ratio = 0.9\nunique_dates = sorted(train_source[\"date\"].unique())\nsplit_date_idx = int(len(unique_dates) * split_ratio)\ncutoff_date = unique_dates[split_date_idx]\n\nprint(f\"Train до даты: {cutoff_date}, Val начиная с даты: {cutoff_date}\")\n\ntrain = train_source[train_source[\"date\"] < cutoff_date].copy()\nval = train_source[train_source[\"date\"] >= cutoff_date].copy()\n\nleak_searches = set(train[\"search_number\"]).intersection(set(val[\"search_number\"]))\nif leak_searches:\n    val = val[~val[\"search_number\"].isin(leak_searches)].copy()\n\nprint(f\"Train rows: {len(train)}, Val rows: {len(val)}\")\nprint(f\"Train searches: {train['search_number'].nunique()}, Val searches: {val['search_number'].nunique()}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:07.211668Z","iopub.execute_input":"2026-10-20T12:16:07.212187Z","iopub.status.idle":"2026-10-20T12:16:12.152632Z","shell.execute_reply.started":"2026-10-20T12:16:07.212163Z","shell.execute_reply":"2026-10-20T12:16:12.151873Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MAX_ITEMS_PER_QUERY = 150\ntrain_gpu = train[train[\"product_position\"] <= MAX_ITEMS_PER_QUERY].copy()\ntrain_gpu = train_gpu.sort_values(by=\"search_number\").reset_index(drop=True)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:12.153691Z","iopub.execute_input":"2026-10-20T12:16:12.154548Z","iopub.status.idle":"2026-10-20T12:16:17.721761Z","shell.execute_reply.started":"2026-10-20T12:16:12.154521Z","shell.execute_reply":"2026-10-20T12:16:17.72108Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nval_sorted = val.sort_values(by=[\"search_number\", \"product_position\"])\n\nval_gpu = val_sorted.groupby(\"search_number\").head(20).copy()\n\n\ncounts = val_gpu.groupby(\"search_number\")[\"product_id\"].transform(\"count\")\nval_gpu = val_gpu[counts == 20].copy()\n\nval_gpu = val_gpu[val_gpu.groupby(\"search_number\")[\"relevance\"].transform(\"max\") > 0]\n\nval_gpu = val_gpu.sort_values(by=\"search_number\").reset_index(drop=True)\n\nprint(f\"Сессий в валидации: {val_gpu['search_number'].nunique()}\")\nprint(f\"Всего строк: {len(val_gpu)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:17.722808Z","iopub.execute_input":"2026-10-20T12:16:17.723076Z","iopub.status.idle":"2026-10-20T12:16:18.222408Z","shell.execute_reply.started":"2026-10-20T12:16:17.723055Z","shell.execute_reply":"2026-10-20T12:16:18.221544Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Генерация и подготовка фичей","metadata":{}},{"cell_type":"code","source":"ignore_cols = [\n    \"user_number\",\n    \"relevance\",\n    \"search_number\",\n    \"relevance\",\n    \"product_id\"\n] + [\n    \"date\", \n    \"date_of_create\",\n    \"replaced_by\",\n    \"query\",\n    \"model\"\n]\n\n\nfeature_cols = [c for c in train_source.columns if c not in ignore_cols]\n\ncat_features = [\n    \"device\",\n    \"type\",\n    \"brand\",\n    \"country\",\n    \"category_name_1\",\n    \"category_name_2\",\n    \"category_name_3\",\n    \"category_name_4\",\n]\n\nlabel_col = \"relevance\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.22336Z","iopub.execute_input":"2026-10-20T12:16:18.223593Z","iopub.status.idle":"2026-10-20T12:16:18.229304Z","shell.execute_reply.started":"2026-10-20T12:16:18.223573Z","shell.execute_reply":"2026-10-20T12:16:18.228349Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Преобразование числовых типов","metadata":{}},{"cell_type":"code","source":"def type_conversion(df):\n    for col in df.select_dtypes(include=[\"int64\"]).columns:\n        df[col] = df[col].astype(\"int32\")\n\n    for col in df.select_dtypes(include=[\"float64\"]).columns:\n        df[col] = df[col].astype(\"float32\")\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.2305Z","iopub.execute_input":"2026-10-20T12:16:18.230843Z","iopub.status.idle":"2026-10-20T12:16:18.241569Z","shell.execute_reply.started":"2026-10-20T12:16:18.230809Z","shell.execute_reply":"2026-10-20T12:16:18.240757Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Временные фичи","metadata":{}},{"cell_type":"code","source":"def add_time_feature(df):\n    df_session = pd.to_datetime(df[\"date\"], yearfirst=True)\n    df_create = pd.to_datetime(df[\"date_of_create\"], yearfirst=True)\n    df[\"session_dayofweek\"] = df_session.dt.dayofweek\n    df[\"session_month\"] = df_session.dt.month\n    df[\"session_dayofmonth\"] = df_session.dt.day\n    df[\"product_age\"] = (df_session - df_create).dt.days\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.24265Z","iopub.execute_input":"2026-10-20T12:16:18.242985Z","iopub.status.idle":"2026-10-20T12:16:18.255569Z","shell.execute_reply.started":"2026-10-20T12:16:18.242954Z","shell.execute_reply":"2026-10-20T12:16:18.254956Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"date_features = [\n    \"session_month\",\n    \"session_dayofweek\",\n    \"product_age\",\n    \"session_dayofmonth\"\n]\n\ndate_cat_features = [\n    \"session_month\",\n    \"session_dayofweek\",\n]","metadata":{"execution":{"iopub.status.busy":"2026-10-20T12:16:18.259408Z","iopub.execute_input":"2026-10-20T12:16:18.259957Z","iopub.status.idle":"2026-10-20T12:16:18.270664Z","shell.execute_reply.started":"2026-10-20T12:16:18.259921Z","shell.execute_reply":"2026-10-20T12:16:18.269954Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Полезные фичи для электронного магазина","metadata":{}},{"cell_type":"code","source":"def add_group_features(df):\n    df[\"discount_pct\"] = (\n        df[\"price_discount\"] / (df[\"price\"] + df[\"price_discount\"] + 1e-5)\n    )\n\n    mean_price = df.groupby(\"search_number\")[\"price\"].transform(\"mean\")\n    df[\"price_ratio\"] = (df[\"price\"] / (mean_price + 1e-5))\n\n    if \"tfidf_similarity\" in df.columns:\n        df[\"tfidf_rank\"] = df.groupby(\"search_number\")[\"tfidf_similarity\"].rank(ascending=False)\n\n    return df\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.271545Z","iopub.execute_input":"2026-10-20T12:16:18.27175Z","iopub.status.idle":"2026-10-20T12:16:18.284703Z","shell.execute_reply.started":"2026-10-20T12:16:18.271731Z","shell.execute_reply":"2026-10-20T12:16:18.284068Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_historical_features_extractor(train_df, smoothing=10):\n    stats = train_df.groupby(\"product_id\")[\"relevance\"].agg([\"sum\", \"count\"])\n    stats[\"product_ctr\"] = (stats[\"sum\"] + smoothing) / (stats[\"count\"] + smoothing)\n    ctr_dict = stats[\"product_ctr\"].to_dict()\n    \n    global_mean = (train_df[\"relevance\"].sum() + smoothing) / (len(train_df) + smoothing)\n\n    def add_historical_features(df):\n        df[\"product_ctr\"] = df[\"product_id\"].map(ctr_dict).fillna(global_mean)\n        return df\n\n    return add_historical_features","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.285659Z","iopub.execute_input":"2026-10-20T12:16:18.286047Z","iopub.status.idle":"2026-10-20T12:16:18.302313Z","shell.execute_reply.started":"2026-10-20T12:16:18.28599Z","shell.execute_reply":"2026-10-20T12:16:18.301517Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"e_commerce_features = [\"tfidf_rank\", \"discount_pct\", \"price_ratio\", \"product_ctr\" ]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.303385Z","iopub.execute_input":"2026-10-20T12:16:18.303695Z","iopub.status.idle":"2026-10-20T12:16:18.316379Z","shell.execute_reply.started":"2026-10-20T12:16:18.303666Z","shell.execute_reply":"2026-10-20T12:16:18.315698Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Текстовые фичи","metadata":{"execution":{"iopub.status.busy":"2026-10-18T15:55:00.718877Z","iopub.execute_input":"2026-10-18T15:55:00.719527Z","iopub.status.idle":"2026-10-18T15:55:00.722672Z","shell.execute_reply.started":"2026-10-18T15:55:00.719501Z","shell.execute_reply":"2026-10-18T15:55:00.721914Z"}}},{"cell_type":"markdown","source":"### Эвристики","metadata":{}},{"cell_type":"code","source":"import re\nimport numpy as np\n\ndef add_lexical_features(df):\n    q_vals = df[\"query\"].astype(str).str.lower().to_list()\n    m_vals = df[\"model\"].astype(str).str.lower().to_list()\n    b_vals = df[\"brand\"].astype(str).str.lower().to_list()\n    c_vals = (\n        df[\"category_name_1\"].astype(str) + \" \" +\n        df[\"category_name_2\"].astype(str) + \" \" +\n        df[\"category_name_3\"].astype(str) + \" \" +\n        df[\"category_name_4\"].astype(str)\n    ).str.lower().to_list()\n    \n    n = len(df)\n    overlap = np.zeros(n, dtype=np.float32)\n    brand_match = np.zeros(n, dtype=np.float32)\n    cat_match = np.zeros(n, dtype=np.float32)\n    dig_match = np.zeros(n, dtype=np.float32)\n    \n    dig_pattern = re.compile(r'\\d+')\n    \n    for i in range(n):\n        q_set = set(q_vals[i].split())\n        m_set = set(m_vals[i].split())\n        b_set = set(b_vals[i].split())\n        c_set = set(c_vals[i].split())\n        \n        q_len = len(q_set)\n        \n        if q_len > 0:\n            overlap[i] = len(q_set & m_set) / q_len\n            \n        if q_set & b_set:\n            brand_match[i] = 1.0\n            \n        if q_set & c_set:\n            cat_match[i] = 1.0\n            \n        q_dig = set(dig_pattern.findall(q_vals[i]))\n        if q_dig:\n            m_dig = set(dig_pattern.findall(m_vals[i]))\n            if q_dig & m_dig:\n                dig_match[i] = 1.0\n                \n    df[\"word_overlap\"] = overlap\n    df[\"brand_match\"] = brand_match\n    df[\"category_match\"] = cat_match\n    df[\"digits_match\"] = dig_match\n    \n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.317378Z","iopub.execute_input":"2026-10-20T12:16:18.318001Z","iopub.status.idle":"2026-10-20T12:16:18.334343Z","shell.execute_reply.started":"2026-10-20T12:16:18.317964Z","shell.execute_reply":"2026-10-20T12:16:18.333595Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### TF-IDF","metadata":{}},{"cell_type":"code","source":"from scipy.sparse import vstack\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\ndef make_tfidf_feature_extractor(products_df):\n    text_corpus = (\n        products_df[\"type\"] + \" \" +\n        products_df[\"model\"] + \" \" +\n        products_df[\"brand\"] + \" \" +\n        products_df[\"country\"] + \" \" +\n        products_df[\"category_name_1\"] + \" \" +\n        products_df[\"category_name_2\"] + \" \" +\n        products_df[\"category_name_3\"] + \" \" +\n        products_df[\"category_name_4\"] + \" \" +\n        \"\"\n    )\n    \n\n\n    tfidf = TfidfVectorizer(\n        max_features = 50_000,\n        sublinear_tf=True,\n        ngram_range=(1, 2),\n        dtype=np.float32\n    )\n\n    tfidf.fit(text_corpus)\n\n    prod_vectors = tfidf.transform(text_corpus)\n    pid_to_row = {pid: i for i, pid in enumerate(products_df[\"product_id\"])}\n\n    def add_tf_idf_feature(df):\n        unique_pairs = df[[\"query\", \"product_id\"]].drop_duplicates().copy()\n\n        unique_queries = unique_pairs[\"query\"].drop_duplicates()\n        q_vectors = tfidf.transform(unique_queries)\n        q_to_row = {q: i for i, q in enumerate(unique_queries)}\n\n        q_indices = [q_to_row[q] for q in unique_pairs[\"query\"]]\n        p_indices = [pid_to_row[pid] for pid in unique_pairs[\"product_id\"]]\n\n        Q_mat = q_vectors[q_indices]\n        P_mat = prod_vectors[p_indices]\n\n        sim_scores = np.asarray(Q_mat.multiply(P_mat).sum(axis=1)).ravel()\n\n        unique_pairs[\"tfidf_similarity\"] = sim_scores.astype(np.float32)\n\n        res_df = df.merge(unique_pairs, on=[\"query\", \"product_id\"], how=\"left\")\n        res_df[\"tfidf_similarity\"] = res_df[\"tfidf_similarity\"].astype(\n            np.float32\n        )\n\n        return res_df\n\n    return add_tf_idf_feature","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:16:18.33536Z","iopub.execute_input":"2026-10-20T12:16:18.335738Z","iopub.status.idle":"2026-10-20T12:16:18.373325Z","shell.execute_reply.started":"2026-10-20T12:16:18.335707Z","shell.execute_reply":"2026-10-20T12:16:18.372474Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### CosSim llm-эмбедингов","metadata":{}},{"cell_type":"code","source":"from sentence_transformers import SentenceTransformer\n\nllm_encoder = SentenceTransformer(\"intfloat/multilingual-e5-small\", device=\"cuda\")\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-10-20T12:16:18.374355Z","iopub.execute_input":"2026-10-20T12:16:18.37552Z","iopub.status.idle":"2026-10-20T12:17:11.3221Z","shell.execute_reply.started":"2026-10-20T12:16:18.375488Z","shell.execute_reply":"2026-10-20T12:17:11.321106Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def make_llm_emb_cossim_feature_extractor(products_df, encoder):\n    \n    products_text = (\n        \"passage: \" +\n        products_df[\"type\"] + \" \" +\n        products_df[\"model\"] + \" \" +\n        products_df[\"brand\"] + \" \" +\n        products_df[\"country\"] + \" \" +\n        products_df[\"category_name_1\"] + \" \" +\n        products_df[\"category_name_2\"] + \" \" +\n        products_df[\"category_name_3\"] + \" \" +\n        products_df[\"category_name_4\"] + \" \" +\n        \"\"\n    )\n    \n    p_vectors = encoder.encode(\n        products_text.to_numpy(),\n        batch_size=256,\n        normalize_embeddings=True,\n    )\n\n    pid_to_row = {pid: i for i, pid in enumerate(products_df[\"product_id\"])}\n\n    def add_llm_emb_cossim(df):\n        unique_pairs = df[[\"query\", \"product_id\"]].drop_duplicates().copy()\n\n        unique_queries = unique_pairs[\"query\"].drop_duplicates()\n        q_vectors = encoder.encode(\n            (\"query: \" + unique_queries).to_numpy(),\n            batch_size=256,\n            normalize_embeddings=True,\n        )\n        q_to_row = {q: i for i, q in enumerate(unique_queries)}\n\n        q_indices = [q_to_row[q] for q in unique_pairs[\"query\"]]\n        p_indices = [pid_to_row[pid] for pid in unique_pairs[\"product_id\"]]\n\n        Q_mat = q_vectors[q_indices]\n        P_mat = p_vectors[p_indices]\n\n        sim_scores = (Q_mat * P_mat).sum(axis=1).ravel()\n\n        unique_pairs[\"llm_cos_sim\"] = sim_scores\n\n        res_df = df.merge(unique_pairs, on=[\"query\", \"product_id\"], how=\"left\")\n\n        return res_df\n\n    return add_llm_emb_cossim","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2026-10-20T12:17:11.323352Z","iopub.execute_input":"2026-10-20T12:17:11.324327Z","iopub.status.idle":"2026-10-20T12:17:11.332744Z","shell.execute_reply.started":"2026-10-20T12:17:11.3243Z","shell.execute_reply":"2026-10-20T12:17:11.331775Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"text_features = [\n    \"tfidf_similarity\",\n    \"llm_cos_sim\",\n    'word_overlap',\n    'brand_match',\n    'category_match',\n    'digits_match'\n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:17:11.333858Z","iopub.execute_input":"2026-10-20T12:17:11.334281Z","iopub.status.idle":"2026-10-20T12:17:11.356666Z","shell.execute_reply.started":"2026-10-20T12:17:11.334251Z","shell.execute_reply":"2026-10-20T12:17:11.355925Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Пайплайн","metadata":{}},{"cell_type":"code","source":"add_tfidf = make_tfidf_feature_extractor(products_df)\nadd_historical_features = make_historical_features_extractor(train_df)\nadd_llm_emb_cossim = make_llm_emb_cossim_feature_extractor(products_df, llm_encoder)\n\ntransforms = [\n    add_lexical_features,\n    add_time_feature,\n    add_llm_emb_cossim,\n    add_tfidf,\n    add_group_features,\n    add_historical_features,\n    type_conversion,\n    \n]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:17:11.357683Z","iopub.execute_input":"2026-10-20T12:17:11.358332Z","iopub.status.idle":"2026-10-20T12:17:49.395971Z","shell.execute_reply.started":"2026-10-20T12:17:11.358298Z","shell.execute_reply":"2026-10-20T12:17:49.395021Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def feature_engineering_pipeline(df, transforms):\n    new_df = df.copy()\n    for transform in transforms:\n        new_df = transform(new_df)\n    return new_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:17:49.39716Z","iopub.execute_input":"2026-10-20T12:17:49.397457Z","iopub.status.idle":"2026-10-20T12:17:49.402359Z","shell.execute_reply.started":"2026-10-20T12:17:49.397434Z","shell.execute_reply":"2026-10-20T12:17:49.401517Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_cols = list(set(feature_cols + date_features + text_features + e_commerce_features))\ncat_features = list(set(cat_features + date_cat_features))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:17:49.403251Z","iopub.execute_input":"2026-10-20T12:17:49.403861Z","iopub.status.idle":"2026-10-20T12:17:49.417861Z","shell.execute_reply.started":"2026-10-20T12:17:49.403835Z","shell.execute_reply":"2026-10-20T12:17:49.417212Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"advanced_train_gpu = feature_engineering_pipeline(train_gpu, transforms)\nadvanced_val_gpu = feature_engineering_pipeline(val_gpu, transforms)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:17:49.418931Z","iopub.execute_input":"2026-10-20T12:17:49.419399Z","iopub.status.idle":"2026-10-20T12:19:27.465902Z","shell.execute_reply.started":"2026-10-20T12:17:49.419374Z","shell.execute_reply":"2026-10-20T12:19:27.465058Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Обучение","metadata":{}},{"cell_type":"code","source":"train_pool = Pool(\n    data=advanced_train_gpu[feature_cols],\n    label=advanced_train_gpu[label_col],\n    group_id=advanced_train_gpu[\"search_number\"],\n    cat_features=cat_features,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:19:27.467077Z","iopub.execute_input":"2026-10-20T12:19:27.467348Z","iopub.status.idle":"2026-10-20T12:19:42.740454Z","shell.execute_reply.started":"2026-10-20T12:19:27.467324Z","shell.execute_reply":"2026-10-20T12:19:42.739761Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_pool = Pool(\n    data=advanced_val_gpu[feature_cols],\n    label=advanced_val_gpu[label_col],\n    group_id=advanced_val_gpu[\"search_number\"],\n    cat_features=cat_features,\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:19:42.741413Z","iopub.execute_input":"2026-10-20T12:19:42.741799Z","iopub.status.idle":"2026-10-20T12:19:43.028985Z","shell.execute_reply.started":"2026-10-20T12:19:42.741752Z","shell.execute_reply":"2026-10-20T12:19:43.027926Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"params = {\n    \"iterations\": 1500,\n    \"learning_rate\": 0.020,\n    \"depth\": 7,\n    \"loss_function\": \"YetiRankPairwise\",\n    \"eval_metric\": \"NDCG:top=20;type=Exp\",\n    \"l2_leaf_reg\": 5,\n    \"task_type\": \"GPU\",\n    \"metric_period\": 20,\n    \"early_stopping_rounds\": 100,\n    \"random_seed\": 126,\n    \"verbose\": 40,\n}","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:19:43.03019Z","iopub.execute_input":"2026-10-20T12:19:43.030666Z","iopub.status.idle":"2026-10-20T12:19:43.035492Z","shell.execute_reply.started":"2026-10-20T12:19:43.030624Z","shell.execute_reply":"2026-10-20T12:19:43.034521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"catboost_model = CatBoostRanker(**params)\ncatboost_model.fit(train_pool, eval_set=val_pool, )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:19:43.036596Z","iopub.execute_input":"2026-10-20T12:19:43.037411Z","iopub.status.idle":"2026-10-20T12:23:18.458138Z","shell.execute_reply.started":"2026-10-20T12:19:43.037384Z","shell.execute_reply":"2026-10-20T12:23:18.457284Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Обучение LGBMRanker","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\nfrom lightgbm import LGBMRanker, log_evaluation, early_stopping\n\nlgb_model = LGBMRanker(\n    objective=\"lambdarank\",\n    metric=\"ndcg\",\n    eval_at=[20],\n    label_gain=[0, 1, 3, 7],\n    n_estimators=2000,\n    learning_rate=0.015,\n    num_leaves=31,\n    min_child_samples=150,\n    colsample_bytree=0.7,\n    subsample=0.8,\n    subsample_freq=1,\n    cat_smooth=50.0,\n    cat_l2=20.0,\n    random_state=42,\n    n_jobs=-1\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:23:18.459264Z","iopub.execute_input":"2026-10-20T12:23:18.459552Z","iopub.status.idle":"2026-10-20T12:23:18.467133Z","shell.execute_reply.started":"2026-10-20T12:23:18.459525Z","shell.execute_reply":"2026-10-20T12:23:18.46637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_lgb = advanced_train_gpu[feature_cols].copy()\nval_lgb = advanced_val_gpu[feature_cols].copy()\n\nfor col in cat_features:\n    train_lgb[col] = train_lgb[col].astype(\"category\")\n    val_lgb[col] = val_lgb[col].astype(\"category\")\n\ntrain_groups = advanced_train_gpu.groupby(\"search_number\", sort=False)[\"product_id\"].count().to_numpy()\nval_groups = advanced_val_gpu.groupby(\"search_number\", sort=False)[\"product_id\"].count().to_numpy()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:23:18.468131Z","iopub.execute_input":"2026-10-20T12:23:18.468422Z","iopub.status.idle":"2026-10-20T12:23:25.034972Z","shell.execute_reply.started":"2026-10-20T12:23:18.468393Z","shell.execute_reply":"2026-10-20T12:23:25.034271Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"lgb_model.fit(\n    X=train_lgb,\n    y=advanced_train_gpu[label_col],\n    group=train_groups,\n    eval_set=[(val_lgb, advanced_val_gpu[label_col])],\n    eval_group=[val_groups],\n    callbacks=[\n        early_stopping(stopping_rounds=100),\n        log_evaluation(period=50)\n    ]\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:23:25.036162Z","iopub.execute_input":"2026-10-20T12:23:25.036438Z","iopub.status.idle":"2026-10-20T12:36:47.365766Z","shell.execute_reply.started":"2026-10-20T12:23:25.036414Z","shell.execute_reply":"2026-10-20T12:36:47.364722Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"test_merged = test_df.merge(products_df.drop_duplicates(), on=\"product_id\", how=\"left\")\n\nadvanced_test_gpu = feature_engineering_pipeline(test_merged, transforms)\n\ntest_pool = Pool(\n    data=advanced_test_gpu[feature_cols],\n    cat_features=cat_features\n)\n\ntest_lgb = advanced_test_gpu[feature_cols].copy()\n\nfor col in cat_features:\n    test_lgb[col] = test_lgb[col].astype(\"category\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:36:47.367183Z","iopub.execute_input":"2026-10-20T12:36:47.367579Z","iopub.status.idle":"2026-10-20T12:36:52.438406Z","shell.execute_reply.started":"2026-10-20T12:36:47.367552Z","shell.execute_reply":"2026-10-20T12:36:52.43765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def min_max_normalize(arr):\n    min_val = arr.min()\n    max_val = arr.max()\n    return (arr - min_val) / (max_val - min_val)\n\np_catboost = min_max_normalize(catboost_model.predict(test_pool))\np_lgb = min_max_normalize(lgb_model.predict(test_lgb))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:37:07.052182Z","iopub.execute_input":"2026-10-20T12:37:07.052664Z","iopub.status.idle":"2026-10-20T12:37:21.6198Z","shell.execute_reply.started":"2026-10-20T12:37:07.052633Z","shell.execute_reply":"2026-10-20T12:37:21.619114Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom scipy.optimize import minimize_scalar\n\ncb_val_preds = catboost_model.predict(val_pool)\nlgb_val_preds = lgb_model.predict(val_lgb)\n\ndef to_session_ranks(df, scores):\n    temp = pd.DataFrame({\n        \"search_number\": df[\"search_number\"].to_numpy(),\n        \"score\": scores\n    })\n    return temp.groupby(\"search_number\")[\"score\"].rank(\n        ascending=False, method=\"average\"\n    ).to_numpy(dtype=np.float32)\n\ncb_val_ranks = to_session_ranks(advanced_val_gpu, cb_val_preds)\nlgb_val_ranks = to_session_ranks(advanced_val_gpu, lgb_val_preds)\n\nn_sessions = len(advanced_val_gpu) // 20\ny_true_mat = advanced_val_gpu[\"relevance\"].to_numpy().reshape(n_sessions, 20)\ncb_ranks_mat = cb_val_ranks.reshape(n_sessions, 20)\nlgb_ranks_mat = lgb_val_ranks.reshape(n_sessions, 20)\n\ndiscounts = 1.0 / np.log2(np.arange(2, 22))\ngains = (2.0 ** y_true_mat) - 1.0\nideal_idx = np.argsort(-y_true_mat, axis=1)\nidcg = np.take_along_axis(gains, ideal_idx, axis=1).dot(discounts)\n\nvalid_mask = idcg > 0\nidcg_valid = idcg[valid_mask]\ngains_valid = gains[valid_mask]\n\ndef eval_blend(w):\n    blended = -(w * cb_ranks_mat[valid_mask] + (1.0 - w) * lgb_ranks_mat[valid_mask])\n    order = np.argsort(-blended, axis=1)\n    dcg = np.take_along_axis(gains_valid, order, axis=1).dot(discounts)\n    return -np.mean(dcg / idcg_valid)\n\nres = minimize_scalar(eval_blend, bounds=(0.0, 1.0), method=\"bounded\")\nbest_w_cb = float(res.x)\nbest_w_lgb = 1.0 - best_w_cb\n\nprint(f\"Вес CatBoost: {best_w_cb:.4f}\")\nprint(f\"Вес LightGBM: {best_w_lgb:.4f}\")\nprint(f\"NDCG@20 на валидации после ансамбля: {-res.fun:.5f}\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:37:21.62108Z","iopub.execute_input":"2026-10-20T12:37:21.621363Z","iopub.status.idle":"2026-10-20T12:37:27.576348Z","shell.execute_reply.started":"2026-10-20T12:37:21.621336Z","shell.execute_reply":"2026-10-20T12:37:27.575318Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\nensemble_preds = best_w_cb * p_catboost +  best_w_lgb * p_lgb\nprint(len(ensemble_preds))\nprint(len(test_merged))\nsubmission = pd.DataFrame({\n    \"session_product_id\": test_df[\"session_product_id\"],\n    \"relevance\": ensemble_preds\n})\n\nassert len(submission) == len(test_df), (\n    f\"Ошибка: строк в сабмите ({len(submission)}) != строк в test ({len(test_df)})\"\n)\nassert submission[\"session_product_id\"].is_unique, (\n    \"Ошибка: обнаружены дубликаты session_product_id!\"\n)\nassert not np.isnan(submission[\"relevance\"]).any(), (\n    \"Ошибка: колонка relevance содержит NaN!\"\n)\nassert not np.isinf(submission[\"relevance\"]).any(), (\n    \"Ошибка: колонка relevance содержит Inf!\"\n)\n\nsubmission.to_csv(\"submission.csv\", index=False)\n\nprint(\" Проверки пройдены успешно!\")\nprint(f\"Файл submission.csv сохранен. Количество строк: {len(submission)}\")\nprint(submission.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T12:37:27.577571Z","iopub.execute_input":"2026-10-20T12:37:27.577939Z","iopub.status.idle":"2026-10-20T12:37:28.405099Z","shell.execute_reply.started":"2026-10-20T12:37:27.5779Z","shell.execute_reply":"2026-10-20T12:37:28.404176Z"}},"outputs":[],"execution_count":null}]}