{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":115104}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Обновление структуры ноутбука:**\n1. Клетки имеют классификацию по выполняемому функционалу. *(Клетки с маркером [EXP] можно закомментировать перед сабмитом для ускорения получения результата)*.\n2. [INIT] - клетки инициализации, содержат определения переменных и фичей, необходимых для дальнейшей работы.\n3. [DEF] - \tтолько функции и константы, без вычислений.\n4. [EXP] - эксперименты и исследования, неоюходимы на этапе разработки, не несут ценности для сабмитов.\n5. [TRAIN] - обучение моделей (одна  клетка - одна модель). Ключевые клетки для сабмитов.\n6. [INIT-HISTORY] - для интеграции исторических метрик товаров.\n7. [INIT-QUERY] - для обработки метрик связанных с запросом и текстовыми характеристиками товаров ","metadata":{}},{"cell_type":"code","source":"# [INIT]\nimport os, gc, warnings\nimport numpy as np, pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nwarnings.filterwarnings('ignore')\npd.set_option('display.max_columns', 100); pd.set_option('display.width', 200)\n\nSEED = 42\nREGIME_START = pd.Timestamp('2025-03-10')\nVALID_START  = pd.Timestamp('2025-06-02')\nCUTOFF_DAYS, K_PROD = 14, 10\nOUT_DIR = '/kaggle/working'\nDATA_DIR = '/kaggle/input/competitions/21-vek-by-searched-products-ranking'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:18:57.694414Z","iopub.execute_input":"2026-10-19T07:18:57.694845Z","iopub.status.idle":"2026-10-19T07:19:03.596547Z","shell.execute_reply.started":"2026-10-19T07:18:57.69481Z","shell.execute_reply":"2026-10-19T07:19:03.595498Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT]\ntrain = pd.read_csv(f'{DATA_DIR}/train.csv')\ntest = pd.read_csv(f'{DATA_DIR}/test.csv')\nproducts = pd.read_csv(f'{DATA_DIR}/products.csv')\nsample_sub = pd.read_csv(f'{DATA_DIR}/sample_submission.csv')\n\nfor d in (train, test, products):\n    for c in d.select_dtypes('int64').columns: d[c] = pd.to_numeric(d[c], downcast='integer')\n    for c in d.select_dtypes('float64').columns: d[c] = pd.to_numeric(d[c], downcast='float')\nfor d in (train, test):\n    d['date'] = pd.to_datetime(d['date'])\n    d['sid'] = d.groupby(['user_number', 'search_number'], sort=False).ngroup()\n\nassert (test['session_product_id'].values == sample_sub['session_product_id'].values).all()\nassert (test.groupby('sid').size() == 20).all()\nassert test.groupby('sid')['product_position'].apply(lambda x: x.is_monotonic_increasing).all()\nprint(train.shape, test.shape, products.shape, '| сессий в test:', test['sid'].nunique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:03.5985Z","iopub.execute_input":"2026-10-19T07:19:03.599035Z","iopub.status.idle":"2026-10-19T07:19:29.705379Z","shell.execute_reply.started":"2026-10-19T07:19:03.599004Z","shell.execute_reply":"2026-10-19T07:19:29.704125Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 1. Структура данных","metadata":{}},{"cell_type":"code","source":"# [EXP] Обзор таблиц\ndef column_overview(df):\n    return pd.DataFrame({'dtype': df.dtypes, 'nulls': df.isna().sum(),\n                         'null_%': (df.isna().mean() * 100).round(2), 'nunique': df.nunique()})\n\nfor name, df in [('train', train), ('test', test), ('products', products)]:\n    print(f'===== {name} {df.shape}, {df.memory_usage(deep=True).sum() / 2**20:.0f} MB')\n    display(column_overview(df))\nprint('Только в train:', sorted(set(train.columns) - set(test.columns)))\nprint('Только в test: ', sorted(set(test.columns) - set(train.columns)))\ndisplay(train.head(3)); display(test.head(3)); display(products.head(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:29.706559Z","iopub.execute_input":"2026-10-19T07:19:29.707214Z","iopub.status.idle":"2026-10-19T07:19:36.294199Z","shell.execute_reply.started":"2026-10-19T07:19:29.707113Z","shell.execute_reply":"2026-10-19T07:19:36.293239Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [EXP] Ключ сессии и формат сабмишена\nr_parsed = test['session_product_id'].str.split('_', expand=True).astype(int)\nr_key_num, r_key_pid = r_parsed[0], r_parsed[1]\nprint('product_id из ключа == product_id:', (r_key_pid == test['product_id']).all())\nprint('номеров сессии в ключе:', r_key_num.nunique(), '| сессий по (user, search):', test['sid'].nunique())\nprint('одно число из ключа на сессию:', (test.assign(k=r_key_num).groupby('sid')['k'].nunique() == 1).all())\nprint('размеры сессий test:', test.groupby('sid').size().value_counts().to_dict())\nfor keys in [['search_number'], ['user_number', 'search_number']]:\n    print(keys, '-> train:', train.groupby(keys).ngroups, '| test:', test.groupby(keys).ngroups)\nprint('дубли (sid, product_id): train', train.duplicated(['sid', 'product_id']).sum(),\n      '| test', test.duplicated(['sid', 'product_id']).sum())\nprint('дубли (search_number, product_id): train', train.duplicated(['search_number', 'product_id']).sum())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:36.296463Z","iopub.execute_input":"2026-10-19T07:19:36.296794Z","iopub.status.idle":"2026-10-19T07:19:40.65352Z","shell.execute_reply.started":"2026-10-19T07:19:36.296766Z","shell.execute_reply":"2026-10-19T07:19:40.652516Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [EXP] Коллизии search_number и дубли (сессия, товар)\nr_g = train.groupby('search_number').agg(n_users=('user_number', 'nunique'), n_dates=('date', 'nunique'),\n                                         n_queries=('query', 'nunique'), n_rows=('product_id', 'size'))\nprint('search_number с >1 пользователя:', (r_g.n_users > 1).sum())\nprint('search_number с >1 датой:      ', (r_g.n_dates > 1).sum())\nprint('search_number с >1 запросом:   ', (r_g.n_queries > 1).sum())\ndisplay(r_g[r_g.n_users > 1].head(10))\n\nr_dup = train[train.duplicated(['user_number', 'search_number', 'product_id'], keep=False)]\nprint('дубликатов по (user, search, product):', len(r_dup))\ndisplay(r_dup.sort_values(['search_number', 'product_id']).head(6)[\n    ['user_number', 'search_number', 'product_id', 'date', 'query', 'product_position', 'relevance']])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:40.655024Z","iopub.execute_input":"2026-10-19T07:19:40.655389Z","iopub.status.idle":"2026-10-19T07:19:44.071112Z","shell.execute_reply.started":"2026-10-19T07:19:40.655355Z","shell.execute_reply":"2026-10-19T07:19:44.070094Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [EXP] Размеры сессий в train\nr_sizes = train.groupby('sid').size()\nprint(r_sizes.describe())\nprint('Сессий ровно с 20 товарами:', (r_sizes == 20).sum(), f'({(r_sizes == 20).mean():.1%})')\nprint('Строк в таких сессиях:', r_sizes[r_sizes == 20].sum(), f'({r_sizes[r_sizes == 20].sum() / len(train):.1%} от train)')\nfig, ax = plt.subplots(figsize=(12, 3.5))\nr_sizes.clip(upper=60).value_counts().sort_index().plot(kind='bar', ax=ax)\nax.set_title('Размер сессии в train (60 = «60 и больше»)'); plt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:44.072261Z","iopub.execute_input":"2026-10-19T07:19:44.072624Z","iopub.status.idle":"2026-10-19T07:19:44.781979Z","shell.execute_reply.started":"2026-10-19T07:19:44.072599Z","shell.execute_reply":"2026-10-19T07:19:44.780966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [EXP] Таргет\nr_sizes = train.groupby('sid').size()\nr_f20 = train[train['sid'].map(r_sizes) == 20]\nprint('relevance, весь train:'); print(train['relevance'].value_counts(normalize=True).sort_index().round(4))\nprint('relevance, сессии размера 20:'); print(r_f20['relevance'].value_counts(normalize=True).sort_index().round(4))\nr_mx = r_f20.groupby('sid')['relevance'].max()\nprint('максимум relevance в сессии (размер 20):'); print(r_mx.value_counts(normalize=True).sort_index().round(4))\nprint('сессий без положительных меток:', f'{(r_mx == 0).mean():.1%}')\nr_pc = (r_f20['relevance'] > 0).groupby(r_f20['sid']).sum()\nprint('товаров с relevance>0 на сессию:'); print(r_pc.describe())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:44.783195Z","iopub.execute_input":"2026-10-19T07:19:44.783489Z","iopub.status.idle":"2026-10-19T07:19:45.143983Z","shell.execute_reply.started":"2026-10-19T07:19:44.783464Z","shell.execute_reply":"2026-10-19T07:19:45.143091Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 2. Метрика и бейзлайн","metadata":{}},{"cell_type":"code","source":"# [DEF]\ndef sess_ndcg(sid, y, s, k=20):\n    \"\"\"NDCG@20 по каждой сессии (Series по sid). В сессиях без положительных меток значение NaN.\"\"\"\n    d = pd.DataFrame({'sid': np.asarray(sid), 'y': np.asarray(y), 's': np.asarray(s)})\n    d['g'] = 2.0 ** d['y'] - 1\n    def dcg(sd):\n        r = sd.groupby('sid', sort=False).cumcount().values\n        disc = np.where(r < k, 1.0 / np.log2(r + 2), 0.0)\n        return pd.Series(sd['g'].values * disc, index=sd['sid'].values).groupby(level=0).sum()\n    p = dcg(d.sort_values(['sid', 's'], ascending=[True, False]))\n    i = dcg(d.sort_values(['sid', 'y'], ascending=[True, False]))\n    return p / i.replace(0, np.nan)\n\ndef ndcg_at_k(sid, y_true, y_score, k=20):\n    \"\"\"Обёртка для разведочных клеток: среднее по всем сессиям (пустые=0), по непустым и доля пустых.\"\"\"\n    per = sess_ndcg(sid, y_true, y_score, k)\n    return pd.Series({'ndcg_all(пустые=0)': per.fillna(0).mean(), 'ndcg_nonempty': per.mean(),\n                      'share_empty': per.isna().mean()})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:45.145403Z","iopub.execute_input":"2026-10-19T07:19:45.145798Z","iopub.status.idle":"2026-10-19T07:19:45.156013Z","shell.execute_reply.started":"2026-10-19T07:19:45.145748Z","shell.execute_reply":"2026-10-19T07:19:45.154972Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [EXP] Простые бейзлайны (ранжирование по одному столбцу)\nr_sizes = train.groupby('sid').size()\nr_f20 = train[train['sid'].map(r_sizes) == 20].copy()\nr_f20['row_in_sess'] = r_f20.groupby('sid').cumcount()\nrng = np.random.default_rng(SEED); n = len(r_f20)\ncands = {'random': rng.random(n),\n         '-product_position': -r_f20['product_position'].values,\n         '+product_position': r_f20['product_position'].values,\n         '-порядок строк': -r_f20['row_in_sess'].values,\n         '+порядок строк': r_f20['row_in_sess'].values,\n         'available': r_f20['available'].values + 1e-3 * rng.random(n),\n         '-price': -r_f20['price'].values.astype(float),\n         'price_is_discounted': r_f20['price_is_discounted'].values + 1e-3 * rng.random(n)}\ndisplay(pd.DataFrame({k: ndcg_at_k(r_f20['sid'], r_f20['relevance'], v) for k, v in cands.items()}).T.round(4))\nprint('-product_position на ВСЁМ train (все размеры):')\nprint(ndcg_at_k(train['sid'], train['relevance'], -train['product_position'].values).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:45.157287Z","iopub.execute_input":"2026-10-19T07:19:45.157654Z","iopub.status.idle":"2026-10-19T07:19:49.517491Z","shell.execute_reply.started":"2026-10-19T07:19:45.15762Z","shell.execute_reply":"2026-10-19T07:19:49.516577Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"* -product_position - наивный бейзлайн (сортивка по столбцу product_position)","metadata":{}},{"cell_type":"markdown","source":"# 3. Test и смена режима выдачи","metadata":{}},{"cell_type":"code","source":"# [DEF]\ndef first20_sample(df):\n    \"\"\"Дубли (сессия, товар) схлопнуть по максимуму relevance; сессии >=20 товаров; первые 20 по позиции.\"\"\"\n    d = df.sort_values(['sid', 'product_id', 'relevance']).drop_duplicates(['sid', 'product_id'], keep='last')\n    d = d.sort_values(['sid', 'product_position'], kind='stable')\n    d = d[d.groupby('sid')['product_id'].transform('size') >= 20]\n    d = d[d.groupby('sid').cumcount() < 20].copy()\n    d['sdate'] = d.groupby('sid')['date'].transform('min')\n    return d\n\ndef pos_only(d):\n    \"\"\"Только сессии с хотя бы одной положительной меткой (в остальных нулевой градиент lambdarank).\"\"\"\n    return d[d.groupby('sid')['relevance'].transform('max') > 0].reset_index(drop=True)\n\ndef summarize(pmin, pmax):\n    span = pmax - pmin + 1\n    return pd.Series({'pmin==1': (pmin == 1).mean(), '(pmin-1)%30==0': ((pmin - 1) % 30 == 0).mean(),\n                      'pmin_median': np.median(pmin), 'pmin_q75': np.quantile(pmin, .75),\n                      'span==20': (span == 20).mean(), 'span<=30': (span <= 30).mean(),\n                      'span_median': np.median(span), 'span_q75': np.quantile(span, .75)})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.520611Z","iopub.execute_input":"2026-10-19T07:19:49.521015Z","iopub.status.idle":"2026-10-19T07:19:49.530069Z","shell.execute_reply.started":"2026-10-19T07:19:49.520989Z","shell.execute_reply":"2026-10-19T07:19:49.529202Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Размеры сессий во времени; доля сессий с положительной меткой по размеру\n# r_sess = pd.DataFrame({'date': train.groupby('sid')['date'].min(),\n#                        'n': train.groupby('sid').size(),\n#                        'has_pos': (train['relevance'] > 0).groupby(train['sid']).max()})\n# r_sess['week'] = r_sess['date'].dt.to_period('W').dt.start_time\n# r_weekly = r_sess.groupby('week').agg(sessions=('n', 'size'), median_size=('n', 'median'),\n#                                       share20=('n', lambda x: (x == 20).mean()))\n# print('Train, последние 12 недель:'); display(r_weekly.tail(12).round(3))\n# fig, ax = plt.subplots(1, 2, figsize=(14, 3.5))\n# r_weekly['share20'].plot(ax=ax[0], title='Доля сессий размера 20 по неделям (train)')\n# r_weekly['sessions'].plot(ax=ax[1], title='Сессий в неделю (train)'); plt.show()\n\n# r_t = test.assign(key_num=test['session_product_id'].str.split('_').str[0].astype(int))\n# r_tw = r_t.groupby(r_t['date'].dt.to_period('W').dt.start_time)['key_num'].agg(['min', 'max', 'nunique'])\n# r_tw['span'] = r_tw['max'] - r_tw['min'] + 1\n# r_tw['share_est'] = r_tw['nunique'] / r_tw['span']\n# print('Test по неделям (номер сессии из ключа):'); display(r_tw)\n# print('номер из ключа растёт вместе с датой:', r_t.groupby('key_num')['date'].first().is_monotonic_increasing)\n\n# r_sess['size_bin'] = pd.cut(r_sess['n'], [0, 1, 2, 3, 4, 6, 8, 12, 19, 20, 30, 60, 10**5])\n# print('Доля сессий хотя бы с одним relevance>0 по размеру сессии:')\n# display(r_sess.groupby('size_bin', observed=True).agg(sessions=('n', 'size'), share_with_pos=('has_pos', 'mean')).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.531392Z","iopub.execute_input":"2026-10-19T07:19:49.531999Z","iopub.status.idle":"2026-10-19T07:19:49.56292Z","shell.execute_reply.started":"2026-10-19T07:19:49.531971Z","shell.execute_reply":"2026-10-19T07:19:49.56179Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Чему из train похож test по распределениям признаков\n# r_sizes = train.groupby('sid').size()\n# r_f20 = train[train['sid'].map(r_sizes) == 20]\n# r_last = train[train['date'] >= train['date'].max() - pd.Timedelta(days=61)]\n# r_parts = {'train_all': train, 'train_last62d': r_last, 'train_size20': r_f20, 'test': test}\n# r_feats = ['filters_applied', 'price_is_discounted', 'price_is_super', 'available', 'response_products',\n#            'product_position', 'browse_time', 'response_time', 'price']\n# print('Средние:'); display(pd.DataFrame({k: d[r_feats].mean() for k, d in r_parts.items()}).round(3))\n# print('Медианы:'); display(pd.DataFrame({k: d[r_feats].median() for k, d in r_parts.items()}).round(3))\n# print('device:'); display(pd.DataFrame({k: d['device'].value_counts(normalize=True) for k, d in r_parts.items()}).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.564133Z","iopub.execute_input":"2026-10-19T07:19:49.564998Z","iopub.status.idle":"2026-10-19T07:19:49.589036Z","shell.execute_reply.started":"2026-10-19T07:19:49.564957Z","shell.execute_reply":"2026-10-19T07:19:49.587833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Симуляция правил отбора 20 товаров (по позициям, без меток)\n# rng = np.random.default_rng(SEED)\n# r_cut = train['date'].max() - pd.Timedelta(days=61)\n# r_rec = train[train['date'] >= r_cut]\n# r_rsz = r_rec.groupby('sid')['product_id'].size()\n# print('Сессий в последних 62 днях:', len(r_rsz), '| с >=20 товарами:', (r_rsz >= 20).sum(),\n#       f'({(r_rsz >= 20).mean():.1%})', '| ровно 20:', (r_rsz == 20).sum())\n# r_rec = r_rec[r_rec['sid'].isin(r_rsz[r_rsz >= 20].index)]\n# r_pos_by_sid = {k: np.sort(v.values) for k, v in r_rec.groupby('sid')['product_position']}\n\n# def simulate(kind):\n#     out = []\n#     for p in r_pos_by_sid.values():\n#         if kind == 'first20':    q = p[:20]\n#         elif kind == 'last20':   q = p[-20:]\n#         elif kind == 'random20': q = np.sort(rng.choice(p, 20, replace=False))\n#         elif kind == 'window20':\n#             i = rng.integers(0, len(p) - 19); q = p[i:i + 20]\n#         out.append((q[0], q[-1]))\n#     a = np.array(out)\n#     return summarize(a[:, 0], a[:, 1])\n\n# r_tg = test.groupby('sid')['product_position']\n# r_sim = {'TEST': summarize(r_tg.min().values, r_tg.max().values)}\n# for kind in ['first20', 'last20', 'random20', 'window20']:\n#     r_sim['sim_' + kind] = simulate(kind)\n# display(pd.DataFrame(r_sim).round(3))\n# print('Топ значений pmin в test:'); print(r_tg.min().value_counts().head(8))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.59014Z","iopub.execute_input":"2026-10-19T07:19:49.590441Z","iopub.status.idle":"2026-10-19T07:19:49.625871Z","shell.execute_reply.started":"2026-10-19T07:19:49.590415Z","shell.execute_reply":"2026-10-19T07:19:49.624625Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Признаки, постоянные внутри сессии (train: сессии >=20 товаров, последние 62 дня; и test)\n# r_rec = train[train['date'] >= train['date'].max() - pd.Timedelta(days=61)]\n# r_rec = r_rec[r_rec.groupby('sid')['product_id'].transform('size') >= 20]\n# r_chk = ['query', 'date', 'device', 'selected_category_id', 'response_products', 'response_time',\n#          'browse_time', 'filters_applied']\n# print('Train:'); print((r_rec.groupby('sid')[r_chk].nunique() == 1).mean().round(3))\n# print('Test:');  print((test.groupby('sid')[r_chk].nunique() == 1).mean().round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.627294Z","iopub.execute_input":"2026-10-19T07:19:49.627624Z","iopub.status.idle":"2026-10-19T07:19:49.655657Z","shell.execute_reply.started":"2026-10-19T07:19:49.627597Z","shell.execute_reply":"2026-10-19T07:19:49.654644Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Связь позиции с кликом (последние 62 дня, сессии >=20 товаров)\n# r_rec = train[train['date'] >= train['date'].max() - pd.Timedelta(days=61)]\n# r_rec = r_rec[r_rec.groupby('sid')['product_id'].transform('size') >= 20].copy()\n# r_rec['pos_bin'] = pd.cut(r_rec['product_position'], [0, 1, 3, 5, 10, 20, 30, 31, 60, 61, 100, 10**5])\n# display(r_rec.groupby('pos_bin', observed=True).agg(\n#     rows=('relevance', 'size'), mean_rel=('relevance', 'mean'),\n#     p_click=('relevance', lambda x: (x > 0).mean()), p_buy=('relevance', lambda x: (x == 3).mean())).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.65675Z","iopub.execute_input":"2026-10-19T07:19:49.657074Z","iopub.status.idle":"2026-10-19T07:19:49.693607Z","shell.execute_reply.started":"2026-10-19T07:19:49.657039Z","shell.execute_reply":"2026-10-19T07:19:49.6924Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Когда сменился режим выдачи\n# r_w = train.groupby(train['date'].dt.to_period('W').dt.start_time).agg(\n#     resp_med=('response_products', 'median'),\n#     resp_eq100=('response_products', lambda x: (x == 100).mean()),\n#     rt_med=('response_time', 'median'),\n#     app_share=('device', lambda x: (x == 'app').mean()),\n#     filters=('filters_applied', 'mean'))\n# display(r_w.round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.694973Z","iopub.execute_input":"2026-10-19T07:19:49.69537Z","iopub.status.idle":"2026-10-19T07:19:49.719118Z","shell.execute_reply.started":"2026-10-19T07:19:49.695338Z","shell.execute_reply":"2026-10-19T07:19:49.718162Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Окна по 6 недель до и после смены режима; исходы и старты сессий\n# OLD_WIN = (pd.Timestamp('2025-01-06'), pd.Timestamp('2025-02-17'))\n# NEW_WIN = (pd.Timestamp('2025-03-17'), pd.Timestamp('2025-04-28'))\n# r_raw_win = {k: train[(train['date'] >= w[0]) & (train['date'] < w[1])] for k, w in [('old', OLD_WIN), ('new', NEW_WIN)]}\n# r_samp_win = {k: first20_sample(v) for k, v in r_raw_win.items()}\n\n# rows = []\n# for k in ['old', 'new']:\n#     raw, s = r_raw_win[k], r_samp_win[k]\n#     sz = raw.groupby('sid').size(); g = s.groupby('sid')\n#     rows.append({'regime': k, 'sessions_all': len(sz), 'share_sessions_20plus': (sz >= 20).mean(),\n#                  'sessions_first20': g.ngroups, 'share_empty': (g['relevance'].max() == 0).mean(),\n#                  'rate_click': (s['relevance'] > 0).mean(), 'rate_cart': (s['relevance'] >= 2).mean(),\n#                  'rate_buy': (s['relevance'] == 3).mean(),\n#                  'resp_products_median': raw['response_products'].median(),\n#                  'resp_time_median': raw['response_time'].median(),\n#                  'app_share': (raw['device'] == 'app').mean(),\n#                  'pmin==1': (g['product_position'].min() == 1).mean(),\n#                  'pmin_median': g['product_position'].min().median(),\n#                  'span_median': (g['product_position'].max() - g['product_position'].min() + 1).median()})\n# display(pd.DataFrame(rows).set_index('regime').T.round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.720394Z","iopub.execute_input":"2026-10-19T07:19:49.721277Z","iopub.status.idle":"2026-10-19T07:19:49.744674Z","shell.execute_reply.started":"2026-10-19T07:19:49.721247Z","shell.execute_reply":"2026-10-19T07:19:49.743542Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Связь позиции с кликом в двух режимах\n# r_bins = [0, 1, 3, 5, 10, 20, 30, 31, 60, 61, 100, 10**5]\n# r_tabs = {}\n# for k, s in r_samp_win.items():\n#     s = s.assign(pos_bin=pd.cut(s['product_position'], r_bins))\n#     r_tabs[k] = s.groupby('pos_bin', observed=True).agg(rows=('relevance', 'size'),\n#                                                        p_click=('relevance', lambda x: (x > 0).mean()),\n#                                                        p_buy=('relevance', lambda x: (x == 3).mean()))\n# r_cmp = pd.concat(r_tabs, axis=1)\n# r_cmp[('ratio', 'p_click new/old')] = r_cmp[('new', 'p_click')] / r_cmp[('old', 'p_click')]\n# display(r_cmp.round(4))\n# for k, s in r_samp_win.items():\n#     print('вокруг границы страницы,', k)\n#     display(s[s['product_position'].between(29, 33)].groupby('product_position')['relevance']\n#             .agg(rows='size', p_click=lambda x: (x > 0).mean()).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.745993Z","iopub.execute_input":"2026-10-19T07:19:49.746361Z","iopub.status.idle":"2026-10-19T07:19:49.773517Z","shell.execute_reply.started":"2026-10-19T07:19:49.746332Z","shell.execute_reply":"2026-10-19T07:19:49.772591Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] NDCG при ранжировании по одному признаку, непустые сессии\n# r_feats1 = ['product_position', 'price', 'price_discount', 'price_is_discounted', 'price_is_super', 'available']\n# rng = np.random.default_rng(SEED); r_out = []\n# for k, s in r_samp_win.items():\n#     s = pos_only(s)\n#     rnd = ndcg_at_k(s['sid'], s['relevance'], rng.random(len(s))).iloc[1]\n#     for f in r_feats1:\n#         v = s[f].values.astype('float64'); v = v + 1e-6 * (v.std() + 1e-9) * rng.random(len(v))\n#         hi = ndcg_at_k(s['sid'], s['relevance'], v).iloc[1]\n#         lo = ndcg_at_k(s['sid'], s['relevance'], -v).iloc[1]\n#         r_out.append({'regime': k, 'feature': f, 'lift_vs_random': round(max(hi, lo) - rnd, 4),\n#                       'dir': 'desc' if hi >= lo else 'asc'})\n# display(pd.DataFrame(r_out).pivot(index='feature', columns='regime', values=['lift_vs_random', 'dir']))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.774722Z","iopub.execute_input":"2026-10-19T07:19:49.775077Z","iopub.status.idle":"2026-10-19T07:19:49.803843Z","shell.execute_reply.started":"2026-10-19T07:19:49.775047Z","shell.execute_reply":"2026-10-19T07:19:49.802843Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Продолжение выдачи: сессия того же (user, day, query), идущая после другой\n# rw = r_raw_win['new']\n# r_S = rw.groupby('sid').agg(user=('user_number', 'first'), sn=('search_number', 'first'), q=('query', 'first'),\n#                             day=('date', 'first'), pmin=('product_position', 'min'),\n#                             pmax=('product_position', 'max'), n=('product_id', 'size')).reset_index()\n# r_first_click = rw.sort_values(['sid', 'product_position'], kind='stable').groupby('sid')['relevance'].first() > 0\n# r_S['first_click'] = r_S['sid'].map(r_first_click)\n# r_S = r_S.sort_values(['user', 'day', 'q', 'sn'])\n# r_grp = r_S.groupby(['user', 'day', 'q'])\n# r_S['prev_pmin'], r_S['prev_pmax'] = r_grp['pmin'].shift(), r_grp['pmax'].shift()\n# r_S['has_prev'] = r_S['prev_pmax'].notna()\n# r_S['continues'] = r_S['has_prev'] & (r_S['pmin'] > r_S['prev_pmin'])\n# r_S['adjacent'] = r_S['has_prev'] & (r_S['pmin'] == r_S['prev_pmax'] + 1)\n# r_S['start_class'] = np.select([r_S['pmin'] == 1, r_S['pmin'] == 31, r_S['pmin'] == 61, r_S['pmin'] == 91],\n#                                ['1', '31', '61', '91'], default='other')\n# display(r_S.groupby('start_class').agg(sessions=('sid', 'size'), median_rows=('n', 'median'),\n#                                        has_prev=('has_prev', 'mean'), continues=('continues', 'mean'),\n#                                        adjacent=('adjacent', 'mean'), first_row_click=('first_click', 'mean')).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.804941Z","iopub.execute_input":"2026-10-19T07:19:49.805411Z","iopub.status.idle":"2026-10-19T07:19:49.829036Z","shell.execute_reply.started":"2026-10-19T07:19:49.805381Z","shell.execute_reply":"2026-10-19T07:19:49.828153Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 4. Выборки и валидация","metadata":{}},{"cell_type":"code","source":"# [INIT]\nsamp = first20_sample(train[train['date'] >= REGIME_START])\ntr_s = samp[samp['sdate'] < VALID_START].reset_index(drop=True)\nva_s = samp[samp['sdate'] >= VALID_START].reset_index(drop=True)\ndel samp; gc.collect()\nprint('tr_s sessions:', tr_s['sid'].nunique(), '| va_s sessions:', va_s['sid'].nunique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:49.830274Z","iopub.execute_input":"2026-10-19T07:19:49.831002Z","iopub.status.idle":"2026-10-19T07:19:52.59907Z","shell.execute_reply.started":"2026-10-19T07:19:49.830972Z","shell.execute_reply":"2026-10-19T07:19:52.598151Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [EXP] Насколько tr_s и va_s похожи на test\ndef sess_stats(d):\n    g = d.groupby('sid')\n    s = summarize(g['product_position'].min().values, g['product_position'].max().values)\n    s['n_req==1'] = (g['response_time'].nunique() == 1).mean()\n    s['share_empty'] = (g['relevance'].max() == 0).mean()\n    return s\n\nr_cmp = pd.DataFrame({'TEST': sess_stats(test.assign(relevance=0)).drop('share_empty'),\n                      'valid_first20': sess_stats(va_s), 'train_first20': sess_stats(tr_s)})\ndisplay(r_cmp.round(3))\nprint('-product_position на va_s:'); print(ndcg_at_k(va_s['sid'], va_s['relevance'], -va_s['product_position'].values).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.600592Z","iopub.execute_input":"2026-10-19T07:19:52.601017Z","iopub.status.idle":"2026-10-19T07:19:52.606894Z","shell.execute_reply.started":"2026-10-19T07:19:52.600991Z","shell.execute_reply":"2026-10-19T07:19:52.60592Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 5. Признаки позиции","metadata":{}},{"cell_type":"code","source":"# [DEF]\ndef add_page_feats(d):\n    d = d.copy(); pos = d['product_position']\n    d['rank_in_session'] = d.groupby('sid').cumcount()\n    d['pos_in_page'] = (pos - 1) % 30\n    d['page'] = (pos - 1) // 30\n    d['is_page_start'] = (d['pos_in_page'] == 0).astype('int8')\n    return d\n\nFEAT_ALL = ['product_position', 'response_products', 'response_time', 'browse_time', 'filters_applied',\n            'price_is_discounted', 'price_is_super', 'price', 'price_discount', 'available',\n            'selected_category_id', 'device']\nF_BASE = FEAT_ALL + ['rank_in_session', 'pos_in_page']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.608087Z","iopub.execute_input":"2026-10-19T07:19:52.608516Z","iopub.status.idle":"2026-10-19T07:19:52.635377Z","shell.execute_reply.started":"2026-10-19T07:19:52.608482Z","shell.execute_reply":"2026-10-19T07:19:52.634464Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT]\ntrn = add_page_feats(pos_only(tr_s))\nval = add_page_feats(va_s)\nvan = val[val.groupby('sid')['relevance'].transform('max') > 0].reset_index(drop=True)\ntest_f = add_page_feats(test.reset_index(drop=True))\n\nfor d in (trn, van):\n    assert (d.groupby('sid').size() == 20).all()\n    assert (d['sid'].values[::20] == d['sid'].values[19::20]).all()\n\ng = van.groupby('sid')['product_position']\nspan, pmin = g.max() - g.min() + 1, g.min()\nslices = {'all': van['sid'].unique(), 'span<=30': span.index[span <= 30], 'span>30': span.index[span > 30],\n          'pmin==1': pmin.index[pmin == 1], 'pmin>1': pmin.index[pmin > 1]}\nprint('trn:', trn['sid'].nunique(), '| van:', van['sid'].nunique(), '| срезы:', {k: len(v) for k, v in slices.items()})","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.637022Z","iopub.execute_input":"2026-10-19T07:19:52.6375Z","iopub.status.idle":"2026-10-19T07:19:52.834737Z","shell.execute_reply.started":"2026-10-19T07:19:52.637464Z","shell.execute_reply":"2026-10-19T07:19:52.83336Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дают ли страничные признаки прирост над raw12, по срезам\n# PAGE4 = ['rank_in_session', 'pos_in_page', 'page', 'is_page_start']\n# r_res = {'raw12': train_on(trn, FEAT_ALL), 'raw12+4pages': train_on(trn, FEAT_ALL + PAGE4)} # функция train_on определена в Главе 6, DEF-2\n# r_pos = sess_ndcg(van['sid'], van['relevance'], -van['product_position'].values)\n# print('-position по срезам:', {sl: round(r_pos.loc[ids].mean(), 4) for sl, ids in slices.items()})\n# display(compare(r_res, 'raw12', slice_names=tuple(slices)).round(4))\n\n# r_m = lgb.train(dict(params_cmp, seed=1), lgb.Dataset(prep(trn, FEAT_ALL + PAGE4), trn['relevance'],\n#                 group=trn.groupby('sid', sort=False).size().values), 300)\n# r_imp = pd.Series(r_m.feature_importance('gain'), index=FEAT_ALL + PAGE4).sort_values(ascending=False)\n# print((r_imp / r_imp.sum()).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.836106Z","iopub.execute_input":"2026-10-19T07:19:52.836514Z","iopub.status.idle":"2026-10-19T07:19:52.841155Z","shell.execute_reply.started":"2026-10-19T07:19:52.836478Z","shell.execute_reply":"2026-10-19T07:19:52.840209Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Что мешает: колонки сессии, абсолютная цена\n# SESSION_COLS = ['browse_time', 'response_time', 'selected_category_id']\n# ABS_PRICE_COLS = ['price', 'price_discount']\n# r_variants = {'base': F_BASE,\n#               'no_session_cols': [f for f in F_BASE if f not in SESSION_COLS],\n#               'no_abs_price': [f for f in F_BASE if f not in ABS_PRICE_COLS],\n#               'no_both': [f for f in F_BASE if f not in SESSION_COLS + ABS_PRICE_COLS]}\n# r_res = {name: train_on(trn, feats) for name, feats in r_variants.items()}\n# display(compare(r_res, 'base').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.842378Z","iopub.execute_input":"2026-10-19T07:19:52.842664Z","iopub.status.idle":"2026-10-19T07:19:52.873238Z","shell.execute_reply.started":"2026-10-19T07:19:52.842633Z","shell.execute_reply":"2026-10-19T07:19:52.872242Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 6. Инструменты моделирования","metadata":{}},{"cell_type":"code","source":"# [DEF]\nparams_cmp = dict(objective='lambdarank', metric='ndcg', lambdarank_truncation_level=20,\n                  label_gain=[0, 1, 3, 7], learning_rate=0.05, num_leaves=31, min_data_in_leaf=100,\n                  feature_fraction=0.9, bagging_fraction=0.8, bagging_freq=1, lambda_l2=10, verbose=-1)\n\ndef prep(d, feats):\n    X = d[feats].copy()\n    if 'device' in feats:\n        X['device'] = pd.Categorical(X['device'].astype(str), categories=['app', 'desktop', 'mobile'])\n    return X\n\ndef need_cols(feats, frames):\n    \"\"\"Падает с понятным сообщением, если нужной колонки нет (обычно не запущена INIT-HISTORY клетка).\"\"\"\n    for name in frames:\n        miss = [f for f in feats if f not in globals()[name].columns]\n        assert not miss, f'в {name} нет колонок {miss}: запустите INIT-клетки, которые их создают (глава 9)'\n\ndef take(d, n):\n    keep = d['sid'].drop_duplicates().sample(n=n, random_state=SEED)\n    return d[d['sid'].isin(keep)].reset_index(drop=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.874536Z","iopub.execute_input":"2026-10-19T07:19:52.874887Z","iopub.status.idle":"2026-10-19T07:19:52.896269Z","shell.execute_reply.started":"2026-10-19T07:19:52.87485Z","shell.execute_reply":"2026-10-19T07:19:52.895409Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [DEF]\ndef train_on(d, feats, gcol='sid', n_iter=300, ckpts=(100, 200, 300), seeds=(1, 2, 3)):\n    \"\"\"Обучение на d, NDCG по сессиям van на чекпоинтах, усреднено по сидам.\"\"\"\n    need_cols(feats, ('van',))\n    assert not [f for f in feats if f not in d.columns], 'в обучающем d нет части признаков'\n    grp = d.groupby(gcol, sort=False).size().values\n    out = {k: [] for k in ckpts}\n    for sd in seeds:\n        p = dict(params_cmp, seed=sd, bagging_seed=sd, feature_fraction_seed=sd)\n        m = lgb.train(p, lgb.Dataset(prep(d, feats), d['relevance'], group=grp), n_iter)\n        for k in ckpts:\n            out[k].append(sess_ndcg(van['sid'], van['relevance'], m.predict(prep(van, feats), num_iteration=k)))\n    return {k: pd.concat(v, axis=1).mean(axis=1) for k, v in out.items()}\n\ndef compare(results, ref, slice_names=('all', 'span>30', 'pmin>1')):\n    rows = []\n    for name, res in results.items():\n        for sl in slice_names:\n            sids = slices[sl]\n            row = {'variant': name, 'slice': sl, 'sessions': len(sids), 'ndcg_200': res[200].loc[sids].mean()}\n            for k in (100, 200, 300):\n                row[f'diff_{k}'] = (res[k].loc[sids] - results[ref][k].loc[sids]).mean()\n            dd = res[200].loc[sids] - results[ref][200].loc[sids]\n            row['±2SE_200'] = 2 * dd.std() / np.sqrt(len(dd))\n            rows.append(row)\n    return pd.DataFrame(rows)\n\ndef train_and_submit(tag, feats, n_iter, seeds=(1, 2, 3, 4, 5)):\n    \"\"\"Обучение на trn+van, усреднение по сидам, запись submission.csv и копии в archive/.\"\"\"\n    need_cols(feats, ('trn', 'van', 'test_f'))\n    final = pd.concat([trn, van], ignore_index=True)\n    grp = final.groupby('sid', sort=False).size().values\n    scores = np.zeros(len(test_f))\n    for sd in seeds:\n        p = dict(params_cmp, seed=sd, bagging_seed=sd, feature_fraction_seed=sd)\n        m = lgb.train(p, lgb.Dataset(prep(final, feats), final['relevance'], group=grp), n_iter)\n        scores += m.predict(prep(test_f, feats))\n    scores /= len(seeds)\n    sub = pd.DataFrame({'session_product_id': test['session_product_id'].values, 'relevance': scores})\n    assert len(sub) == len(sample_sub) and np.isfinite(sub['relevance']).all()\n    assert (sub['session_product_id'].values == sample_sub['session_product_id'].values).all()\n    os.makedirs(f'{OUT_DIR}/archive', exist_ok=True)\n    sub.to_csv(f'{OUT_DIR}/archive/{tag}.csv', index=False)\n    sub.to_csv(f'{OUT_DIR}/submission.csv', index=False)\n    print(f'{tag}: {len(feats)} признаков, {n_iter} итераций, {len(seeds)} сидов | '\n          f'уникальных скоров {pd.Series(scores).nunique() / len(scores):.3f} | submission.csv записан')\n    return scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.90247Z","iopub.execute_input":"2026-10-19T07:19:52.902847Z","iopub.status.idle":"2026-10-19T07:19:52.931221Z","shell.execute_reply.started":"2026-10-19T07:19:52.90281Z","shell.execute_reply":"2026-10-19T07:19:52.93022Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] raw12, F_BASE, raw12+4pages: пять сидов, чекпоинты 150/200/250\n# PAGE4 = ['rank_in_session', 'pos_in_page', 'page', 'is_page_start']\n# CK2, SEEDS5 = (150, 200, 250), (1, 2, 3, 4, 5)\n# MODELS = {'raw12': FEAT_ALL, 'F_BASE': F_BASE, 'raw12+4pages': FEAT_ALL + PAGE4}\n# r_F1 = {name: train_on(trn, feats, n_iter=max(CK2), ckpts=CK2, seeds=SEEDS5) for name, feats in MODELS.items()}\n# r_pos = sess_ndcg(van['sid'], van['relevance'], -van['product_position'].values)\n\n# rows = []\n# for sl, sids in slices.items():\n#     for name in MODELS:\n#         for k in CK2:\n#             ps = r_F1[name][k].loc[sids]; d = ps - r_F1['raw12'][k].loc[sids]\n#             rows.append({'slice': sl, 'sessions': len(sids), 'model': name, 'iters': k,\n#                          '-position': r_pos.loc[sids].mean(), 'ndcg': ps.mean(),\n#                          'diff_vs_raw12': d.mean(), '±2SE': 2 * d.std() / np.sqrt(len(d))})\n# r_tab = pd.DataFrame(rows)\n# display(r_tab[(r_tab['iters'] == 200) & r_tab['slice'].isin(['all', 'span>30', 'pmin>1'])].round(4))\n# print('срез all по числу итераций:')\n# display(r_tab[r_tab['slice'] == 'all'].pivot(index='model', columns='iters', values='ndcg').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.932483Z","iopub.execute_input":"2026-10-19T07:19:52.932858Z","iopub.status.idle":"2026-10-19T07:19:52.961931Z","shell.execute_reply.started":"2026-10-19T07:19:52.932824Z","shell.execute_reply":"2026-10-19T07:19:52.960808Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"12 сырых колонок (позиция, цена, скидка, флаги, колонки сессии).","metadata":{}},{"cell_type":"markdown","source":"# 7. Режимы и выборки\n**Что:** новый режим выдачи (с 2025-03-10), первые 20 строк сессий с ≥20 товарами, дубли схлопнуты, в обучении только сессии с положительной меткой (11 071 сессий), валидация 4 515 сессий.\\\n**Результат:** старты сессий совпадают с test. Размах позиций в test шире, причина неизвестна. Старый режим один хуже простой позиции и к новым данным ничего не добавляет. Выборки B (сессии ≥10) и C (все окна по 20 строк) не лучше первого окна.","metadata":{}},{"cell_type":"code","source":"# # [EXP] Старый режим, новый режим, оба: обучение на окне, оценка на van\n# FEAT_NOSERV = [f for f in FEAT_ALL if f not in ('response_products', 'response_time')]\n# po, pn = pos_only(r_samp_win['old']), pos_only(r_samp_win['new'])\n# print('положительных сессий: old', po['sid'].nunique(), '| new', pn['sid'].nunique())\n# n_eq = min(po['sid'].nunique(), pn['sid'].nunique())\n# po, pn = take(po, n_eq), take(pn, n_eq)                       # поровну сессий, чтобы не смешивать с размером выборки\n# r_sets = {'old_only': po, 'new_only': pn, 'old+new': pd.concat([po, pn], ignore_index=True)}\n# print('-position на van:', round(sess_ndcg(van['sid'], van['relevance'], -van['product_position'].values).mean(), 4))\n\n# r_ps = {}\n# for fs_name, feats in [('all12', FEAT_ALL), ('no_service', FEAT_NOSERV)]:\n#     for tname, d in r_sets.items():\n#         r_ps[(fs_name, tname)] = train_on(d, feats, n_iter=200, ckpts=(200,), seeds=(1, 2))[200]\n# rows = []\n# for (fs, t), ps in r_ps.items():\n#     dl = ps - r_ps[(fs, 'new_only')]\n#     rows.append({'features': fs, 'train': t, 'train_sessions': r_sets[t]['sid'].nunique(),\n#                  'ndcg_nonempty': ps.mean(), 'diff_vs_new_only': dl.mean(), '±2SE': 2 * dl.std() / np.sqrt(len(dl))})\n# display(pd.DataFrame(rows).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.963229Z","iopub.execute_input":"2026-10-19T07:19:52.964247Z","iopub.status.idle":"2026-10-19T07:19:52.990007Z","shell.execute_reply.started":"2026-10-19T07:19:52.964208Z","shell.execute_reply":"2026-10-19T07:19:52.988917Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Помогает ли больше сессий нового режима\n# r_base = sess_ndcg(van['sid'], van['relevance'], -van['product_position'].values)\n# rows = []\n# for n_sess in [1000, 2000, 4000, 8000, trn['sid'].nunique()]:\n#     d = trn if n_sess >= trn['sid'].nunique() else take(trn, n_sess)\n#     ps = train_on(d, FEAT_ALL, n_iter=200, ckpts=(200,), seeds=(1, 2))[200]\n#     dl = ps - r_base\n#     rows.append({'train_sessions': d['sid'].nunique(), 'ndcg_nonempty': ps.mean(),\n#                  'gain_over_position': dl.mean(), '±2SE': 2 * dl.std() / np.sqrt(len(dl))})\n# display(pd.DataFrame(rows).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:52.991216Z","iopub.execute_input":"2026-10-19T07:19:52.992279Z","iopub.status.idle":"2026-10-19T07:19:53.024877Z","shell.execute_reply.started":"2026-10-19T07:19:52.992243Z","shell.execute_reply":"2026-10-19T07:19:53.023791Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Старый и новый режим на сессиях со стартом с позиции 1 и после\n# FEAT_NOPOS = [f for f in FEAT_NOSERV if f != 'product_position']\n# pn_p1 = pn[pn.groupby('sid')['product_position'].transform('min') == 1]\n# r_sets = {'old_only': po, 'new_only': pn, 'new_pmin1_only': pn_p1}\n# print({k: d['sid'].nunique() for k, d in r_sets.items()})\n# r_pmin = van.groupby('sid')['product_position'].transform('min')\n# r_sid_sets = {'valid_pmin1': van.loc[r_pmin == 1, 'sid'].unique(), 'valid_pmin>1': van.loc[r_pmin > 1, 'sid'].unique()}\n# r_base = sess_ndcg(van['sid'], van['relevance'], -van['product_position'].values)\n\n# rows = []\n# for fs_name, feats in [('no_service', FEAT_NOSERV), ('no_position', FEAT_NOPOS)]:\n#     ps = {t: train_on(d, feats, n_iter=200, ckpts=(200,), seeds=(1, 2))[200] for t, d in r_sets.items()}\n#     for sub_name, sids in r_sid_sets.items():\n#         for t in r_sets:\n#             dl = ps[t].loc[sids] - ps['new_only'].loc[sids]\n#             rows.append({'features': fs_name, 'valid_subset': sub_name, 'sessions': len(sids), 'train': t,\n#                          'ndcg': ps[t].loc[sids].mean(), '-position': r_base.loc[sids].mean(),\n#                          'diff_vs_new_only': dl.mean(), '±2SE': 2 * dl.std() / np.sqrt(len(dl))})\n# display(pd.DataFrame(rows).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:53.025948Z","iopub.execute_input":"2026-10-19T07:19:53.026338Z","iopub.status.idle":"2026-10-19T07:19:53.050289Z","shell.execute_reply.started":"2026-10-19T07:19:53.026312Z","shell.execute_reply":"2026-10-19T07:19:53.049274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] A: первое окно, сессии >=20 | B: первое окно, сессии >=10 | C: все окна по 20 строк\n# r_raw = (train[train['date'] >= REGIME_START]\n#          .sort_values(['sid', 'product_id', 'relevance'], kind='stable')\n#          .drop_duplicates(['sid', 'product_id'], keep='last')\n#          .sort_values(['sid', 'product_position'], kind='stable').reset_index(drop=True))\n# r_raw['sdate'] = r_raw.groupby('sid')['date'].transform('min')\n# r_raw = r_raw[r_raw['sdate'] < VALID_START].copy()\n# r_raw['n_rows'] = r_raw.groupby('sid')['product_id'].transform('size')\n# r_raw['rank_global'] = r_raw.groupby('sid').cumcount()\n# r_raw['window'] = r_raw['rank_global'] // 20\n# r_raw['rank_in_session'] = r_raw['rank_global'] % 20           # как в test: 0..19 внутри окна\n# r_raw['pos_in_page'] = (r_raw['product_position'] - 1) % 30\n# r_raw['gid'] = r_raw['sid'].astype('int64') * 1000 + r_raw['window']\n\n# r_samples = {'A_first20_ge20': r_raw[(r_raw['n_rows'] >= 20) & (r_raw['window'] == 0)],\n#              'B_first20_ge10': r_raw[(r_raw['n_rows'] >= 10) & (r_raw['window'] == 0)],\n#              'C_all_windows':  r_raw[(r_raw['n_rows'] >= 20) & (r_raw['window'] < r_raw['n_rows'] // 20)]}\n# for name in r_samples:\n#     d = r_samples[name]\n#     r_samples[name] = d[d.groupby('gid')['relevance'].transform('max') > 0].reset_index(drop=True)\n#     s = r_samples[name]\n#     print(f'{name:16s} положительных групп: {s[\"gid\"].nunique():6d} | строк: {len(s):8d} | '\n#           f'медиана размера группы: {s.groupby(\"gid\").size().median():.0f}')\n# print('для сверки: в trn', trn['sid'].nunique(), 'положительных сессий')\n\n# r_S3 = {name: train_on(d, F_BASE, gcol='gid') for name, d in r_samples.items()}\n# display(compare(r_S3, 'A_first20_ge20').round(4))\n# del r_raw; gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:53.051389Z","iopub.execute_input":"2026-10-19T07:19:53.051755Z","iopub.status.idle":"2026-10-19T07:19:53.078506Z","shell.execute_reply.started":"2026-10-19T07:19:53.051689Z","shell.execute_reply":"2026-10-19T07:19:53.077489Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 8. Справочник и анализ групп","metadata":{}},{"cell_type":"code","source":"# [INIT]\nprod = products.drop_duplicates('product_id', keep='last').copy()\nprod['date_of_create'] = pd.to_datetime(prod['date_of_create'], errors='coerce')\nprod['type_code'] = prod['type'].astype('category').cat.codes\nprod['brand_code'] = prod['brand'].astype('category').cat.codes\nprint('products:', products.shape, '-> prod:', prod.shape,\n      '| доля строк tr_s без записи в справочнике:', round((~tr_s['product_id'].isin(prod['product_id'])).mean(), 4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:53.079837Z","iopub.execute_input":"2026-10-19T07:19:53.080197Z","iopub.status.idle":"2026-10-19T07:19:53.152202Z","shell.execute_reply.started":"2026-10-19T07:19:53.080163Z","shell.execute_reply":"2026-10-19T07:19:53.151154Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Чем отличаются дубли в справочнике\n# r_dup_ids = products['product_id'][products['product_id'].duplicated(keep=False)].unique()\n# r_d = products[products['product_id'].isin(r_dup_ids)].sort_values('product_id')\n# r_cols = [c for c in products.columns if c != 'product_id']\n# r_diff = r_d.groupby('product_id')[r_cols].nunique().gt(1).sum()\n# print('дублирующихся product_id:', len(r_dup_ids)); print('в скольких различается колонка:'); print(r_diff[r_diff > 0])\n# display(r_d.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:53.153404Z","iopub.execute_input":"2026-10-19T07:19:53.153729Z","iopub.status.idle":"2026-10-19T07:19:53.158031Z","shell.execute_reply.started":"2026-10-19T07:19:53.15367Z","shell.execute_reply":"2026-10-19T07:19:53.157182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Привлекательность групп сверх позиции (clicks over expected) и надёжность (split-half по времени)\n# r_R = tr_s[['date', 'query', 'user_number', 'product_id', 'product_position', 'relevance']].copy()\n# r_R['clk'] = (r_R['relevance'] > 0).astype('int8')\n# r_R['pb'] = np.minimum(r_R['product_position'], 100)\n# r_R['exp'] = r_R.groupby('pb')['clk'].transform('mean')\n# r_R = r_R.merge(prod[['product_id', 'type', 'brand', 'category_name_3', 'category_name_4']], on='product_id', how='left')\n\n# def coe_table(key, min_n=1500, k=5):\n#     g = r_R.groupby(key, observed=True).agg(n=('clk', 'size'), clk=('clk', 'mean'), exp=('exp', 'mean'))\n#     g['coe'] = g['clk'] / g['exp']\n#     g = g[g['n'] >= min_n].sort_values('coe')\n#     print(f'--- {key}: групп с n>={min_n}: {len(g)}, std(coe)={g[\"coe\"].std():.3f}')\n#     display(pd.concat([g.head(k), g.tail(k)]).round(4))\n\n# for key in ['category_name_3', 'type', 'brand']:\n#     coe_table(key)\n\n# r_mid = r_R['date'].min() + (r_R['date'].max() - r_R['date'].min()) / 2\n# r_A, r_B = r_R[r_R['date'] < r_mid], r_R[r_R['date'] >= r_mid]\n# r_keys = {'product': (['product_id'], 30), 'brand': (['brand'], 100), 'type': (['type'], 100),\n#           'query': (['query'], 30), 'query×product': (['query', 'product_id'], 8),\n#           'query×brand': (['query', 'brand'], 15), 'user': (['user_number'], 20)}\n# rel = []\n# for name, (key, mn) in r_keys.items():\n#     a = r_A.groupby(key, observed=True).agg(n=('clk', 'size'), c=('clk', 'sum'), e=('exp', 'sum'))\n#     b = r_B.groupby(key, observed=True).agg(n=('clk', 'size'), c=('clk', 'sum'), e=('exp', 'sum'))\n#     j = a.join(b, lsuffix='_a', rsuffix='_b', how='inner')\n#     j = j[(j['n_a'] >= mn) & (j['n_b'] >= mn)]\n#     r = (j['c_a'] / j['e_a']).corr(j['c_b'] / j['e_b'], method='spearman')\n#     known = r_B.set_index(key).index.isin(a.index[a['n'] >= mn]).mean()\n#     rel.append({'key': name, 'keys_in_both': len(j), 'spearman_A_vs_B': r, 'B_rows_covered': known})\n# display(pd.DataFrame(rel).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:53.15942Z","iopub.execute_input":"2026-10-19T07:19:53.159767Z","iopub.status.idle":"2026-10-19T07:19:53.182621Z","shell.execute_reply.started":"2026-10-19T07:19:53.159733Z","shell.execute_reply":"2026-10-19T07:19:53.181477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Простая модель: ожидаемый клик для позиции × множитель группы, оценка по второй половине периода\n# r_R = tr_s[['sid', 'date', 'query', 'user_number', 'product_id', 'product_position', 'relevance']].copy()\n# r_R['clk'] = (r_R['relevance'] > 0).astype('int8')\n# r_R['pb'] = np.minimum(r_R['product_position'], 100)\n# r_R = r_R.merge(prod[['product_id', 'type', 'brand', 'category_name_3']], on='product_id', how='left')\n# for c in ['type', 'brand', 'category_name_3']:\n#     r_R[c] = r_R[c].astype(str)\n# r_mid = r_R['date'].min() + (r_R['date'].max() - r_R['date'].min()) / 2\n# r_A = r_R[r_R['date'] < r_mid].copy()\n# r_B = r_R[r_R['date'] >= r_mid].copy()\n# r_B = r_B[r_B.groupby('sid')['relevance'].transform('max') > 0].reset_index(drop=True)\n# print('сессий в A:', r_A['sid'].nunique(), '| непустых сессий в B:', r_B['sid'].nunique())\n\n# r_exp_pos = r_A.groupby('pb')['clk'].mean()\n# r_A['exp'] = r_A['pb'].map(r_exp_pos)\n# r_B['exp'] = r_B['pb'].map(r_exp_pos).fillna(r_A['clk'].mean())\n\n# rng = np.random.default_rng(SEED)\n# noise = lambda n: 1e-9 * rng.random(n)\n# r_scores = {'random': rng.random(len(r_B)), '-position': -r_B['product_position'].values + noise(len(r_B)),\n#             'exp(position)': r_B['exp'].values + noise(len(r_B))}\n# r_keys = {'product': ['product_id'], 'brand': ['brand'], 'type': ['type'], 'category_3': ['category_name_3'],\n#           'query×brand': ['query', 'brand'], 'query×product': ['query', 'product_id'],\n#           'query': ['query'], 'user': ['user_number']}\n\n# def key_multiplier(key, k=10):\n#     st = r_A.groupby(key).agg(c=('clk', 'sum'), e=('exp', 'sum')).reset_index()\n#     st['coe'] = (st['c'] + k) / (st['e'] + k)\n#     return r_B[key].merge(st[key + ['coe']], on=key, how='left')['coe'].fillna(1.0).values\n\n# r_multi = {}\n# for name, key in r_keys.items():\n#     r_scores['exp × ' + name] = r_B['exp'].values * key_multiplier(key) + noise(len(r_B))\n#     r_multi[name] = (r_B.groupby(['sid'] + key).size().groupby('sid').size() > 1).mean()\n\n# r_per = {n: sess_ndcg(r_B['sid'], r_B['relevance'], s) for n, s in r_scores.items()}\n# rows = []\n# for n, ps in r_per.items():\n#     d = ps - r_per['exp(position)']\n#     rows.append({'score': n, 'ndcg': ps.mean(), 'diff_vs_exp(position)': d.mean(), '±2SE': 2 * d.std() / np.sqrt(len(d))})\n# display(pd.DataFrame(rows).round(4))\n# print('доля сессий, где ключ принимает больше одного значения среди 20 товаров:')\n# print(pd.Series(r_multi).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:53.184076Z","iopub.execute_input":"2026-10-19T07:19:53.184399Z","iopub.status.idle":"2026-10-19T07:19:53.217094Z","shell.execute_reply.started":"2026-10-19T07:19:53.184366Z","shell.execute_reply":"2026-10-19T07:19:53.215809Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 9. Историческая привлекательность товара","metadata":{}},{"cell_type":"code","source":"# [INIT-HISTORY]\nTEST_START = test['date'].min()\nfor d in (trn, van):\n    k = (d['sdate'] - REGIME_START).dt.days // CUTOFF_DAYS\n    d['cutoff'] = REGIME_START + pd.to_timedelta(k * CUTOFF_DAYS, unit='D')\ntest_f['cutoff'] = TEST_START\ncutoffs = sorted(set(trn['cutoff']) | set(van['cutoff']) | {TEST_START})\n\nHh = train[['date', 'product_id', 'product_position', 'relevance']].copy()\nHh['regime'] = (Hh['date'] >= REGIME_START).astype('int8')\nHh['pb'] = np.minimum(Hh['product_position'], 100)\nHh['clk'] = (Hh['relevance'] > 0).astype('float32')\nHh['gain'] = np.array([0, 1, 3, 7], dtype='float32')[Hh['relevance'].values]\nHh['exp_clk'] = Hh.groupby(['regime', 'pb'])['clk'].transform('mean').astype('float32')\nHh['exp_gain'] = Hh.groupby(['regime', 'pb'])['gain'].transform('mean').astype('float32')\nprint('окна:', [c.strftime('%m-%d') for c in cutoffs], '| строк в истории:', len(Hh))\nprint(Hh.groupby('regime')[['clk', 'exp_clk']].mean().round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:53.21838Z","iopub.execute_input":"2026-10-19T07:19:53.218683Z","iopub.status.idle":"2026-10-19T07:19:54.822422Z","shell.execute_reply.started":"2026-10-19T07:19:53.21865Z","shell.execute_reply":"2026-10-19T07:19:54.821542Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## A\n**Привлекательность товара** - сглаженная доля кликов и гейна товара с поправкой на позицию, число прошлых показов (3 признака).","metadata":{}},{"cell_type":"code","source":"# [INIT-HISTORY]\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in d.columns if c.startswith('h_')], inplace=True)\n\nparts = []\nfor c in cutoffs:\n    h = Hh[Hh['date'] < c]\n    s = h.groupby('product_id').agg(n=('clk', 'size'), c=('clk', 'sum'), e=('exp_clk', 'sum'),\n                                    g=('gain', 'sum'), eg=('exp_gain', 'sum')).reset_index()\n    s['cutoff'] = c\n    parts.append(s)\nps = pd.concat(parts, ignore_index=True)\nps['h_prod_logn'] = np.log1p(ps['n'])\nps['h_prod_coe_click'] = (ps['c'] + K_PROD) / (ps['e'] + K_PROD)\nps['h_prod_coe_gain'] = (ps['g'] + K_PROD) / (ps['eg'] + K_PROD)\nps = ps.rename(columns={'c': 'h_prod_c', 'e': 'h_prod_e'})\n\ncols_A = ['h_prod_logn', 'h_prod_coe_click', 'h_prod_coe_gain']\nkeep = ['product_id', 'cutoff', 'h_prod_c', 'h_prod_e'] + cols_A\nn_before = (len(trn), len(van), len(test_f))\ntrn, van, test_f = [d.merge(ps[keep], on=['product_id', 'cutoff'], how='left') for d in (trn, van, test_f)]\nassert (len(trn), len(van), len(test_f)) == n_before\nfor d in (trn, van, test_f):\n    d[['h_prod_logn', 'h_prod_c', 'h_prod_e']] = d[['h_prod_logn', 'h_prod_c', 'h_prod_e']].fillna(0)\n    d[['h_prod_coe_click', 'h_prod_coe_gain']] = d[['h_prod_coe_click', 'h_prod_coe_gain']].fillna(1.0)\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('доля строк с историей товара: trn', round((trn['h_prod_logn'] > 0).mean(), 3),\n      '| van', round((van['h_prod_logn'] > 0).mean(), 3), '| test', round((test_f['h_prod_logn'] > 0).mean(), 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:19:54.823718Z","iopub.execute_input":"2026-10-19T07:19:54.824026Z","iopub.status.idle":"2026-10-19T07:20:01.133218Z","shell.execute_reply.started":"2026-10-19T07:19:54.824Z","shell.execute_reply":"2026-10-19T07:20:01.132083Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дают ли признаки привлекательности товара прирост над raw12\n# r_res = {'raw12': train_on(trn, FEAT_ALL), 'raw12+A': train_on(trn, FEAT_ALL + cols_A)}\n# display(compare(r_res, 'raw12').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:01.13461Z","iopub.execute_input":"2026-10-19T07:20:01.1349Z","iopub.status.idle":"2026-10-19T07:20:01.139109Z","shell.execute_reply.started":"2026-10-19T07:20:01.134876Z","shell.execute_reply":"2026-10-19T07:20:01.138293Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] NDCG при ранжировании только по историческому признаку (сравнить с -position ≈ 0.48 и random ≈ 0.40)\n# rng = np.random.default_rng(SEED)\n# print('доля строк с историей товара: trn', round((trn['h_prod_logn'] > 0).mean(), 3),\n#       '| van', round((van['h_prod_logn'] > 0).mean(), 3), '| test', round((test_f['h_prod_logn'] > 0).mean(), 3))\n# for f in cols_A:\n#     v = van[f].values + 1e-9 * rng.random(len(van))\n#     print(f'{f:20s} NDCG только по нему:', round(sess_ndcg(van['sid'], van['relevance'], v).mean(), 4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:01.14018Z","iopub.execute_input":"2026-10-19T07:20:01.14044Z","iopub.status.idle":"2026-10-19T07:20:01.162078Z","shell.execute_reply.started":"2026-10-19T07:20:01.140418Z","shell.execute_reply":"2026-10-19T07:20:01.16079Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Как падает польза группы A при старении истории\n# def group_A_features(frame, cutoff):\n#     h = Hh[Hh['date'] < cutoff]\n#     s = h.groupby('product_id').agg(n=('clk', 'size'), c=('clk', 'sum'), e=('exp_clk', 'sum'),\n#                                     g=('gain', 'sum'), eg=('exp_gain', 'sum')).reset_index()\n#     s['h_prod_logn'] = np.log1p(s['n'])\n#     s['h_prod_coe_click'] = (s['c'] + K_PROD) / (s['e'] + K_PROD)\n#     s['h_prod_coe_gain'] = (s['g'] + K_PROD) / (s['eg'] + K_PROD)\n#     out = frame.drop(columns=[c for c in cols_A + ['h_prod_c', 'h_prod_e'] if c in frame.columns])\n#     out = out.merge(s[['product_id'] + cols_A], on='product_id', how='left')\n#     out['h_prod_logn'] = out['h_prod_logn'].fillna(0)\n#     out[['h_prod_coe_click', 'h_prod_coe_gain']] = out[['h_prod_coe_click', 'h_prod_coe_gain']].fillna(1.0)\n#     assert len(out) == len(frame) and (out['sid'].values == frame['sid'].values).all()\n#     return out\n\n# def fit_models(feats, seeds=(1, 2, 3), n_iter=200):\n#     grp = trn.groupby('sid', sort=False).size().values\n#     return [lgb.train(dict(params_cmp, seed=sd, bagging_seed=sd, feature_fraction_seed=sd),\n#                       lgb.Dataset(prep(trn, feats), trn['relevance'], group=grp), n_iter) for sd in seeds]\n\n# def eval_models(models, feats, frame):\n#     per = [sess_ndcg(frame['sid'], frame['relevance'], m.predict(prep(frame, feats))) for m in models]\n#     return pd.concat(per, axis=1).mean(axis=1)\n\n# r_m_raw, r_m_A = fit_models(FEAT_ALL), fit_models(FEAT_ALL + cols_A)\n# r_base = eval_models(r_m_raw, FEAT_ALL, van)\n\n# r_versions = {'окна по 14 дней': van,\n#               'заморозка 06-02': group_A_features(van, VALID_START),\n#               'заморозка 05-05': group_A_features(van, VALID_START - pd.Timedelta(days=28))}\n# r_lag = {'окна по 14 дней': (van['sdate'] - van['cutoff']).dt.days.mean(),\n#          'заморозка 06-02': (van['sdate'] - VALID_START).dt.days.mean(),\n#          'заморозка 05-05': (van['sdate'] - (VALID_START - pd.Timedelta(days=28))).dt.days.mean()}\n# rows = []\n# for name, frame in r_versions.items():\n#     ps_A = eval_models(r_m_A, FEAT_ALL + cols_A, frame)\n#     dl = ps_A - r_base\n#     rows.append({'история': name, 'средний возраст, дней': round(r_lag[name], 1), 'raw12': r_base.mean(),\n#                  'raw12+A': ps_A.mean(), 'diff': dl.mean(), '±2SE': 2 * dl.std() / np.sqrt(len(dl))})\n# display(pd.DataFrame(rows).round(4))\n# print('test: средний возраст истории, дней:', round((test_f['date'] - TEST_START).dt.days.mean(), 1),\n#       '| максимум:', (test_f['date'] - TEST_START).dt.days.max())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:01.163627Z","iopub.execute_input":"2026-10-19T07:20:01.164469Z","iopub.status.idle":"2026-10-19T07:20:01.202821Z","shell.execute_reply.started":"2026-10-19T07:20:01.164359Z","shell.execute_reply":"2026-10-19T07:20:01.201804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Помогает ли обучение на истории разной свежести и признак её возраста\n# LAGS = [0, 14, 28, 42]\n# FA = FEAT_ALL + cols_A\n\n# def group_A_by_cutoff(frame, cut):\n#     \"\"\"Признаки группы A с историей, замороженной на своей дате для каждой строки (cut: массив дат длины frame).\"\"\"\n#     out = frame.drop(columns=[c for c in cols_A + ['h_prod_c', 'h_prod_e'] if c in frame.columns]).copy()\n#     out['_cut'] = np.asarray(cut)\n#     parts = []\n#     for c in sorted(out['_cut'].unique()):\n#         h = Hh[Hh['date'] < c]\n#         s = h.groupby('product_id').agg(n=('clk', 'size'), c=('clk', 'sum'), e=('exp_clk', 'sum'),\n#                                         g=('gain', 'sum'), eg=('exp_gain', 'sum')).reset_index()\n#         s['h_prod_logn'] = np.log1p(s['n'])\n#         s['h_prod_coe_click'] = (s['c'] + K_PROD) / (s['e'] + K_PROD)\n#         s['h_prod_coe_gain'] = (s['g'] + K_PROD) / (s['eg'] + K_PROD)\n#         s['_cut'] = c\n#         parts.append(s[['product_id', '_cut'] + cols_A])\n#     n0 = len(out)\n#     out = out.merge(pd.concat(parts, ignore_index=True), on=['product_id', '_cut'], how='left')\n#     assert len(out) == n0 and (out['sid'].values == frame['sid'].values).all()\n#     out['h_prod_logn'] = out['h_prod_logn'].fillna(0)\n#     out[['h_prod_coe_click', 'h_prod_coe_gain']] = out[['h_prod_coe_click', 'h_prod_coe_gain']].fillna(1.0)\n#     out['h_age_days'] = (out['sdate'] - out['_cut']).dt.days\n#     return out.drop(columns='_cut')\n\n# r_aug = pd.concat([group_A_by_cutoff(trn, trn['cutoff'] - pd.Timedelta(days=L)) for L in LAGS], ignore_index=True)\n# assert len(r_aug) == len(trn) * len(LAGS)\n# r_blocks = r_aug['sid'].values.reshape(-1, 20)\n# assert (r_blocks == r_blocks[:, :1]).all()\n# r_groups = [20] * (len(r_aug) // 20)\n# print('возраст истории в обучении, дней:', r_aug['h_age_days'].describe().round(1)[['mean', 'min', 'max']].to_dict())\n\n# def fit_aug(feats, seeds=(1, 2, 3), n_iter=200):\n#     return [lgb.train(dict(params_cmp, seed=sd, bagging_seed=sd, feature_fraction_seed=sd),\n#                       lgb.Dataset(prep(r_aug, feats), r_aug['relevance'], group=r_groups), n_iter) for sd in seeds]\n\n# r_m_aug = fit_aug(FA)\n# r_m_aug_age = fit_aug(FA + ['h_age_days'])\n\n# r_cuts = {'окна по 14 дней': van['cutoff'],\n#           'заморозка 06-02': pd.Series(VALID_START, index=van.index),\n#           'заморозка 05-05': pd.Series(VALID_START - pd.Timedelta(days=28), index=van.index)}\n# r_vv = {k: group_A_by_cutoff(van, c) for k, c in r_cuts.items()}\n\n# rows, r_ps = [], {}\n# for ver, frame in r_vv.items():\n#     age = frame['h_age_days'].mean()\n#     r_ps[(ver, 'A, обучена на свежей истории')] = eval_models(r_m_A, FA, frame)\n#     r_ps[(ver, 'A, обучена с лагами')] = eval_models(r_m_aug, FA, frame)\n#     r_ps[(ver, 'A + возраст, с лагами')] = eval_models(r_m_aug_age, FA + ['h_age_days'], frame)\n#     for model in ['A, обучена на свежей истории', 'A, обучена с лагами', 'A + возраст, с лагами']:\n#         ps = r_ps[(ver, model)]\n#         d_raw = ps - r_base\n#         d_fresh = ps - r_ps[(ver, 'A, обучена на свежей истории')]\n#         rows.append({'история': ver, 'возраст, дней': round(age, 1), 'модель': model, 'ndcg': ps.mean(),\n#                      'diff_vs_raw12': d_raw.mean(), '±2SE_raw': 2 * d_raw.std() / np.sqrt(len(d_raw)),\n#                      'diff_vs_fresh_model': d_fresh.mean(), '±2SE_fresh': 2 * d_fresh.std() / np.sqrt(len(d_fresh))})\n# display(pd.DataFrame(rows).round(4))\n# print('возраст истории в test, дней: среднее', round((test_f['date'] - TEST_START).dt.days.mean(), 1),\n#       '| максимум', (test_f['date'] - TEST_START).dt.days.max())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:01.203948Z","iopub.execute_input":"2026-10-19T07:20:01.204229Z","iopub.status.idle":"2026-10-19T07:20:01.242025Z","shell.execute_reply.started":"2026-10-19T07:20:01.204206Z","shell.execute_reply":"2026-10-19T07:20:01.240807Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## B \n**Тип, бренд, сжатие товара к типу** - сглаженная привлекательность типа и бренда, оценка товара, подтянутая к типу (3 признака).","metadata":{}},{"cell_type":"code","source":"# [INIT-HISTORY]\nK_GROUP = 50\ncols_B = ['h_type_coe_click', 'h_brand_coe_click', 'h_prod_coe_click_shrunk']\nneed_cols(['h_prod_c', 'h_prod_e'], ('trn', 'van', 'test_f'))\nfor d in (Hh, trn, van, test_f):\n    d.drop(columns=[c for c in ['type_code', 'brand_code'] + cols_B if c in d.columns], inplace=True)\n\nn_before = (len(trn), len(van), len(test_f))\ncodes = prod[['product_id', 'type_code', 'brand_code']]\nHh, trn, van, test_f = [d.merge(codes, on='product_id', how='left') for d in (Hh, trn, van, test_f)]\nfor d in (Hh, trn, van, test_f):\n    d[['type_code', 'brand_code']] = d[['type_code', 'brand_code']].fillna(-1).astype('int32')\n\npt, pb_ = [], []\nfor c in cutoffs:\n    h = Hh[Hh['date'] < c]\n    st = h.groupby('type_code').agg(c=('clk', 'sum'), e=('exp_clk', 'sum')).reset_index(); st['cutoff'] = c; pt.append(st)\n    sb = h.groupby('brand_code').agg(c=('clk', 'sum'), e=('exp_clk', 'sum')).reset_index(); sb['cutoff'] = c; pb_.append(sb)\nst, sb = pd.concat(pt, ignore_index=True), pd.concat(pb_, ignore_index=True)\nst['h_type_coe_click'] = (st['c'] + K_GROUP) / (st['e'] + K_GROUP)\nsb['h_brand_coe_click'] = (sb['c'] + K_GROUP) / (sb['e'] + K_GROUP)\n\ntrn, van, test_f = [d.merge(st[['type_code', 'cutoff', 'h_type_coe_click']], on=['type_code', 'cutoff'], how='left')\n                     .merge(sb[['brand_code', 'cutoff', 'h_brand_coe_click']], on=['brand_code', 'cutoff'], how='left')\n                    for d in (trn, van, test_f)]\nassert (len(trn), len(van), len(test_f)) == n_before\nfor d in (trn, van, test_f):\n    d[['h_type_coe_click', 'h_brand_coe_click']] = d[['h_type_coe_click', 'h_brand_coe_click']].fillna(1.0)\n    d['h_prod_coe_click_shrunk'] = (d['h_prod_c'] + K_PROD * d['h_type_coe_click']) / (d['h_prod_e'] + K_PROD)\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('группа B готова:', cols_B)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:01.243409Z","iopub.execute_input":"2026-10-19T07:20:01.243776Z","iopub.status.idle":"2026-10-19T07:20:08.60773Z","shell.execute_reply.started":"2026-10-19T07:20:01.243739Z","shell.execute_reply":"2026-10-19T07:20:08.606815Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дают ли тип, бренд и сжатие прирост над группой A\n# rng = np.random.default_rng(SEED)\n# for f in cols_B:\n#     v = van[f].values + 1e-9 * rng.random(len(van))\n#     print(f'{f:26s} NDCG только по нему:', round(sess_ndcg(van['sid'], van['relevance'], v).mean(), 4))\n# r_res = {'A': train_on(trn, FEAT_ALL + cols_A), 'A+B': train_on(trn, FEAT_ALL + cols_A + cols_B)}\n# display(compare(r_res, 'A').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:08.609138Z","iopub.execute_input":"2026-10-19T07:20:08.609495Z","iopub.status.idle":"2026-10-19T07:20:08.614722Z","shell.execute_reply.started":"2026-10-19T07:20:08.609463Z","shell.execute_reply":"2026-10-19T07:20:08.613397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дрейф цен: отношение цены к медиане за 28 дней до заморозки истории\n# WIN = pd.Timedelta(days=28)\n# r_Hp = train[['date', 'product_id', 'price']]\n# r_hv = r_Hp[(r_Hp['date'] >= VALID_START - WIN) & (r_Hp['date'] < VALID_START)].groupby('product_id')['price'].median()\n# r_ht = r_Hp[(r_Hp['date'] >= TEST_START - WIN) & (r_Hp['date'] < TEST_START)].groupby('product_id')['price'].median()\n# r_vq = van.assign(ratio=van['price'] / van['product_id'].map(r_hv), week=(van['date'] - VALID_START).dt.days // 7)\n# r_tq = test.assign(ratio=test['price'] / test['product_id'].map(r_ht), week=(test['date'] - TEST_START).dt.days // 7)\n# for name, q in [('valid (заморозка 06-02)', r_vq), ('test (заморозка = начало test)', r_tq)]:\n#     print(name)\n#     display(q.groupby('week')['ratio'].agg(rows='size', median='median', q10=lambda x: x.quantile(.1),\n#             q90=lambda x: x.quantile(.9), share_changed=lambda x: (x.dropna() != 1).mean(),\n#             share_missing=lambda x: x.isna().mean()).round(3))\n# print('флаг скидки и размер скидки (trn):')\n# print(trn.groupby('price_is_discounted')['price_discount'].describe().round(0))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:08.616145Z","iopub.execute_input":"2026-10-19T07:20:08.616631Z","iopub.status.idle":"2026-10-19T07:20:08.661343Z","shell.execute_reply.started":"2026-10-19T07:20:08.61659Z","shell.execute_reply":"2026-10-19T07:20:08.660234Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## C \n**Цена относительно нормы товара** - цена к медиане за 28 дней, скидка минус обычная скидка товара (2 признака).","metadata":{}},{"cell_type":"code","source":"# [INIT-HISTORY]\ncols_C = ['price_to_usual', 'discount_minus_usual']\nWIN = pd.Timedelta(days=28)\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_C + ['price_usual', 'disc_usual'] if c in d.columns], inplace=True)\n\nHp = train[['date', 'product_id', 'price', 'price_discount']]\nparts = []\nfor c in cutoffs:\n    h = Hp[(Hp['date'] >= c - WIN) & (Hp['date'] < c)]\n    s = h.groupby('product_id').agg(price_usual=('price', 'median'), disc_usual=('price_discount', 'mean')).reset_index()\n    s['cutoff'] = c\n    parts.append(s)\npu = pd.concat(parts, ignore_index=True)\n\nn_before = (len(trn), len(van), len(test_f))\ntrn, van, test_f = [d.merge(pu, on=['product_id', 'cutoff'], how='left') for d in (trn, van, test_f)]\nassert (len(trn), len(van), len(test_f)) == n_before\nfor d in (trn, van, test_f):\n    d['price_to_usual'] = (d['price'] / d['price_usual']).replace([np.inf, -np.inf], np.nan)\n    d['discount_minus_usual'] = d['price_discount'] - d['disc_usual']\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('доля строк с нормой цены: trn', round(trn['price_usual'].notna().mean(), 3),\n      '| van', round(van['price_usual'].notna().mean(), 3), '| test', round(test_f['price_usual'].notna().mean(), 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:08.662727Z","iopub.execute_input":"2026-10-19T07:20:08.663095Z","iopub.status.idle":"2026-10-19T07:20:09.958845Z","shell.execute_reply.started":"2026-10-19T07:20:08.663064Z","shell.execute_reply":"2026-10-19T07:20:09.957985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Относительная цена; можно ли заменить ею абсолютную\n# ABS_PRICE_COLS = ['price', 'price_discount']\n# r_prev = FEAT_ALL + cols_A + cols_B\n# r_res = {'A+B': train_on(trn, r_prev),\n#          'A+B+C': train_on(trn, r_prev + cols_C),\n#          'A+B+C_no_abs_price': train_on(trn, [f for f in r_prev if f not in ABS_PRICE_COLS] + cols_C)}\n# display(compare(r_res, 'A+B').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:09.960281Z","iopub.execute_input":"2026-10-19T07:20:09.960835Z","iopub.status.idle":"2026-10-19T07:20:09.964953Z","shell.execute_reply.started":"2026-10-19T07:20:09.960803Z","shell.execute_reply":"2026-10-19T07:20:09.96403Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## D\n**Центрирование внутри сессии** - отклонение от среднего по сессии и процентильный ранг для признаков A, B, C.","metadata":{}},{"cell_type":"code","source":"# [INIT-HISTORY]\ncent_cols = ['h_prod_coe_click', 'h_prod_coe_gain', 'h_prod_coe_click_shrunk',\n             'h_type_coe_click', 'h_brand_coe_click', 'price_to_usual']\nrank_cols = ['h_prod_coe_click_shrunk', 'h_prod_coe_gain', 'price_to_usual']\ncols_D = [c + '_dev' for c in cent_cols] + [c + '_rk' for c in rank_cols]\nneed_cols(cent_cols, ('trn', 'van', 'test_f'))\nfor d in (trn, van, test_f):\n    for col in cent_cols:\n        d[col + '_dev'] = d[col] - d.groupby('sid')[col].transform('mean')\n    for col in rank_cols:\n        d[col + '_rk'] = d.groupby('sid')[col].rank(pct=True)\nprint('группа D готова:', len(cols_D), 'признаков')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:09.966318Z","iopub.execute_input":"2026-10-19T07:20:09.966894Z","iopub.status.idle":"2026-10-19T07:20:10.464548Z","shell.execute_reply.started":"2026-10-19T07:20:09.966845Z","shell.execute_reply":"2026-10-19T07:20:10.463488Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дают ли центрированные признаки прирост; важность признаков\n# r_prev = FEAT_ALL + cols_A + cols_B + cols_C\n# r_res = {'A+B+C': train_on(trn, r_prev), 'A+B+C+D': train_on(trn, r_prev + cols_D)}\n# display(compare(r_res, 'A+B+C').round(4))\n\n# r_m = lgb.train(dict(params_cmp, seed=1), lgb.Dataset(prep(trn, r_prev + cols_D), trn['relevance'],\n#                 group=trn.groupby('sid', sort=False).size().values), 300)\n# r_imp = pd.Series(r_m.feature_importance('gain'), index=r_prev + cols_D).sort_values(ascending=False)\n# print((r_imp / r_imp.sum()).round(4).head(20))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:10.46582Z","iopub.execute_input":"2026-10-19T07:20:10.4662Z","iopub.status.idle":"2026-10-19T07:20:10.472005Z","shell.execute_reply.started":"2026-10-19T07:20:10.466174Z","shell.execute_reply":"2026-10-19T07:20:10.470636Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 10. Гиперпараметры","metadata":{}},{"cell_type":"code","source":"# [DEF]\ndef feat_t6():\n    \"\"\"Набор признаков модели T6 (лучшая на лидерборде: 0.49792). Вызывать после INIT-HISTORY клеток.\"\"\"\n    return FEAT_ALL + cols_A + cols_B + cols_C + cols_D\n\ndef train_on_params(d, feats, extra=None, n_iter=400, ckpts=(150, 250, 400), seeds=(1, 2)):\n    \"\"\"Обучение на d с параметрами params_cmp + extra; NDCG по сессиям van на чекпоинтах, усреднено по сидам.\"\"\"\n    need_cols(feats, ('van',))\n    assert not [f for f in feats if f not in d.columns], 'в обучающем d нет части признаков'\n    grp = d.groupby('sid', sort=False).size().values\n    out = {k: [] for k in ckpts}\n    for sd in seeds:\n        p = dict(params_cmp, **(extra or {}))\n        p.update(seed=sd, bagging_seed=sd, feature_fraction_seed=sd)\n        m = lgb.train(p, lgb.Dataset(prep(d, feats), d['relevance'], group=grp), n_iter)\n        for k in ckpts:\n            out[k].append(sess_ndcg(van['sid'], van['relevance'], m.predict(prep(van, feats), num_iteration=k)))\n    return {k: pd.concat(v, axis=1).mean(axis=1) for k, v in out.items()}\n\ndef compare_ck(results, ref, sl='all'):\n    \"\"\"Таблица: NDCG и парная разность с ref по чекпоинтам, срез sl.\"\"\"\n    sids, rows = slices[sl], []\n    for name, res in results.items():\n        for k in sorted(res):\n            ps = res[k].loc[sids]; d = ps - results[ref][k].loc[sids]\n            rows.append({'variant': name, 'iters': k, 'ndcg': ps.mean(), f'diff_vs_{ref}': d.mean(),\n                         '±2SE': 2 * d.std() / np.sqrt(len(d))})\n    return pd.DataFrame(rows)\n\ndef train_and_submit_params(tag, feats, n_iter, extra=None, seeds=(1, 2, 3, 4, 5)):\n    \"\"\"Как train_and_submit, но с изменёнными параметрами модели.\"\"\"\n    need_cols(feats, ('trn', 'van', 'test_f'))\n    final = pd.concat([trn, van], ignore_index=True)\n    grp = final.groupby('sid', sort=False).size().values\n    scores = np.zeros(len(test_f))\n    for sd in seeds:\n        p = dict(params_cmp, **(extra or {}))\n        p.update(seed=sd, bagging_seed=sd, feature_fraction_seed=sd)\n        m = lgb.train(p, lgb.Dataset(prep(final, feats), final['relevance'], group=grp), n_iter)\n        scores += m.predict(prep(test_f, feats))\n    scores /= len(seeds)\n    sub = pd.DataFrame({'session_product_id': test['session_product_id'].values, 'relevance': scores})\n    assert len(sub) == len(sample_sub) and np.isfinite(sub['relevance']).all()\n    assert (sub['session_product_id'].values == sample_sub['session_product_id'].values).all()\n    os.makedirs(f'{OUT_DIR}/archive', exist_ok=True)\n    sub.to_csv(f'{OUT_DIR}/archive/{tag}.csv', index=False)\n    sub.to_csv(f'{OUT_DIR}/submission.csv', index=False)\n    print(f'{tag}: {len(feats)} признаков, {n_iter} итераций, параметры {extra}, {len(seeds)} сидов | submission.csv записан')\n    return scores\n\ndef train_and_submit_blend(tag, feature_sets, n_iter=250, seeds=(1, 2, 3, 4, 5)):\n    \"\"\"Среднее по сессии процентильных рангов скоров моделей на разных наборах признаков; writes submission.csv.\"\"\"\n    need_cols(sorted({f for fs in feature_sets for f in fs}), ('trn', 'van', 'test_f'))\n    final = pd.concat([trn, van], ignore_index=True)\n    grp = final.groupby('sid', sort=False).size().values\n    sid_test = test_f['sid'].values\n    rank_sum, raw_sum = np.zeros(len(test_f)), np.zeros(len(test_f))\n    for feats in feature_sets:\n        s = np.zeros(len(test_f))\n        for sd in seeds:\n            p = dict(params_cmp, seed=sd, bagging_seed=sd, feature_fraction_seed=sd)\n            m = lgb.train(p, lgb.Dataset(prep(final, feats), final['relevance'], group=grp), n_iter)\n            s += m.predict(prep(test_f, feats))\n        s /= len(seeds)\n        rank_sum += pd.Series(s).groupby(sid_test).rank(pct=True).values\n        raw_sum += s\n    scores = (rank_sum + 1e-6 * raw_sum) / len(feature_sets)\n    sub = pd.DataFrame({'session_product_id': test['session_product_id'].values, 'relevance': scores})\n    assert len(sub) == len(sample_sub) and np.isfinite(sub['relevance']).all()\n    assert (sub['session_product_id'].values == sample_sub['session_product_id'].values).all()\n    os.makedirs(f'{OUT_DIR}/archive', exist_ok=True)\n    sub.to_csv(f'{OUT_DIR}/archive/{tag}.csv', index=False)\n    sub.to_csv(f'{OUT_DIR}/submission.csv', index=False)\n    print(f'{tag}: {len(feature_sets)} наборов, {n_iter} итераций, {len(seeds)} сидов | '\n          f'уникальных скоров {pd.Series(scores).nunique() / len(scores):.3f} | submission.csv записан')\n    return scores","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:36:41.619231Z","iopub.execute_input":"2026-10-19T08:36:41.619661Z","iopub.status.idle":"2026-10-19T08:36:41.645436Z","shell.execute_reply.started":"2026-10-19T08:36:41.619629Z","shell.execute_reply":"2026-10-19T08:36:41.644229Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Быстрая сетка на наборе T6: база + 11 вариантов, 2 сида, чекпоинты 150/250/400\n# GRID = {'base': {},\n#         'num_leaves=15': {'num_leaves': 15}, 'num_leaves=63': {'num_leaves': 63},\n#         'min_data=50': {'min_data_in_leaf': 50}, 'min_data=300': {'min_data_in_leaf': 300},\n#         'feature_fraction=0.5': {'feature_fraction': 0.5},\n#         'lambda_l2=1': {'lambda_l2': 1}, 'lambda_l2=50': {'lambda_l2': 50},\n#         'trunc=5': {'lambdarank_truncation_level': 5}, 'trunc=10': {'lambdarank_truncation_level': 10},\n#         'gain=[0,1,2,4]': {'label_gain': [0, 1, 2, 4]}, 'gain=[0,1,3,15]': {'label_gain': [0, 1, 3, 15]}}\n# r_grid = {}\n# for name, extra in GRID.items():\n#     r_grid[name] = train_on_params(trn, feat_t6(), extra)\n#     print(f'{name:22s}', {k: round(v.loc[slices['all']].mean(), 4) for k, v in r_grid[name].items()})\n\n# r_tab = compare_ck(r_grid, 'base')\n# print('разность с base по чекпоинтам:')\n# display(r_tab.pivot(index='variant', columns='iters', values='diff_vs_base').round(4))\n# print('подробно, 250 итераций:')\n# display(r_tab[r_tab['iters'] == 250].round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:10.507099Z","iopub.execute_input":"2026-10-19T07:20:10.507378Z","iopub.status.idle":"2026-10-19T07:20:10.531021Z","shell.execute_reply.started":"2026-10-19T07:20:10.507342Z","shell.execute_reply":"2026-10-19T07:20:10.52966Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 11. Поля справочника","metadata":{}},{"cell_type":"code","source":"# # [EXP] Вариативность полей справочника внутри сессии, возраст товара, замены\n# FIELDS = ['type', 'brand', 'country', 'category_name_1', 'category_name_2', 'category_name_3',\n#           'category_name_4', 'model']\n# r_pu = prod.set_index('product_id')\n# r_f = trn[['sid', 'product_id']].copy()\n# rows = []\n# for f in FIELDS:\n#     r_f[f] = r_f['product_id'].map(r_pu[f]).astype(str)\n#     nun = r_f.groupby('sid')[f].nunique()\n#     rows.append({'field': f, 'nunique_in_prod': prod[f].nunique(),\n#                  'missing_rows_%': (r_f[f] == 'nan').mean() * 100,\n#                  'top_value_share_%': r_f[f].value_counts(normalize=True).iloc[0] * 100,\n#                  'sessions_with_variation_%': (nun > 1).mean() * 100})\n# display(pd.DataFrame(rows).round(2))\n\n# r_age = (trn['sdate'] - trn['product_id'].map(r_pu['date_of_create'])).dt.days\n# r_ab = pd.cut(r_age, [-10**5, -1, 30, 90, 365, 10**5],\n#               labels=['до создания', '0-30', '30-90', '90-365', '365+']).astype(object).fillna('нет даты')\n# display(trn.assign(age_bin=r_ab.values).groupby('age_bin').agg(\n#     rows=('relevance', 'size'), p_click=('relevance', lambda x: (x > 0).mean()),\n#     p_buy=('relevance', lambda x: (x == 3).mean())).round(4))\n# r_g = pd.DataFrame({'sid': trn['sid'].values, 'age': r_age.values}).groupby('sid')['age']\n# print('сессий с разбросом возраста товаров > 90 дней:', round(((r_g.max() - r_g.min()) > 90).mean(), 3))\n\n# r_sent = products['replaced_by'].mode()[0]\n# print('replaced_by, топ значений:'); print(products['replaced_by'].value_counts().head(5))\n# r_rep = products.drop_duplicates('product_id', keep='last')\n# r_rep = r_rep[(r_rep['replaced_by'] != r_sent) & (r_rep['replaced_by'] != r_rep['product_id'])]\n# print('товаров с заменой:', len(r_rep))\n# for name, df in [('trn', trn), ('test', test)]:\n#     print(name, 'доля строк с «заменённым» товаром:', round(df['product_id'].isin(r_rep['product_id']).mean(), 4))\n# r_ids = set(train['product_id'])\n# print('история в train: у заменяющих товаров', round(r_rep['replaced_by'].isin(r_ids).mean(), 3),\n#       '| у заменённых', round(r_rep['product_id'].isin(r_ids).mean(), 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:10.532495Z","iopub.execute_input":"2026-10-19T07:20:10.532802Z","iopub.status.idle":"2026-10-19T07:20:10.565613Z","shell.execute_reply.started":"2026-10-19T07:20:10.532773Z","shell.execute_reply":"2026-10-19T07:20:10.56461Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Ожидаемый клик для позиции × множитель группы, оценка на второй половине периода\n# r_pu = prod.set_index('product_id')\n# TEST_FIELDS = ['country', 'category_name_1', 'category_name_2', 'category_name_4', 'type', 'brand']\n# r_R = tr_s[['sid', 'date', 'product_id', 'product_position', 'relevance']].copy()\n# r_R['clk'] = (r_R['relevance'] > 0).astype('int8')\n# r_R['pb'] = np.minimum(r_R['product_position'], 100)\n# for f in TEST_FIELDS:\n#     r_R[f] = r_R['product_id'].map(r_pu[f]).astype(str)\n# r_R['product'] = r_R['product_id']\n\n# r_mid = r_R['date'].min() + (r_R['date'].max() - r_R['date'].min()) / 2\n# r_A = r_R[r_R['date'] < r_mid].copy()\n# r_B = r_R[r_R['date'] >= r_mid].copy()\n# r_B = r_B[r_B.groupby('sid')['relevance'].transform('max') > 0].reset_index(drop=True)\n# r_exp = r_A.groupby('pb')['clk'].mean()\n# r_A['exp'] = r_A['pb'].map(r_exp)\n# r_B['exp'] = r_B['pb'].map(r_exp).fillna(r_A['clk'].mean())\n\n# def key_mult(key, k=10):\n#     s = r_A.groupby(key).agg(c=('clk', 'sum'), e=('exp', 'sum')).reset_index()\n#     s['coe'] = (s['c'] + k) / (s['e'] + k)\n#     return r_B[[key]].merge(s[[key, 'coe']], on=key, how='left')['coe'].fillna(1.0).values\n\n# rng = np.random.default_rng(SEED)\n# noise = lambda n: 1e-9 * rng.random(n)\n# r_sc = {'exp(position)': r_B['exp'].values + noise(len(r_B))}\n# r_var = {}\n# for key in TEST_FIELDS + ['product']:\n#     r_sc['exp × ' + key] = r_B['exp'].values * key_mult(key) + noise(len(r_B))\n#     r_var[key] = (r_B.groupby(['sid', key]).size().groupby('sid').size() > 1).mean()\n\n# r_per = {n: sess_ndcg(r_B['sid'], r_B['relevance'], s) for n, s in r_sc.items()}\n# rows = []\n# for n, ps in r_per.items():\n#     d = ps - r_per['exp(position)']\n#     rows.append({'score': n, 'ndcg': ps.mean(), 'diff_vs_exp(position)': d.mean(),\n#                  '±2SE': 2 * d.std() / np.sqrt(len(d)), 'sessions_with_variation': r_var.get(n.replace('exp × ', ''), np.nan)})\n# print('сессий в B (непустых):', r_B['sid'].nunique())\n# display(pd.DataFrame(rows).round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:10.566921Z","iopub.execute_input":"2026-10-19T07:20:10.56727Z","iopub.status.idle":"2026-10-19T07:20:10.60325Z","shell.execute_reply.started":"2026-10-19T07:20:10.567242Z","shell.execute_reply":"2026-10-19T07:20:10.602008Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 12. R\n**Относительные цена и скидка** - глубина скидки price_discount / (price + price_discount) (price это цена после скидки), ранги и отклонения внутри сессии и внутри типа (10 признаков).","metadata":{}},{"cell_type":"code","source":"# # [EXP] Включает ли price скидку: сравниваем цену товара в состояниях «со скидкой» и «без»\n# r_t = train.loc[train['date'] >= REGIME_START, ['product_id', 'price', 'price_discount', 'price_is_discounted']]\n# print('размер скидки по флагу:'); display(r_t.groupby('price_is_discounted')['price_discount'].describe().round(0))\n# print('скидка>0 при флаге 0:', round(((r_t.price_is_discounted == 0) & (r_t.price_discount > 0)).mean(), 4),\n#       '| флаг 1 и скидка 0:', round(((r_t.price_is_discounted == 1) & (r_t.price_discount == 0)).mean(), 4),\n#       '| скидка > цены:', round((r_t.price_discount > r_t.price).mean(), 4))\n\n# r_g = r_t.groupby(['product_id', 'price_is_discounted']).agg(\n#     price=('price', 'median'), disc=('price_discount', 'median'), n=('price', 'size')).unstack()\n# r_g = r_g[(r_g[('n', 0)] >= 5) & (r_g[('n', 1)] >= 5)]\n# r_obs = r_g[('price', 1)] / r_g[('price', 0)]\n# r_aft = r_g[('price', 1)] / (r_g[('price', 1)] + r_g[('disc', 1)])\n# print('товаров в обоих состояниях:', len(r_g))\n# print('наблюдаемое отношение цен (скидка/без скидки):'); print(r_obs.describe().round(3))\n# print('ожидаемое, если price — цена ПОСЛЕ скидки:');    print(r_aft.describe().round(3))\n# print('корреляция наблюдаемого и ожидаемого:', round(r_obs.corr(r_aft), 3), '(около 1.0 → цена после скидки; наблюдаемое около 1.0 → цена до скидки)')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:10.604437Z","iopub.execute_input":"2026-10-19T07:20:10.60482Z","iopub.status.idle":"2026-10-19T07:20:10.635023Z","shell.execute_reply.started":"2026-10-19T07:20:10.604792Z","shell.execute_reply":"2026-10-19T07:20:10.633953Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-HISTORY]\ncols_R = ['disc_over_price', 'disc_over_total', 'price_pct_sess', 'price_to_sess_median', 'price_pct_in_type',\n          'disc_over_total_rk', 'disc_over_total_dev', 'available_dev', 'h_prod_logn_dev', 'h_prod_logn_rk']\nneed_cols(['h_prod_logn', 'type_code'], ('trn', 'van', 'test_f'))\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_R if c in d.columns], inplace=True)\n    price, disc = d['price'].astype('float64'), d['price_discount'].astype('float64')\n    d['disc_over_price'] = disc / price.clip(lower=1)\n    d['disc_over_total'] = disc / (price + disc).clip(lower=1)\n    d['price_pct_sess'] = price.groupby(d['sid']).rank(pct=True)\n    d['price_to_sess_median'] = price / price.groupby(d['sid']).transform('median').clip(lower=1)\n    key = [d['sid'], d['type_code']]\n    d['price_pct_in_type'] = price.groupby(key).rank(pct=True)\n    d.loc[price.groupby(key).transform('size') == 1, 'price_pct_in_type'] = 0.5\n    d['disc_over_total_rk'] = d['disc_over_total'].groupby(d['sid']).rank(pct=True)\n    d['disc_over_total_dev'] = d['disc_over_total'] - d['disc_over_total'].groupby(d['sid']).transform('mean')\n    d['available_dev'] = d['available'] - d['available'].groupby(d['sid']).transform('mean')\n    d['h_prod_logn_dev'] = d['h_prod_logn'] - d['h_prod_logn'].groupby(d['sid']).transform('mean')\n    d['h_prod_logn_rk'] = d['h_prod_logn'].groupby(d['sid']).rank(pct=True)\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('группа R готова:', len(cols_R), 'признаков;', 'пропусков в trn:', int(trn[cols_R].isna().sum().sum()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:10.636216Z","iopub.execute_input":"2026-10-19T07:20:10.636515Z","iopub.status.idle":"2026-10-19T07:20:11.375259Z","shell.execute_reply.started":"2026-10-19T07:20:10.636481Z","shell.execute_reply":"2026-10-19T07:20:11.374171Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дают ли расширенные относительные признаки прирост над T6\n# r_prev = feat_t6()\n# r_res = {'T6': train_on(trn, r_prev),\n#          'T6+R': train_on(trn, r_prev + cols_R),\n#          'T6+R_base5': train_on(trn, r_prev + cols_R[:5])}\n# display(compare(r_res, 'T6').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:11.376402Z","iopub.execute_input":"2026-10-19T07:20:11.376693Z","iopub.status.idle":"2026-10-19T07:20:11.38155Z","shell.execute_reply.started":"2026-10-19T07:20:11.376668Z","shell.execute_reply":"2026-10-19T07:20:11.380521Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [DEF]\ndef feat_t8():\n    \"\"\"T6 + группа R. Вызывать после INIT-HISTORY клеток 9.x и 12.2.\"\"\"\n    return feat_t6() + cols_R","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:22:02.17807Z","iopub.execute_input":"2026-10-19T07:22:02.178496Z","iopub.status.idle":"2026-10-19T07:22:02.184416Z","shell.execute_reply.started":"2026-10-19T07:22:02.178468Z","shell.execute_reply":"2026-10-19T07:22:02.183398Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Неглубокие деревья и больше итераций; набор T8; 2 сида; чекпоинты до 800\n# GRID2 = {'base (31 лист, lr 0.05)': {},\n#          'nl=15': {'num_leaves': 15}, 'nl=7': {'num_leaves': 7},\n#          'nl=15, lr=0.03': {'num_leaves': 15, 'learning_rate': 0.03},\n#          'nl=7, lr=0.03': {'num_leaves': 7, 'learning_rate': 0.03},\n#          'nl=15, ff=0.5': {'num_leaves': 15, 'feature_fraction': 0.5}}\n# r_g2 = {}\n# for name, extra in GRID2.items():\n#     r_g2[name] = train_on_params(trn, feat_t8(), extra, n_iter=800, ckpts=(200, 400, 600, 800))\n#     print(f'{name:26s}', {k: round(v.loc[slices['all']].mean(), 4) for k, v in r_g2[name].items()})\n# r_t = compare_ck(r_g2, 'base (31 лист, lr 0.05)')\n# display(r_t.pivot(index='variant', columns='iters', values='ndcg').round(4))\n# display(r_t.pivot(index='variant', columns='iters', values='diff_vs_base (31 лист, lr 0.05)').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:22:09.612941Z","iopub.execute_input":"2026-10-19T07:22:09.613289Z","iopub.status.idle":"2026-10-19T07:26:48.554396Z","shell.execute_reply.started":"2026-10-19T07:22:09.613261Z","shell.execute_reply":"2026-10-19T07:26:48.553488Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 13. F\n**Корзина и покупка** - раздельные ставки по товару, бренду, типу и конверсия «корзина на клик» (13 признаков).\\\n\\\n*(отклонено, просадка качества модели)*","metadata":{}},{"cell_type":"code","source":"# # [EXP] События в обучении и split-half надёжность ставок «клик, корзина, покупка» (по всей истории)\n# for name, d in [('trn', trn), ('van', van)]:\n#     print(name, '| строк по relevance:', d['relevance'].value_counts().sort_index().to_dict(),\n#           '| сессий по максимальной relevance:', d.groupby('sid')['relevance'].max().value_counts().sort_index().to_dict())\n# assert all(c in Hh.columns for c in ['type_code', 'brand_code']), 'нужна клетка 9.5 (коды типа и бренда в Hh)'\n\n# r_H = Hh[['date', 'product_id', 'type_code', 'brand_code', 'pb', 'regime', 'relevance']].copy()\n# r_H['clk'] = (r_H['relevance'] > 0).astype('float32')\n# r_H['cart'] = (r_H['relevance'] >= 2).astype('float32')\n# r_H['buy'] = (r_H['relevance'] == 3).astype('float32')\n# for t in ['clk', 'cart', 'buy']:\n#     r_H['e_' + t] = r_H.groupby(['regime', 'pb'])[t].transform('mean').astype('float32')\n# print('события во всей истории: клики', int(r_H['clk'].sum()), '| корзины+покупки', int(r_H['cart'].sum()),\n#       '| покупки', int(r_H['buy'].sum()))\n\n# r_mid = r_H['date'].min() + (r_H['date'].max() - r_H['date'].min()) / 2\n# r_A, r_B = r_H[r_H['date'] < r_mid], r_H[r_H['date'] >= r_mid]\n# E_MIN = 3.0\n# rows = []\n# for key in ['product_id', 'brand_code', 'type_code']:\n#     for t in ['clk', 'cart', 'buy']:\n#         a = r_A.groupby(key).agg(c=(t, 'sum'), e=('e_' + t, 'sum'))\n#         b = r_B.groupby(key).agg(c=(t, 'sum'), e=('e_' + t, 'sum'))\n#         j = a.join(b, lsuffix='_a', rsuffix='_b', how='inner')\n#         j = j[(j['e_a'] >= E_MIN) & (j['e_b'] >= E_MIN)]\n#         rho = (j['c_a'] / j['e_a']).corr(j['c_b'] / j['e_b'], method='spearman')\n#         rows.append({'key': key, 'target': t, 'keys_in_both': len(j), 'spearman_A_vs_B': rho,\n#                      'B_rows_covered': r_B[key].isin(a.index[a['e'] >= E_MIN]).mean()})\n# display(pd.DataFrame(rows).round(3))\n# del r_H, r_A, r_B; gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:11.383184Z","iopub.execute_input":"2026-10-19T07:20:11.383473Z","iopub.status.idle":"2026-10-19T07:20:11.413207Z","shell.execute_reply.started":"2026-10-19T07:20:11.383448Z","shell.execute_reply":"2026-10-19T07:20:11.412254Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-HISTORY]\nK_GROUP_CB, K_PROD_CART, K_PROD_BUY, K_CONV = 50, 10, 20, 20\ncols_F = ['h_type_coe_cart', 'h_type_coe_buy', 'h_brand_coe_cart', 'h_brand_coe_buy',\n          'h_prod_coe_cart_shrunk', 'h_prod_coe_buy_shrunk', 'h_prod_cart_per_click', 'h_brand_cart_per_click']\ncols_F_dev = [c + '_dev' for c in ['h_prod_coe_cart_shrunk', 'h_prod_coe_buy_shrunk', 'h_prod_cart_per_click',\n                                   'h_brand_coe_buy', 'h_brand_cart_per_click']]\nassert 'type_code' in Hh.columns, 'нужна клетка 9.5'\nneed_cols(['type_code', 'brand_code'], ('trn', 'van', 'test_f'))\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_F + cols_F_dev if c in d.columns], inplace=True)\n\nHh['cart'] = (Hh['relevance'] >= 2).astype('float32')\nHh['buy'] = (Hh['relevance'] == 3).astype('float32')\nHh['exp_cart'] = Hh.groupby(['regime', 'pb'])['cart'].transform('mean').astype('float32')\nHh['exp_buy'] = Hh.groupby(['regime', 'pb'])['buy'].transform('mean').astype('float32')\nconv_prior = Hh['cart'].sum() / Hh['clk'].sum()\nprint('корзина на клик, глобально:', round(float(conv_prior), 4))\n\nAGG = dict(k=('clk', 'sum'), c=('cart', 'sum'), ec=('exp_cart', 'sum'), b=('buy', 'sum'), eb=('exp_buy', 'sum'))\ntype_of = prod.set_index('product_id')['type_code']\npt, pb_, pp = [], [], []\nfor cut in cutoffs:\n    h = Hh[Hh['date'] < cut]\n    for key, store in (('type_code', pt), ('brand_code', pb_), ('product_id', pp)):\n        s = h.groupby(key).agg(**AGG).reset_index(); s['cutoff'] = cut; store.append(s)\nst, sb, sp = [pd.concat(x, ignore_index=True) for x in (pt, pb_, pp)]\n\nst['h_type_coe_cart'] = (st['c'] + K_GROUP_CB) / (st['ec'] + K_GROUP_CB)\nst['h_type_coe_buy'] = (st['b'] + K_GROUP_CB) / (st['eb'] + K_GROUP_CB)\nsb['h_brand_coe_cart'] = (sb['c'] + K_GROUP_CB) / (sb['ec'] + K_GROUP_CB)\nsb['h_brand_coe_buy'] = (sb['b'] + K_GROUP_CB) / (sb['eb'] + K_GROUP_CB)\nsb['h_brand_cart_per_click'] = (sb['c'] + K_CONV * conv_prior) / (sb['k'] + K_CONV)\n\nsp['type_code'] = sp['product_id'].map(type_of).fillna(-1).astype('int32')\nsp = sp.merge(st[['type_code', 'cutoff', 'h_type_coe_cart', 'h_type_coe_buy']], on=['type_code', 'cutoff'], how='left')\nsp[['h_type_coe_cart', 'h_type_coe_buy']] = sp[['h_type_coe_cart', 'h_type_coe_buy']].fillna(1.0)\nsp['h_prod_coe_cart_shrunk'] = (sp['c'] + K_PROD_CART * sp['h_type_coe_cart']) / (sp['ec'] + K_PROD_CART)\nsp['h_prod_coe_buy_shrunk'] = (sp['b'] + K_PROD_BUY * sp['h_type_coe_buy']) / (sp['eb'] + K_PROD_BUY)\nsp['h_prod_cart_per_click'] = (sp['c'] + K_CONV * conv_prior) / (sp['k'] + K_CONV)\n\nn_before = (len(trn), len(van), len(test_f))\ntrn, van, test_f = [d.merge(sp[['product_id', 'cutoff', 'h_prod_coe_cart_shrunk', 'h_prod_coe_buy_shrunk', 'h_prod_cart_per_click']],\n                            on=['product_id', 'cutoff'], how='left')\n                     .merge(st[['type_code', 'cutoff', 'h_type_coe_cart', 'h_type_coe_buy']], on=['type_code', 'cutoff'], how='left')\n                     .merge(sb[['brand_code', 'cutoff', 'h_brand_coe_cart', 'h_brand_coe_buy', 'h_brand_cart_per_click']],\n                            on=['brand_code', 'cutoff'], how='left')\n                    for d in (trn, van, test_f)]\nassert (len(trn), len(van), len(test_f)) == n_before\nfor d in (trn, van, test_f):\n    d[['h_type_coe_cart', 'h_type_coe_buy', 'h_brand_coe_cart', 'h_brand_coe_buy']] = \\\n        d[['h_type_coe_cart', 'h_type_coe_buy', 'h_brand_coe_cart', 'h_brand_coe_buy']].fillna(1.0)\n    d['h_prod_coe_cart_shrunk'] = d['h_prod_coe_cart_shrunk'].fillna(d['h_type_coe_cart'])\n    d['h_prod_coe_buy_shrunk'] = d['h_prod_coe_buy_shrunk'].fillna(d['h_type_coe_buy'])\n    d[['h_prod_cart_per_click', 'h_brand_cart_per_click']] = d[['h_prod_cart_per_click', 'h_brand_cart_per_click']].fillna(conv_prior)\n    for c in ['h_prod_coe_cart_shrunk', 'h_prod_coe_buy_shrunk', 'h_prod_cart_per_click', 'h_brand_coe_buy', 'h_brand_cart_per_click']:\n        d[c + '_dev'] = d[c] - d.groupby('sid')[c].transform('mean')\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('группа F готова:', len(cols_F), '+', len(cols_F_dev), 'признаков; пропусков:',\n      int(trn[cols_F + cols_F_dev].isna().sum().sum()))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:11.414438Z","iopub.execute_input":"2026-10-19T07:20:11.414769Z","iopub.status.idle":"2026-10-19T07:20:25.861754Z","shell.execute_reply.started":"2026-10-19T07:20:11.414735Z","shell.execute_reply":"2026-10-19T07:20:25.860846Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP]\n# r_prev = feat_t6()\n# rng = np.random.default_rng(SEED)\n# for f in ['h_prod_coe_cart_shrunk', 'h_prod_coe_buy_shrunk', 'h_prod_cart_per_click', 'h_brand_coe_buy']:\n#     v = van[f].values + 1e-9 * rng.random(len(van))\n#     print(f'{f:26s} NDCG только по нему:', round(sess_ndcg(van['sid'], van['relevance'], v).mean(), 4))\n# r_res = {'T6': train_on(trn, r_prev),\n#          'T6+F': train_on(trn, r_prev + cols_F),\n#          'T6+F+F_dev': train_on(trn, r_prev + cols_F + cols_F_dev)}\n# display(compare(r_res, 'T6').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:25.862908Z","iopub.execute_input":"2026-10-19T07:20:25.863524Z","iopub.status.idle":"2026-10-19T07:20:25.868117Z","shell.execute_reply.started":"2026-10-19T07:20:25.863495Z","shell.execute_reply":"2026-10-19T07:20:25.867077Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 14. G\n**Цена до скидки, минимум за 90 дней, возраст товара** - цена до скидки к норме, цена к минимуму за 90 дней, глубина скидки к обычной, возраст по date_of_create (10 признаков).\\\n\\\n*(отклонено, прироста нет)*","metadata":{}},{"cell_type":"code","source":"# [INIT-HISTORY]\ncols_G = ['list_price', 'list_to_usual', 'price_to_min90', 'depth_minus_usual', 'list_to_sess_median', 'age_days',\n          'price_to_min90_rk', 'price_to_min90_dev', 'depth_minus_usual_dev', 'age_rk']\nneed_cols(['h_prod_logn', 'type_code'], ('trn', 'van', 'test_f'))\nW28, W90 = pd.Timedelta(days=28), pd.Timedelta(days=90)\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_G + ['_min90', '_list_usual', '_depth_usual'] if c in d.columns], inplace=True)\n\nHq = train[['date', 'product_id', 'price', 'price_discount']].copy()\nHq['list'] = Hq['price'] + Hq['price_discount']\nHq['depth'] = Hq['price_discount'] / Hq['list'].clip(lower=1)\nHq['price_pos'] = Hq['price'].where(Hq['price'] > 0)\nprint('доля строк с price==0: train', round((train['price'] == 0).mean(), 4), '| test', round((test['price'] == 0).mean(), 4))\n\nparts = []\nfor c in cutoffs:\n    h90 = Hq[(Hq['date'] >= c - W90) & (Hq['date'] < c)]\n    h28 = h90[h90['date'] >= c - W28]\n    s90 = h90.groupby('product_id')['price_pos'].min().rename('_min90')\n    s28 = h28.groupby('product_id').agg(_list_usual=('list', 'median'), _depth_usual=('depth', 'mean'))\n    s = pd.concat([s90, s28], axis=1).reset_index(); s['cutoff'] = c\n    parts.append(s)\ngs = pd.concat(parts, ignore_index=True)\n\nn_before = (len(trn), len(van), len(test_f))\ntrn, van, test_f = [d.merge(gs, on=['product_id', 'cutoff'], how='left') for d in (trn, van, test_f)]\nassert (len(trn), len(van), len(test_f)) == n_before\ncreated = prod.set_index('product_id')['date_of_create']\nfor d in (trn, van, test_f):\n    price, disc = d['price'].astype('float64'), d['price_discount'].astype('float64')\n    d['list_price'] = price + disc\n    d['list_to_usual'] = (d['list_price'] / d['_list_usual']).replace([np.inf, -np.inf], np.nan)\n    d['price_to_min90'] = (price / d['_min90']).replace([np.inf, -np.inf], np.nan)\n    d['depth_minus_usual'] = disc / d['list_price'].clip(lower=1) - d['_depth_usual']\n    d['list_to_sess_median'] = d['list_price'] / d['list_price'].groupby(d['sid']).transform('median').clip(lower=1)\n    d['age_days'] = (d['date'] - d['product_id'].map(created)).dt.days\n    d['price_to_min90_rk'] = d['price_to_min90'].groupby(d['sid']).rank(pct=True)\n    d['price_to_min90_dev'] = d['price_to_min90'] - d['price_to_min90'].groupby(d['sid']).transform('mean')\n    d['depth_minus_usual_dev'] = d['depth_minus_usual'] - d['depth_minus_usual'].groupby(d['sid']).transform('mean')\n    d['age_rk'] = d['age_days'].groupby(d['sid']).rank(pct=True)\n    d.drop(columns=['_min90', '_list_usual', '_depth_usual'], inplace=True)\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('доля непустых значений (trn / van / test):')\nfor c in ['list_to_usual', 'price_to_min90', 'depth_minus_usual', 'age_days']:\n    print(f'{c:20s}', round(trn[c].notna().mean(), 3), round(van[c].notna().mean(), 3), round(test_f[c].notna().mean(), 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:27:46.944333Z","iopub.execute_input":"2026-10-19T07:27:46.944719Z","iopub.status.idle":"2026-10-19T07:27:49.859088Z","shell.execute_reply.started":"2026-10-19T07:27:46.944667Z","shell.execute_reply":"2026-10-19T07:27:49.858191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дают ли признаки группы G прирост над T8; одиночные признаки\n# rng = np.random.default_rng(SEED)\n# for f in ['list_to_usual', 'price_to_min90', 'depth_minus_usual', 'age_days']:\n#     v = van[f].fillna(van[f].median()).values + 1e-9 * rng.random(len(van))\n#     print(f'{f:20s} NDCG только по нему: по возрастанию {sess_ndcg(van[\"sid\"], van[\"relevance\"], -v).mean():.4f}'\n#           f' | по убыванию {sess_ndcg(van[\"sid\"], van[\"relevance\"], v).mean():.4f}   (случайный порядок около 0.403)')\n# r_prev = feat_t8()\n# r_res = {'T8': train_on(trn, r_prev),\n#          'T8+G': train_on(trn, r_prev + cols_G),\n#          'T8+G_noage': train_on(trn, r_prev + [c for c in cols_G if 'age' not in c])}\n# display(compare(r_res, 'T8').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:27:49.860537Z","iopub.execute_input":"2026-10-19T07:27:49.860853Z","iopub.status.idle":"2026-10-19T07:29:34.79228Z","shell.execute_reply.started":"2026-10-19T07:27:49.860825Z","shell.execute_reply":"2026-10-19T07:29:34.791191Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Убираем группу B или C целиком (вместе с производными в D)\n# B_ALL = cols_B + [c for c in cols_D if any(k in c for k in ('type_coe', 'brand_coe', 'shrunk'))]\n# C_ALL = cols_C + [c for c in cols_D if 'price_to_usual' in c]\n# drop = lambda cols: [f for f in feat_t8() if f not in cols]\n# r_res = {'T8': train_on(trn, feat_t8()),\n#          'T8-B': train_on(trn, drop(B_ALL)),\n#          'T8-C': train_on(trn, drop(C_ALL)),\n#          'T8-B-C': train_on(trn, drop(B_ALL + C_ALL))}\n# print('убрано признаков: B', len(B_ALL), '| C', len(C_ALL), '| всего в T8', len(feat_t8()))\n# display(compare(r_res, 'T8').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:29:34.793806Z","iopub.execute_input":"2026-10-19T07:29:34.794096Z","iopub.status.idle":"2026-10-19T07:31:39.833208Z","shell.execute_reply.started":"2026-10-19T07:29:34.794073Z","shell.execute_reply":"2026-10-19T07:31:39.83189Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 15. P\n**Ценовая история по train и test** - скользящие нормы цены по всем строкам обоих файлов, всегда свежие, без меток (12 признаков).\n\n*(отклонено, прироста нет)*","metadata":{}},{"cell_type":"code","source":"# # [EXP] Что означает price == 0 и как его доля меняется во времени\n# r_tr = train[train['date'] >= REGIME_START]\n# for name, df in [('train (новый режим)', r_tr), ('test', test)]:\n#     z = df['price'] == 0\n#     print(f'{name}: доля price==0 = {z.mean():.4f}')\n#     print('   available при price==0:', df.loc[z, 'available'].value_counts(normalize=True).round(3).to_dict(),\n#           '| при price>0:', df.loc[~z, 'available'].value_counts(normalize=True).round(3).to_dict())\n#     print('   доля price_discount>0 среди price==0:', round((df.loc[z, 'price_discount'] > 0).mean(), 3))\n# z = r_tr['price'] == 0\n# print('доля кликов (relevance>0): price==0', round((r_tr.loc[z, 'relevance'] > 0).mean(), 4),\n#       '| price>0', round((r_tr.loc[~z, 'relevance'] > 0).mean(), 4))\n# print('доля price==0 по неделям, train (новый режим):')\n# display(r_tr.groupby(r_tr['date'].dt.to_period('W').dt.start_time)['price'].apply(lambda x: (x == 0).mean()).round(4).to_frame('share_zero').T)\n# print('доля price==0 по неделям, test:')\n# display(test.groupby(test['date'].dt.to_period('W').dt.start_time)['price'].apply(lambda x: (x == 0).mean()).round(4).to_frame('share_zero').T)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:52:29.713086Z","iopub.execute_input":"2026-10-19T07:52:29.713465Z","iopub.status.idle":"2026-10-19T07:52:30.480925Z","shell.execute_reply.started":"2026-10-19T07:52:29.713434Z","shell.execute_reply":"2026-10-19T07:52:30.480045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-HISTORY]\nbase_P = ['p_price_to_min90', 'p_price_to_mean28', 'p_list_to_mean28', 'p_depth_minus_mean28']\ncols_P = base_P + [c + '_rk' for c in base_P] + [c + '_dev' for c in base_P]\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_P if c in d.columns], inplace=True)\n\nrows = pd.concat([train[['date', 'product_id', 'price', 'price_discount']],\n                  test[['date', 'product_id', 'price', 'price_discount']]], ignore_index=True)\nrows = rows[rows['price'] > 0].copy()\nrows['list'] = rows['price'] + rows['price_discount']\nrows['depth'] = rows['price_discount'] / rows['list']\ndaily = rows.groupby(['product_id', 'date']).agg(pmin=('price', 'min'), psum=('price', 'sum'), lsum=('list', 'sum'),\n                                                 dsum=('depth', 'sum'), n=('price', 'size')).reset_index()\ndel rows; gc.collect()\n\ndates = pd.date_range(daily['date'].min(), daily['date'].max())\nprod_ids = np.sort(daily['product_id'].unique())\nd_idx = (daily['date'] - dates[0]).dt.days.values\np_idx = np.searchsorted(prod_ids, daily['product_id'].values)\n\ndef grid(col, fill):\n    a = np.full((len(dates), len(prod_ids)), fill, dtype='float64')\n    a[d_idx, p_idx] = daily[col].values\n    return pd.DataFrame(a)\n\nG_N = grid('n', 0.0)\nmin90 = grid('pmin', np.nan).rolling(90, min_periods=1).min().shift(1).to_numpy()\ndef mean_prev(sum_col, window):\n    s = grid(sum_col, 0.0).rolling(window, min_periods=1).sum().shift(1).to_numpy()\n    n = G_N.rolling(window, min_periods=1).sum().shift(1).to_numpy()\n    return np.where(n > 0, s / np.maximum(n, 1), np.nan)\nmean_p28, mean_l28, mean_d28 = mean_prev('psum', 28), mean_prev('lsum', 28), mean_prev('dsum', 28)\n\ndef lookup(arr, frame):\n    di = (frame['date'] - dates[0]).dt.days.values\n    pi = np.clip(np.searchsorted(prod_ids, frame['product_id'].values), 0, len(prod_ids) - 1)\n    ok = (prod_ids[pi] == frame['product_id'].values) & (di >= 0) & (di < len(dates))\n    out = np.full(len(frame), np.nan)\n    out[ok] = arr[di[ok], pi[ok]]\n    return out\n\nfor d in (trn, van, test_f):\n    price, disc = d['price'].astype('float64'), d['price_discount'].astype('float64')\n    lst = price + disc\n    d['p_price_to_min90'] = price / lookup(min90, d)\n    d['p_price_to_mean28'] = price / lookup(mean_p28, d)\n    d['p_list_to_mean28'] = lst / lookup(mean_l28, d)\n    d['p_depth_minus_mean28'] = disc / lst.clip(lower=1) - lookup(mean_d28, d)\n    d[base_P] = d[base_P].replace([np.inf, -np.inf], np.nan)\n    for c in base_P:\n        d[c + '_rk'] = d[c].groupby(d['sid']).rank(pct=True)\n        d[c + '_dev'] = d[c] - d[c].groupby(d['sid']).transform('mean')\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\n\nprint('доля непустых значений (trn / van / test) и медиана:')\nfor c in base_P:\n    print(f'{c:22s}', round(trn[c].notna().mean(), 3), round(van[c].notna().mean(), 3), round(test_f[c].notna().mean(), 3),\n          '| медиана', round(trn[c].median(), 3), round(van[c].median(), 3), round(test_f[c].median(), 3))\ndel G_N, min90, mean_p28, mean_l28, mean_d28; gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:53:33.769187Z","iopub.execute_input":"2026-10-19T07:53:33.76954Z","iopub.status.idle":"2026-10-19T07:53:43.537714Z","shell.execute_reply.started":"2026-10-19T07:53:33.769512Z","shell.execute_reply":"2026-10-19T07:53:43.536985Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Группа P в составе T8; замена группы C группой P\n# C_ALL = cols_C + [c for c in cols_D if 'price_to_usual' in c]\n# rng = np.random.default_rng(SEED)\n# for f in base_P:\n#     v = van[f].fillna(van[f].median()).values + 1e-9 * rng.random(len(van))\n#     print(f'{f:22s} NDCG только по нему: по возрастанию {sess_ndcg(van[\"sid\"], van[\"relevance\"], -v).mean():.4f}'\n#           f' | по убыванию {sess_ndcg(van[\"sid\"], van[\"relevance\"], v).mean():.4f}   (случайный порядок около 0.403)')\n# r_prev = feat_t8()\n# r_res = {'T8': train_on(trn, r_prev),\n#          'T8+P': train_on(trn, r_prev + cols_P),\n#          'T8-C+P': train_on(trn, [f for f in r_prev if f not in C_ALL] + cols_P)}\n# display(compare(r_res, 'T8').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:53:57.028054Z","iopub.execute_input":"2026-10-19T07:53:57.028442Z","iopub.status.idle":"2026-10-19T07:55:51.789458Z","shell.execute_reply.started":"2026-10-19T07:53:57.028409Z","shell.execute_reply":"2026-10-19T07:55:51.788572Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 16. Текстовые признаки (Q | TM)\n**Q. История пар «запрос × товар / бренд / тип»** - сглаженные доли кликов пар, подтянутые к оценкам товара, бренда, типа (9 признаков).\\\n**TM. Совпадение запроса с карточкой** - доли слов бренда и типа в запросе, слова модели, доля слов запроса, объяснённая карточкой. Меняет порядок только в ~41% сессий (12 признаков).","metadata":{}},{"cell_type":"code","source":"# # [EXP] Примеры запросов и карточек; проверка идеи токенизации\n# r_pu = prod.set_index('product_id')\n# for r_sid in trn['sid'].drop_duplicates().sample(6, random_state=SEED):\n#     s = trn[trn['sid'] == r_sid]\n#     print('\\nзапрос:', s['query'].iloc[0])\n#     for pid, rel in s[['product_id', 'relevance']].head(4).itertuples(index=False):\n#         if pid in r_pu.index:\n#             r = r_pu.loc[pid]\n#             print(f'  rel={rel} | {r[\"type\"]} | {r[\"brand\"]} | {str(r[\"model\"])[:60]}')\n#         else:\n#             print(f'  rel={rel} | товара нет в справочнике')\n# r_qu = trn['query'].drop_duplicates()\n# print('\\nуникальных запросов в trn:', len(r_qu), '| среднее число слов:', round(r_qu.str.split().str.len().mean(), 2),\n#       '| доля запросов только из латиницы и цифр:', round(r_qu.str.fullmatch(r'[a-z0-9\\s\\-\\.]+', case=False).mean(), 3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:17:52.71102Z","iopub.execute_input":"2026-10-19T08:17:52.716609Z","iopub.status.idle":"2026-10-19T08:17:52.845521Z","shell.execute_reply.started":"2026-10-19T08:17:52.716462Z","shell.execute_reply":"2026-10-19T08:17:52.843871Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-HISTORY]\nK_QP, K_QB, K_QT = 3, 10, 20\ncols_Q = ['h_qp_logn', 'h_qp_coe', 'h_qb_logn', 'h_qb_coe', 'h_qt_coe',\n          'h_qp_coe_dev', 'h_qb_coe_dev', 'h_qt_coe_dev', 'h_qp_coe_rk']\nneed_cols(['h_prod_coe_click_shrunk', 'h_brand_coe_click', 'h_type_coe_click', 'type_code', 'brand_code'],\n          ('trn', 'van', 'test_f'))\nassert len(Hh) == len(train)\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_Q if c in d.columns], inplace=True)\n\nq_codes, q_uniq = pd.factorize(pd.concat([train['query'], test['query']], ignore_index=True))\nHh['q_code'] = q_codes[:len(train)].astype('int32')\nfor d in (trn, van, test_f):\n    d['q_code'] = q_uniq.get_indexer(d['query']).astype('int32')\nassert (trn['q_code'] >= 0).all() and (van['q_code'] >= 0).all() and (test_f['q_code'] >= 0).all()\n\ndef add_pair_stats(keys, tag):\n    \"\"\"Показы, клики и ожидаемые клики пары keys по истории до окна отсечения каждой строки (временные колонки _tag_*).\"\"\"\n    for f in (trn, van, test_f):\n        for col in ('n', 'c', 'e'):\n            f[f'_{tag}_{col}'] = 0.0\n    for cut in cutoffs:\n        h = Hh[Hh['date'] < cut]\n        s = h.groupby(keys).agg(n=('clk', 'size'), c=('clk', 'sum'), e=('exp_clk', 'sum'))\n        for f in (trn, van, test_f):\n            m = (f['cutoff'] == cut).values\n            if not m.any():\n                continue\n            j = f.loc[m, keys].join(s, on=keys)\n            for col in ('n', 'c', 'e'):\n                f.loc[m, f'_{tag}_{col}'] = j[col].fillna(0).values\n        del h, s; gc.collect()\n\nfor tag, keys in (('qp', ['q_code', 'product_id']), ('qb', ['q_code', 'brand_code']), ('qt', ['q_code', 'type_code'])):\n    add_pair_stats(keys, tag)\n\nprint('доля строк с историей пары (trn / van / test):')\ndisplay(pd.DataFrame({name: {k: (d[f'_{k}_n'] > 0).mean() for k in ('qp', 'qb', 'qt')}\n                      for name, d in (('trn', trn), ('van', van), ('test', test_f))}).round(3))\n\nfor d in (trn, van, test_f):\n    d['h_qp_logn'] = np.log1p(d['_qp_n'])\n    d['h_qb_logn'] = np.log1p(d['_qb_n'])\n    d['h_qp_coe'] = (d['_qp_c'] + K_QP * d['h_prod_coe_click_shrunk']) / (d['_qp_e'] + K_QP)\n    d['h_qb_coe'] = (d['_qb_c'] + K_QB * d['h_brand_coe_click']) / (d['_qb_e'] + K_QB)\n    d['h_qt_coe'] = (d['_qt_c'] + K_QT * d['h_type_coe_click']) / (d['_qt_e'] + K_QT)\n    for c in ('h_qp_coe', 'h_qb_coe', 'h_qt_coe'):\n        d[c + '_dev'] = d[c] - d.groupby('sid')[c].transform('mean')\n    d['h_qp_coe_rk'] = d.groupby('sid')['h_qp_coe'].rank(pct=True)\n    d.drop(columns=[c for c in d.columns if c.startswith('_q')], inplace=True)\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('группа Q готова:', len(cols_Q), 'признаков')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:19:04.503416Z","iopub.execute_input":"2026-10-19T08:19:04.503998Z","iopub.status.idle":"2026-10-19T08:19:53.695957Z","shell.execute_reply.started":"2026-10-19T08:19:04.503963Z","shell.execute_reply":"2026-10-19T08:19:53.694769Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-QUERY]\nimport re\nTOK = re.compile(r'[a-zа-яё0-9]+')\ndef toks(s):\n    return TOK.findall(str(s).lower()) if pd.notna(s) else []\n\nbase_TM = ['tm_brand_share', 'tm_type_share', 'tm_type_share_exact', 'tm_model_hits', 'tm_q_cover', 'tm_card_hits']\ncols_TM = base_TM + [c + '_dev' for c in base_TM]\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_TM if c in d.columns], inplace=True)\n\ncard = {}\nfor pid, t, b, m in prod[['product_id', 'type', 'brand', 'model']].itertuples(index=False):\n    T, B, M = toks(t), toks(b), toks(m)\n    card[pid] = (set(T), {x[:5] for x in T}, {x[:5] for x in B}, {x[:5] for x in M})\nempty_card = (set(), set(), set(), set())\n\npairs = pd.concat([d[['query', 'product_id']] for d in (trn, van, test_f)], ignore_index=True)\npairs = pairs.drop_duplicates().reset_index(drop=True)\nqtok, out = {}, []\nfor q, pid in pairs.itertuples(index=False):\n    if q not in qtok:\n        Q = toks(q)\n        qtok[q] = (set(Q), {x[:5] for x in Q})\n    Qe, Qs = qtok[q]\n    Te, Ts, Bs, Ms = card.get(pid, empty_card)\n    hit = Qs & (Ts | Bs | Ms)\n    out.append((len(Bs & Qs) / len(Bs) if Bs else np.nan,\n                len(Ts & Qs) / len(Ts) if Ts else np.nan,\n                len(Te & Qe) / len(Te) if Te else np.nan,\n                len(Ms & Qs),\n                len(hit) / len(Qs) if Qs else np.nan,\n                len(hit)))\npairs[base_TM] = pd.DataFrame(out, columns=base_TM).values\n\nn_before = (len(trn), len(van), len(test_f))\ntrn, van, test_f = [d.merge(pairs, on=['query', 'product_id'], how='left') for d in (trn, van, test_f)]\nassert (len(trn), len(van), len(test_f)) == n_before\nfor d in (trn, van, test_f):\n    for c in base_TM:\n        d[c + '_dev'] = d[c] - d.groupby('sid')[c].transform('mean')\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\n\nfor name, d in (('trn', trn), ('van', van), ('test', test_f)):\n    print(f'{name}: card_hits>0 {(d[\"tm_card_hits\"] > 0).mean():.3f} | brand_share>0 {(d[\"tm_brand_share\"] > 0).mean():.3f}'\n          f' | type_share_exact>0 {(d[\"tm_type_share_exact\"] > 0).mean():.3f}'\n          f' | сессий, где card_hits различается между товарами: {(d.groupby(\"sid\")[\"tm_card_hits\"].nunique() > 1).mean():.3f}')\nprint('группа TM готова:', len(cols_TM), 'признаков')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:22:19.541559Z","iopub.execute_input":"2026-10-19T08:22:19.54263Z","iopub.status.idle":"2026-10-19T08:22:22.683593Z","shell.execute_reply.started":"2026-10-19T08:22:19.542596Z","shell.execute_reply":"2026-10-19T08:22:22.682629Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Дают ли группы Q и TM прирост над T8; одиночные признаки\n# rng = np.random.default_rng(SEED)\n# for f in ['h_qp_coe', 'h_qb_coe', 'h_qt_coe', 'tm_card_hits', 'tm_type_share', 'tm_brand_share']:\n#     v = van[f].fillna(van[f].median()).values + 1e-9 * rng.random(len(van))\n#     print(f'{f:16s} NDCG только по нему: по возрастанию {sess_ndcg(van[\"sid\"], van[\"relevance\"], -v).mean():.4f}'\n#           f' | по убыванию {sess_ndcg(van[\"sid\"], van[\"relevance\"], v).mean():.4f}   (случайный порядок около 0.403)')\n# r_prev = feat_t8()\n# r_res = {'T8': train_on(trn, r_prev),\n#          'T8+Q': train_on(trn, r_prev + cols_Q),\n#          'T8+TM': train_on(trn, r_prev + cols_TM),\n#          'T8+Q+TM': train_on(trn, r_prev + cols_Q + cols_TM)}\n# display(compare(r_res, 'T8').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:23:19.490984Z","iopub.execute_input":"2026-10-19T08:23:19.491445Z","iopub.status.idle":"2026-10-19T08:25:37.186367Z","shell.execute_reply.started":"2026-10-19T08:23:19.491409Z","shell.execute_reply":"2026-10-19T08:25:37.185474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 17. Нормализация запроса\n**Q2 и TM2. Нормализация запроса (9 + 12 признаков)** - транслитерация кириллицы, исправление раскладки и порядково-независимый ключ запроса.","metadata":{}},{"cell_type":"code","source":"# [DEF]\nimport re\nfrom collections import Counter\n\nEN_TO_RU = dict(zip(\"qwertyuiop[]asdfghjkl;'zxcvbnm,.`\", \"йцукенгшщзхъфывапролджэячсмитьбюё\"))\nRU_TO_LAT = {'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'e', 'ж': 'zh', 'з': 'z', 'и': 'i',\n             'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm', 'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't',\n             'у': 'u', 'ф': 'f', 'х': 'h', 'ц': 'c', 'ч': 'ch', 'ш': 'sh', 'щ': 'sh', 'ъ': '', 'ы': 'y', 'ь': '',\n             'э': 'e', 'ю': 'yu', 'я': 'ya'}\nWORD = re.compile(r'[a-zа-яё0-9]+')\nHAS_LAT = re.compile(r\"[a-z\\[\\];',.`]\")\nIS_CYR = re.compile(r'[а-яё]')\n\ndef fix_layout(s):\n    return ''.join(EN_TO_RU.get(ch, ch) for ch in s.lower())\n\ndef translit(s):\n    return ''.join(RU_TO_LAT.get(ch, ch) for ch in s.lower())\n\ndef clean_tokens(s):\n    return WORD.findall(str(s).lower())\n\ndef fixed_if_wrong_layout(w):\n    \"\"\"Исправленное слово, если w похоже на русское слово в латинской раскладке; иначе None. Опирается на CARD_VOCAB и CYR_VOCAB.\"\"\"\n    w = w.lower()\n    if not HAS_LAT.search(w) or any(t in CARD_VOCAB for t in clean_tokens(w)):\n        return None\n    f = fix_layout(w)\n    ft = clean_tokens(f)\n    return f if ft and all(len(t) >= 3 and t in CYR_VOCAB for t in ft) else None\n\ndef norm_tokens(q):\n    \"\"\"Токены запроса после исправления раскладки и транслитерации кириллицы в латиницу.\"\"\"\n    out = []\n    for w in str(q).lower().split():\n        f = fixed_if_wrong_layout(w)\n        for t in clean_tokens(f if f else w):\n            out.append(translit(t) if IS_CYR.search(t) else t)\n    return out\n\nassert fix_layout(',tkbpyf') == 'белизна' and translit('самсунг') == 'samsung'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:40:43.427732Z","iopub.execute_input":"2026-10-19T08:40:43.428109Z","iopub.status.idle":"2026-10-19T08:40:43.440806Z","shell.execute_reply.started":"2026-10-19T08:40:43.428079Z","shell.execute_reply":"2026-10-19T08:40:43.439887Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-QUERY]\nCARD_VOCAB, BRAND_LAT = set(), set()\nfor col in ('type', 'brand', 'model'):\n    for v in prod[col].dropna().astype(str).unique():\n        toks_ = clean_tokens(v)\n        CARD_VOCAB.update(toks_)\n        if col == 'brand':\n            BRAND_LAT.update(t for t in toks_ if re.fullmatch(r'[a-z0-9]+', t) and len(t) >= 3 and not t.isdigit())\nCARD_TL = {translit(t) if IS_CYR.search(t) else t for t in CARD_VOCAB}\n\nqcount = pd.concat([train['query'], test['query']], ignore_index=True).value_counts()\nuq = qcount.reset_index(); uq.columns = ['query', 'rows']\ntok_w = Counter()\nfor q, n in zip(uq['query'], uq['rows']):\n    for t in set(clean_tokens(q)):\n        tok_w[t] += n\nCYR_VOCAB = {t for t in CARD_VOCAB if re.fullmatch(r'[а-яё]+', t) and len(t) >= 3}\nCYR_VOCAB |= {t for t, n in tok_w.items() if n >= 100 and re.fullmatch(r'[а-яё]+', t) and len(t) >= 3}\nprint('уникальных запросов:', len(uq), '| слов в словаре карточек:', len(CARD_VOCAB),\n      '| русских слов в словаре:', len(CYR_VOCAB), '| латинских брендов:', len(BRAND_LAT))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:41:09.019875Z","iopub.execute_input":"2026-10-19T08:41:09.02092Z","iopub.status.idle":"2026-10-19T08:41:10.479387Z","shell.execute_reply.started":"2026-10-19T08:41:09.020873Z","shell.execute_reply":"2026-10-19T08:41:10.478412Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Диагностика: доля запросов с ошибочной раскладкой и бренды, написанные кириллицей\n# flag = {}\n# for q in uq['query']:\n#     pairs_ = [(w, fixed_if_wrong_layout(w)) for w in str(q).lower().split()]\n#     flag[q] = [(w, f) for w, f in pairs_ if f]\n# uq['n_flag'] = uq['query'].map(lambda q: len(flag[q]))\n# print(f'запросов с ошибкой раскладки: {(uq.n_flag > 0).mean():.3%} (по строкам: {(uq.rows * (uq.n_flag > 0)).sum() / uq.rows.sum():.3%})')\n# ex = Counter()\n# for q, n in zip(uq['query'], uq['rows']):\n#     for w, f in flag[q]:\n#         ex[(w, f)] += n\n# print('частые исправления (проверьте глазами, что они разумны):')\n# display(pd.DataFrame([(w, f, n) for (w, f), n in ex.most_common(25)], columns=['было', 'стало', 'строк']))\n\n# brand_cyr = Counter()\n# for q, n in zip(uq['query'], uq['rows']):\n#     for t in set(clean_tokens(q)):\n#         if len(t) >= 3 and IS_CYR.search(t) and t not in CARD_VOCAB and translit(t) in BRAND_LAT:\n#             brand_cyr[(t, translit(t))] += n\n# print(f'слов, похожих на латинский бренд, написанный кириллицей: {len(brand_cyr)} разных')\n# display(pd.DataFrame([(a, b, n) for (a, b), n in brand_cyr.most_common(15)], columns=['в запросе', 'как бренд', 'строк']))\n\n# def token_cov(fn, vocab):\n#     hit = tot = 0\n#     for q, n in zip(uq['query'], uq['rows']):\n#         for t in set(fn(q)):\n#             if len(t) >= 3 and not t.isdigit():\n#                 tot += n; hit += n * (t in vocab)\n#     return hit / tot\n# tl_only = lambda q: [translit(t) if IS_CYR.search(t) else t for t in clean_tokens(q)]\n# print('доля слов запроса, найденных в карточках (по строкам):')\n# print(f'  сырые слова:                       {token_cov(clean_tokens, CARD_VOCAB):.3f}')\n# print(f'  + транслитерация с обеих сторон:   {token_cov(tl_only, CARD_TL):.3f}')\n# print(f'  + исправление раскладки:           {token_cov(norm_tokens, CARD_TL):.3f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:41:26.119228Z","iopub.execute_input":"2026-10-19T08:41:26.119595Z","iopub.status.idle":"2026-10-19T08:41:29.236741Z","shell.execute_reply.started":"2026-10-19T08:41:26.119567Z","shell.execute_reply":"2026-10-19T08:41:29.235852Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Сколько запросов склеивается нормализацией и растёт ли покрытие test историей train\n# keymap = {q: ' '.join(sorted(set(norm_tokens(q)))) for q in uq['query']}\n# tr_u, te_u = train['query'].unique(), test['query'].unique()\n# print('уникальных запросов в train: сырых', len(tr_u), '| после нормализации', len({keymap[q] for q in tr_u}))\n# print('уникальных запросов в test:  сырых', len(te_u), '| после нормализации', len({keymap[q] for q in te_u}))\n# raw_cov = test['query'].isin(set(tr_u)).mean()\n# norm_cov = test['query'].map(keymap).isin({keymap[q] for q in tr_u}).mean()\n# print(f'доля строк test с известным запросом: сырой {raw_cov:.3f} | нормализованный {norm_cov:.3f}')\n# raw_cov_u = np.mean([q in set(tr_u) for q in te_u])\n# norm_cov_u = np.mean([keymap[q] in {keymap[x] for x in tr_u} for q in te_u])\n# print(f'доля уникальных запросов test с историей: сырой {raw_cov_u:.3f} | нормализованный {norm_cov_u:.3f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:42:09.887598Z","iopub.execute_input":"2026-10-19T08:42:09.88796Z","iopub.status.idle":"2026-10-19T08:46:07.807217Z","shell.execute_reply.started":"2026-10-19T08:42:09.887932Z","shell.execute_reply":"2026-10-19T08:46:07.806249Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [DEF]\ndef norm_key(q):\n    \"\"\"Нормализованный ключ запроса: транслитерация и исправление раскладки, порядок слов не важен.\"\"\"\n    return ' '.join(sorted(set(norm_tokens(q))))\n\ndef pair_stats(keys, tag):\n    \"\"\"Показы, клики и ожидаемые клики пары keys по истории до окна отсечения; пишет _{tag}_n/_c/_e в trn, van, test_f.\"\"\"\n    for f in (trn, van, test_f):\n        for col in ('n', 'c', 'e'):\n            f[f'_{tag}_{col}'] = 0.0\n    for cut in cutoffs:\n        h = Hh[Hh['date'] < cut]\n        s = h.groupby(keys).agg(n=('clk', 'size'), c=('clk', 'sum'), e=('exp_clk', 'sum'))\n        for f in (trn, van, test_f):\n            m = (f['cutoff'] == cut).values\n            if not m.any():\n                continue\n            j = f.loc[m, keys].join(s, on=keys)\n            for col in ('n', 'c', 'e'):\n                f.loc[m, f'_{tag}_{col}'] = j[col].fillna(0).values\n        del h, s; gc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T09:36:31.965614Z","iopub.execute_input":"2026-10-19T09:36:31.966295Z","iopub.status.idle":"2026-10-19T09:36:31.980604Z","shell.execute_reply.started":"2026-10-19T09:36:31.966261Z","shell.execute_reply":"2026-10-19T09:36:31.979125Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-QUERY]\nbase_TM2 = ['tm2_brand_share', 'tm2_type_share', 'tm2_type_share_exact', 'tm2_model_hits', 'tm2_q_cover', 'tm2_card_hits']\ncols_TM2 = base_TM2 + [c + '_dev' for c in base_TM2]\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_TM2 if c in d.columns], inplace=True)\n\ncard2 = {}\nfor pid, t, b, m in prod[['product_id', 'type', 'brand', 'model']].itertuples(index=False):\n    T, B, M = norm_tokens(t), norm_tokens(b), norm_tokens(m)\n    card2[pid] = (set(T), {x[:5] for x in T}, {x[:5] for x in B}, {x[:5] for x in M})\nempty_card = (set(), set(), set(), set())\n\npairs2 = pd.concat([d[['query', 'product_id']] for d in (trn, van, test_f)], ignore_index=True)\npairs2 = pairs2.drop_duplicates().reset_index(drop=True)\nqtok2, out = {}, []\nfor q, pid in pairs2.itertuples(index=False):\n    if q not in qtok2:\n        Q = norm_tokens(q)\n        qtok2[q] = (set(Q), {x[:5] for x in Q})\n    Qe, Qs = qtok2[q]\n    Te, Ts, Bs, Ms = card2.get(pid, empty_card)\n    hit = Qs & (Ts | Bs | Ms)\n    out.append((len(Bs & Qs) / len(Bs) if Bs else np.nan, len(Ts & Qs) / len(Ts) if Ts else np.nan,\n                len(Te & Qe) / len(Te) if Te else np.nan, len(Ms & Qs),\n                len(hit) / len(Qs) if Qs else np.nan, len(hit)))\npairs2[base_TM2] = pd.DataFrame(out, columns=base_TM2).values\n\nn_before = (len(trn), len(van), len(test_f))\ntrn, van, test_f = [d.merge(pairs2, on=['query', 'product_id'], how='left') for d in (trn, van, test_f)]\nassert (len(trn), len(van), len(test_f)) == n_before\nfor d in (trn, van, test_f):\n    for c in base_TM2:\n        d[c + '_dev'] = d[c] - d.groupby('sid')[c].transform('mean')\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\n\nfor name, d in (('trn', trn), ('van', van), ('test', test_f)):\n    print(f'{name}: card_hits>0 {(d[\"tm2_card_hits\"] > 0).mean():.3f} | brand_share>0 {(d[\"tm2_brand_share\"] > 0).mean():.3f}'\n          f' | сессий, где card_hits различается: {(d.groupby(\"sid\")[\"tm2_card_hits\"].nunique() > 1).mean():.3f}')\nfor q in ['самсунг', 'хонор', 'samsung', 'honor']:\n    s = trn[trn['query'] == q]\n    if len(s):\n        print(f'запрос «{q}»: {len(s)} строк | brand_share сырой {s[\"tm_brand_share\"].mean():.3f} | с транслитерацией {s[\"tm2_brand_share\"].mean():.3f}')\nprint('группа TM2 готова:', len(cols_TM2), 'признаков')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T09:46:54.45707Z","iopub.execute_input":"2026-10-19T09:46:54.459496Z","iopub.status.idle":"2026-10-19T09:46:59.299556Z","shell.execute_reply.started":"2026-10-19T09:46:54.459448Z","shell.execute_reply":"2026-10-19T09:46:59.298463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# [INIT-HISTORY]\nK_QP, K_QB, K_QT = 3, 10, 20\ncols_Q2 = ['h_q2p_logn', 'h_q2p_coe', 'h_q2b_logn', 'h_q2b_coe', 'h_q2t_coe',\n           'h_q2p_coe_dev', 'h_q2b_coe_dev', 'h_q2t_coe_dev', 'h_q2p_coe_rk']\nneed_cols(['h_prod_coe_click_shrunk', 'h_brand_coe_click', 'h_type_coe_click', 'type_code', 'brand_code'],\n          ('trn', 'van', 'test_f'))\nassert len(Hh) == len(train)\nfor d in (trn, van, test_f):\n    d.drop(columns=[c for c in cols_Q2 if c in d.columns], inplace=True)\n\nallq = pd.concat([train['query'], test['query']], ignore_index=True)\nkey_of = {q: norm_key(q) for q in allq.drop_duplicates()}\nc2, u2 = pd.factorize(allq.map(key_of))\nHh['q2_code'] = c2[:len(train)].astype('int32')\nfor d in (trn, van, test_f):\n    d['q2_code'] = u2.get_indexer(d['query'].map(key_of)).astype('int32')\n    assert (d['q2_code'] >= 0).all()\nprint('уникальных запросов: сырых', allq.nunique(), '| нормализованных', len(u2))\n\nfor tag, keys in (('q2p', ['q2_code', 'product_id']), ('q2b', ['q2_code', 'brand_code']), ('q2t', ['q2_code', 'type_code'])):\n    pair_stats(keys, tag)\n\nprint('доля строк с историей пары «запрос × товар»: сырой запрос → нормализованный')\nfor name, d in (('trn', trn), ('van', van), ('test', test_f)):\n    print(f'  {name}: {(d[\"h_qp_logn\"] > 0).mean():.3f} → {(d[\"_q2p_n\"] > 0).mean():.3f}')\n\nfor d in (trn, van, test_f):\n    d['h_q2p_logn'] = np.log1p(d['_q2p_n'])\n    d['h_q2b_logn'] = np.log1p(d['_q2b_n'])\n    d['h_q2p_coe'] = (d['_q2p_c'] + K_QP * d['h_prod_coe_click_shrunk']) / (d['_q2p_e'] + K_QP)\n    d['h_q2b_coe'] = (d['_q2b_c'] + K_QB * d['h_brand_coe_click']) / (d['_q2b_e'] + K_QB)\n    d['h_q2t_coe'] = (d['_q2t_c'] + K_QT * d['h_type_coe_click']) / (d['_q2t_e'] + K_QT)\n    for c in ('h_q2p_coe', 'h_q2b_coe', 'h_q2t_coe'):\n        d[c + '_dev'] = d[c] - d.groupby('sid')[c].transform('mean')\n    d['h_q2p_coe_rk'] = d.groupby('sid')['h_q2p_coe'].rank(pct=True)\n    d.drop(columns=[c for c in d.columns if c.startswith('_q2')], inplace=True)\nassert (test_f['session_product_id'].values == sample_sub['session_product_id'].values).all()\nprint('группа Q2 готова:', len(cols_Q2), 'признаков')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T09:46:59.301505Z","iopub.execute_input":"2026-10-19T09:46:59.302097Z","iopub.status.idle":"2026-10-19T09:47:54.167584Z","shell.execute_reply.started":"2026-10-19T09:46:59.302064Z","shell.execute_reply":"2026-10-19T09:47:54.166482Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Заменяют ли нормализованные группы сырые Q и TM\n# for f in ['h_q2p_coe', 'tm2_card_hits', 'tm2_brand_share']:\n#     v = van[f].fillna(van[f].median()).values + 1e-9 * np.random.default_rng(SEED).random(len(van))\n#     print(f'{f:16s} NDCG только по нему: по возрастанию {sess_ndcg(van[\"sid\"], van[\"relevance\"], -v).mean():.4f}'\n#           f' | по убыванию {sess_ndcg(van[\"sid\"], van[\"relevance\"], v).mean():.4f}   (случайный порядок около 0.403)')\n# r_prev = feat_t8()\n# r_res = {'T18 (Q+TM сырые)': train_on(trn, r_prev + cols_Q + cols_TM),\n#          'Q2+TM': train_on(trn, r_prev + cols_Q2 + cols_TM),\n#          'Q+TM2': train_on(trn, r_prev + cols_Q + cols_TM2),\n#          'Q2+TM2': train_on(trn, r_prev + cols_Q2 + cols_TM2)}\n# display(compare(r_res, 'T18 (Q+TM сырые)').round(4))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T09:37:35.052525Z","iopub.execute_input":"2026-10-19T09:37:35.052853Z","iopub.status.idle":"2026-10-19T09:40:21.282012Z","shell.execute_reply.started":"2026-10-19T09:37:35.052825Z","shell.execute_reply":"2026-10-19T09:40:21.280841Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# # T1\n# train_and_submit('raw12_it250', FEAT_ALL, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:25.869315Z","iopub.execute_input":"2026-10-19T07:20:25.869668Z","iopub.status.idle":"2026-10-19T07:20:25.89405Z","shell.execute_reply.started":"2026-10-19T07:20:25.869633Z","shell.execute_reply":"2026-10-19T07:20:25.892905Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T2\n# train_and_submit('F_BASE_it250', F_BASE, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:25.895367Z","iopub.execute_input":"2026-10-19T07:20:25.895756Z","iopub.status.idle":"2026-10-19T07:20:25.923734Z","shell.execute_reply.started":"2026-10-19T07:20:25.89572Z","shell.execute_reply":"2026-10-19T07:20:25.922558Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T3\n# train_and_submit('raw12_histA_it250', FEAT_ALL + cols_A, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:25.925009Z","iopub.execute_input":"2026-10-19T07:20:25.925536Z","iopub.status.idle":"2026-10-19T07:20:25.949086Z","shell.execute_reply.started":"2026-10-19T07:20:25.925496Z","shell.execute_reply":"2026-10-19T07:20:25.948092Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T4\n# train_and_submit('raw12_histAB_it250', FEAT_ALL + cols_A + cols_B, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:25.950161Z","iopub.execute_input":"2026-10-19T07:20:25.950537Z","iopub.status.idle":"2026-10-19T07:20:25.981049Z","shell.execute_reply.started":"2026-10-19T07:20:25.950509Z","shell.execute_reply":"2026-10-19T07:20:25.980008Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T5\n# train_and_submit('raw12_histABC_it250', FEAT_ALL + cols_A + cols_B + cols_C, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:25.982495Z","iopub.execute_input":"2026-10-19T07:20:25.982866Z","iopub.status.idle":"2026-10-19T07:20:26.007165Z","shell.execute_reply.started":"2026-10-19T07:20:25.982837Z","shell.execute_reply":"2026-10-19T07:20:26.005615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T6\n# train_and_submit('raw12_histABCD_it250', FEAT_ALL + cols_A + cols_B + cols_C + cols_D, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:26.00843Z","iopub.execute_input":"2026-10-19T07:20:26.008729Z","iopub.status.idle":"2026-10-19T07:20:26.041987Z","shell.execute_reply.started":"2026-10-19T07:20:26.008682Z","shell.execute_reply":"2026-10-19T07:20:26.040849Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T7\n# train_and_submit('raw12_histABC_noabsprice_it250',\n#                  [f for f in FEAT_ALL if f not in ('price', 'price_discount')] + cols_A + cols_B + cols_C, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:26.043349Z","iopub.execute_input":"2026-10-19T07:20:26.043691Z","iopub.status.idle":"2026-10-19T07:20:26.062688Z","shell.execute_reply.started":"2026-10-19T07:20:26.043664Z","shell.execute_reply":"2026-10-19T07:20:26.061565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T8\n# train_and_submit('T6_R_it250', feat_t6() + cols_R, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:26.064177Z","iopub.execute_input":"2026-10-19T07:20:26.064519Z","iopub.status.idle":"2026-10-19T07:20:26.10049Z","shell.execute_reply.started":"2026-10-19T07:20:26.064493Z","shell.execute_reply":"2026-10-19T07:20:26.099216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T9\n# train_and_submit('T6_F_it250', feat_t6() + cols_F + cols_F_dev, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:26.101567Z","iopub.execute_input":"2026-10-19T07:20:26.101931Z","iopub.status.idle":"2026-10-19T07:20:26.126634Z","shell.execute_reply.started":"2026-10-19T07:20:26.101865Z","shell.execute_reply":"2026-10-19T07:20:26.125717Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T10\n# train_and_submit('T6_R_F_it250', feat_t6() + cols_R + cols_F + cols_F_dev, n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:26.12794Z","iopub.execute_input":"2026-10-19T07:20:26.128367Z","iopub.status.idle":"2026-10-19T07:20:26.148225Z","shell.execute_reply.started":"2026-10-19T07:20:26.128332Z","shell.execute_reply":"2026-10-19T07:20:26.147396Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T11\n# BEST_EXTRA = {'num_leaves': 31}\n# train_and_submit_params('T6_tuned', feat_t6(), n_iter=250, extra=BEST_EXTRA)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T07:20:26.149204Z","iopub.execute_input":"2026-10-19T07:20:26.149524Z","iopub.status.idle":"2026-10-19T07:20:26.17715Z","shell.execute_reply.started":"2026-10-19T07:20:26.149497Z","shell.execute_reply":"2026-10-19T07:20:26.17601Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T12\n# train_and_submit('T8_G_it250', feat_t8() + cols_G, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T13\n# BEST_EXTRA = {'num_leaves': 15, 'learning_rate': 0.03}\n# train_and_submit_params('T8_tuned', feat_t8(), n_iter=400, extra=BEST_EXTRA)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T14\n# train_and_submit('T8_P_it250', feat_t8() + cols_P, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T15\n# C_ALL = cols_C + [c for c in cols_D if 'price_to_usual' in c]\n# train_and_submit('T8_noC_P_it250', [f for f in feat_t8() if f not in C_ALL] + cols_P, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T16\n# train_and_submit('T8_Q_it250', feat_t8() + cols_Q, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T17\n# train_and_submit('T8_TM_it250', feat_t8() + cols_TM, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T18\n# train_and_submit('T8_Q_TM_it250', feat_t8() + cols_Q + cols_TM, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T19\n# C_ALL = cols_C + [c for c in cols_D if 'price_to_usual' in c]\n# train_and_submit_blend('T19_blend_T8_T12_T15',\n#                        [feat_t8(), feat_t8() + cols_G, [f for f in feat_t8() if f not in C_ALL] + cols_P], n_iter=250)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T08:36:51.615295Z","iopub.execute_input":"2026-10-19T08:36:51.615759Z","iopub.status.idle":"2026-10-19T08:40:14.817413Z","shell.execute_reply.started":"2026-10-19T08:36:51.615694Z","shell.execute_reply":"2026-10-19T08:40:14.816479Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# T20\ntrain_and_submit('T8_Q2_TM2_it250', feat_t8() + cols_Q2 + cols_TM2, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T21\n# train_and_submit('T8_Q_TM2_it250', feat_t8() + cols_Q + cols_TM2, n_iter=250)","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Ensemble\nСравнение с T20","metadata":{}},{"cell_type":"code","source":"# [DEF]\ndef feat_t18():   return feat_t8() + cols_Q + cols_TM\ndef feat_q2tm2(): return feat_t8() + cols_Q2 + cols_TM2\ndef feat_wide():  return feat_q2tm2() + cols_G + cols_P\n\nGAIN_W = np.array([0, 1, 3, 7])\nBINARY_PARAMS = dict(objective='binary', learning_rate=0.05, num_leaves=31, min_data_in_leaf=100,\n                     feature_fraction=0.9, bagging_fraction=0.8, bagging_freq=1, lambda_l2=10, verbose=-1)\n\ndef member_params(kind, seed, extra=None):\n    p = dict(params_cmp) if kind == 'rank' else dict(BINARY_PARAMS)\n    p.update(extra or {})\n    p.update(seed=seed, bagging_seed=seed, feature_fraction_seed=seed)\n    return p\n\ndef make_dataset(d, feats, kind):\n    X = prep(d, feats)\n    if kind == 'rank':\n        return lgb.Dataset(X, d['relevance'], group=d.groupby('sid', sort=False).size().values)\n    pos = (d['relevance'] > 0).values\n    return lgb.Dataset(X, pos.astype(int), weight=np.where(pos, GAIN_W[d['relevance'].values], 1.0))\n\ndef fit_predict(d_train, targets, feats, kind, n_iter, seeds, extra=None):\n    \"\"\"Обучение на d_train; средний по сидам скор для каждого датафрейма из targets.\"\"\"\n    need_cols(feats, ('trn', 'van', 'test_f'))\n    out = [np.zeros(len(t)) for t in targets]\n    for sd in seeds:\n        m = lgb.train(member_params(kind, sd, extra), make_dataset(d_train, feats, kind), n_iter)\n        for i, t in enumerate(targets):\n            out[i] += m.predict(prep(t, feats))\n    return [o / len(seeds) for o in out]\n\ndef blend_ranks(score_list, sid):\n    \"\"\"Среднее процентильных рангов внутри сессии; ничьи разрываются сырым скором первого члена.\"\"\"\n    ranks = [pd.Series(s).groupby(np.asarray(sid)).rank(pct=True).values for s in score_list]\n    return np.mean(ranks, axis=0) + 1e-9 * np.asarray(score_list[0])\n\nMEMBERS = [dict(name='rank_T18',       feats=feat_t18,   kind='rank',   extra=None,               n_iter=250),\n           dict(name='rank_Q2TM2',     feats=feat_q2tm2, kind='rank',   extra=None,               n_iter=250),\n           dict(name='binary_Q2TM2',   feats=feat_q2tm2, kind='binary', extra=None,               n_iter=250),\n           dict(name='rank_wide_nl15', feats=feat_wide,  kind='rank',   extra={'num_leaves': 15}, n_iter=400)]\n\ndef train_and_submit_ensemble(tag, members, seeds=(1, 2, 3, 4, 5)):\n    \"\"\"Обучение каждого члена на trn+van, смесь рангов, запись submission.csv и копии в archive/.\"\"\"\n    final = pd.concat([trn, van], ignore_index=True)\n    scores = []\n    for mb in members:\n        (s,) = fit_predict(final, [test_f], mb['feats'](), mb['kind'], mb['n_iter'], seeds, mb['extra'])\n        scores.append(s)\n        print(f\"{mb['name']:18s} обучен: {len(mb['feats']())} признаков, {mb['n_iter']} итераций\")\n    blended = blend_ranks(scores, test_f['sid'].values)\n    sub = pd.DataFrame({'session_product_id': test['session_product_id'].values, 'relevance': blended})\n    assert len(sub) == len(sample_sub) and np.isfinite(sub['relevance']).all()\n    assert (sub['session_product_id'].values == sample_sub['session_product_id'].values).all()\n    os.makedirs(f'{OUT_DIR}/archive', exist_ok=True)\n    sub.to_csv(f'{OUT_DIR}/archive/{tag}.csv', index=False)\n    sub.to_csv(f'{OUT_DIR}/submission.csv', index=False)\n    print(f'{tag}: {len(members)} членов | уникальных скоров {pd.Series(blended).nunique() / len(blended):.3f} | submission.csv записан')\n    return blended","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T09:47:54.169202Z","iopub.execute_input":"2026-10-19T09:47:54.16951Z","iopub.status.idle":"2026-10-19T09:47:54.192668Z","shell.execute_reply.started":"2026-10-19T09:47:54.169485Z","shell.execute_reply":"2026-10-19T09:47:54.191689Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # [EXP] Каждый член отдельно и разные смеси; обучение на trn, оценка на van\n# r_scores = {}\n# for mb in MEMBERS:\n#     (r_scores[mb['name']],) = fit_predict(trn, [van], mb['feats'](), mb['kind'], mb['n_iter'], (1, 2, 3), mb['extra'])\n# names = list(r_scores)\n# r_var = {n: sess_ndcg(van['sid'], van['relevance'], r_scores[n]) for n in names}\n# def blend_per(sub):\n#     return sess_ndcg(van['sid'], van['relevance'], blend_ranks([r_scores[n] for n in sub], van['sid'].values))\n# r_var['blend_all'] = blend_per(names)\n# r_var['blend_rank_only'] = blend_per([n for n in names if n.startswith('rank')])\n# for n in names:\n#     r_var['blend_without_' + n] = blend_per([m for m in names if m != n])\n# ref = names[0]\n# rows = []\n# for n, ps in r_var.items():\n#     d = ps - r_var[ref]\n#     rows.append({'variant': n, 'ndcg': ps.mean(), f'diff_vs_{ref}': d.mean(), '±2SE': 2 * d.std() / np.sqrt(len(d))})\n# display(pd.DataFrame(rows).round(4))\n\n# r_rk = pd.DataFrame({n: pd.Series(r_scores[n]).groupby(van['sid'].values).rank(pct=True).values for n in names})\n# print('корреляция рангов внутри сессий между членами (чем ниже, тем больше разнообразие):')\n# display(r_rk.corr().round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-19T09:47:56.809055Z","iopub.execute_input":"2026-10-19T09:47:56.809399Z","iopub.status.idle":"2026-10-19T09:50:24.150293Z","shell.execute_reply.started":"2026-10-19T09:47:56.809372Z","shell.execute_reply":"2026-10-19T09:50:24.148916Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# # T22\n# train_and_submit_ensemble('T22_blend', MEMBERS)","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}