{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":115104,"databundleVersionId":13748315,"sourceType":"competition"}],"dockerImageVersionId":31089,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Описание решения","metadata":{}},{"cell_type":"markdown","source":"## Назначение\nРанжирование товаров в поисковой выдаче на основе пользовательских запросов. Модель ранжирует товары по ожидаемой релевантности (клики/корзины/покупки).\n\n## Архитектура\n0. EDA\n1. Предобработка:\n - обработка товарных замен\n - обрезка групп до топ-20 релевантных товаров\n - исключение групп с менее чем 5 товарами \n3. Текстовая обработка:\n - нормализация\n - лемматизация кириллицы\n - транслитерация в кириллицу либо исправление раскладки клавиатуры\n - исправление опечаток\n4. Поисковая релевантность:\n - совпадение по бренду\n - BM25 для запроса и модели\n - косинусная схожесть эмбеддингов запроса и характеристик товара (тип, категории, страна)\n6. Создание признаков: товарные, пользовательские, временные, ценовые, взаимодействие признаков\n7. Модель: LightGBM LambdaMART (learning to rank)\n\n## Технический стек\nPython, sklearn, pandas, scipy, numpy, LightGBM, Sentence Transformers, pymorphy3, symspellpy","metadata":{}},{"cell_type":"code","source":"pip install symspellpy","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true,"outputs_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:18.278312Z","iopub.execute_input":"2025-10-25T02:41:18.279018Z","iopub.status.idle":"2025-10-25T02:41:22.189117Z","shell.execute_reply.started":"2025-10-25T02:41:18.278991Z","shell.execute_reply":"2025-10-25T02:41:22.187776Z"},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pip install pymorphy3","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true,"outputs_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:22.191270Z","iopub.execute_input":"2025-10-25T02:41:22.191565Z","iopub.status.idle":"2025-10-25T02:41:26.069572Z","shell.execute_reply.started":"2025-10-25T02:41:22.191538Z","shell.execute_reply":"2025-10-25T02:41:26.068464Z"},"collapsed":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======= ПОДКЛЮЧЕНИЕ БИБЛИОТЕК =======\n\nimport random\nimport numpy as np\nimport pandas as pd\nimport lightgbm as lgb\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom scipy import stats\nfrom scipy.stats import chi2_contingency\nimport scipy.sparse as sp\n\nfrom sklearn.feature_selection import mutual_info_classif\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.preprocessing import RobustScaler, QuantileTransformer\n\nfrom datetime import datetime\nfrom collections import Counter\nfrom functools import lru_cache\n\nimport re\nimport pymorphy3\nfrom symspellpy import SymSpell, Verbosity\n\nfrom sentence_transformers import SentenceTransformer\n\nimport warnings\nwarnings.filterwarnings('ignore')","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.071034Z","iopub.execute_input":"2025-10-25T02:41:26.071463Z","iopub.status.idle":"2025-10-25T02:41:26.081633Z","shell.execute_reply.started":"2025-10-25T02:41:26.071430Z","shell.execute_reply":"2025-10-25T02:41:26.080600Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ======= КОНФИГУРАЦИЯ ПАЙПЛАЙНА =======\n\nclass Config:\n SEED = 42\n IS_VALIDATION = False\n \n DATA_PATHS = {\n 'train': '/kaggle/input/21-vek-by-searched-products-ranking/train.csv',\n 'test': '/kaggle/input/21-vek-by-searched-products-ranking/test.csv', \n 'products': '/kaggle/input/21-vek-by-searched-products-ranking/products.csv'\n }\n \n SAMPLING_CONFIG = {\n 'min_group_size': 5,\n 'validation_group_size': 20,\n 'max_group_size': 20,\n 'validation_months': 2\n }\n \n TEXT_PROCESSING = {\n 'lemmatize_min_length': 3,\n 'batch_size': 100000,\n 'chunk_size': 100000\n }\n \n SPELL_CHECKER_CONFIG = {\n 'max_edit_distance': 6,\n 'prefix_length': 7,\n 'count_threshold': 1,\n }\n \n BM25_CONFIG = {\n 'k1': 1.5,\n 'b': 0.75,\n 'vocab_size': 50000,\n 'batch_size': 100000,\n 'query_column': 'query_model',\n 'document_column': 'model'\n }\n \n EMBEDDING_CONFIG = {\n 'model_name': 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2',\n 'chunk_size': 100000,\n 'device': 'cpu',\n 'normalize_embeddings': True,\n 'show_progress_bar': True,\n 'convert_to_tensor': False,\n 'text_configs': [\n {\n 'name': 'query_text',\n 'source': 'single_column',\n 'column': 'query_general',\n },\n {\n 'name': 'category_text',\n 'source': 'combined_columns',\n 'columns': ['type', 'category_name_1', 'category_name_2', \n 'category_name_3', 'category_name_4', 'country'],\n }\n ]\n }\n \n LGBM_PARAMS = {\n 'objective': 'lambdarank',\n 'metric': 'ndcg',\n 'boosting_type': 'gbdt',\n 'lambdarank_norm': True,\n 'lambdarank_truncation_level': 20,\n 'ndcg_eval_at': [20],\n \n 'num_leaves': 127,\n 'min_data_in_leaf': 50,\n 'min_child_samples': 20,\n 'reg_alpha': 0.2,\n 'reg_lambda': 0.2,\n 'subsample': 0.7,\n 'colsample_bytree': 0.7,\n 'subsample_freq': 1,\n\n 'learning_rate': 0.05,\n 'n_estimators': 500,\n 'n_jobs': 8,\n \n 'max_depth': -1,\n 'max_bin': 255,\n 'verbosity': -1,\n 'force_row_wise': True\n }\n \n OUTPUT_CONFIG = {\n 'submission_file': 'submission.csv',\n 'feature_importance_file': 'lgb_feature_importance.csv',\n }\n\n\n# ======= УТИЛИТЫ ДЛЯ РАБОТЫ С КОНФИГОМ ======= \n\ndef print_complete_config_summary(config):\n print(\"# ==== КОНФИГУРАЦИЯ ПАЙПЛАЙНА ====\")\n \n sections = {\n \"ДАННЫЕ\": config.DATA_PATHS,\n \"СЭМПЛИРОВАНИЕ\": config.SAMPLING_CONFIG,\n \"ТЕКСТОВАЯ ОБРАБОТКА\": config.TEXT_PROCESSING,\n \"BM25\": config.BM25_CONFIG,\n \"ЭМБЕДДИНГИ\": config.EMBEDDING_CONFIG,\n \"LIGHTGBM\": {k: v for k, v in config.LGBM_PARAMS.items()},\n \"ВЫВОД\": config.OUTPUT_CONFIG\n }\n \n for section_name, section_data in sections.items():\n print(f\"\\n{section_name}\")\n print(\"-\" * 40)\n for key, value in section_data.items():\n print(f\" {key:.<30} {value}\")\n\nconfig = Config()\nnp.random.seed(config.SEED)\nrandom.seed(config.SEED)","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.084376Z","iopub.execute_input":"2025-10-25T02:41:26.084668Z","iopub.status.idle":"2025-10-25T02:41:26.109101Z","shell.execute_reply.started":"2025-10-25T02:41:26.084647Z","shell.execute_reply":"2025-10-25T02:41:26.108129Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===== КОНСТАНТЫ ДЛЯ ТЕКСТОВОЙ ПРЕДОБРАБОТКИ =====\n\nWORD_PATTERNS = {\n 'word': re.compile(r'^[a-zA-Z0-9\\-\\+]+$'),\n 'percent': re.compile(r'(\\d+)%'),\n 'size': re.compile(r'(\\d+)[xх](\\d+)'),\n 'plus': re.compile(r'([a-zA-Zа-яА-Я]+)[+]'),\n 'non_word': re.compile(r'[^\\w\\s\\.\\+-]'),\n 'multi_space': re.compile(r'\\s+'),\n 'latin': re.compile('[a-zA-Z]'),\n 'letters_digits': re.compile(r'([a-zA-Zа-яА-Я]+)(\\d+)'),\n 'digits_letters': re.compile(r'(\\d+)([a-zA-Zа-яА-Я]+)'),\n 'comma_numbers': re.compile(r'(\\d+),(\\d+)')\n}\n\n# ===== СЛОВАРЬ ДЛЯ ТРАНСЛИТЕРАЦИИ =====\n\nSPECIAL_CASES = {\n # Бренды и модели\n 'iphone': 'айфон', 'apple': 'эпл', 'ipad': 'айпад', 'macbook': 'макбук',\n 'galaxy': 'галакси', 'thinkpad': 'тинкпад', 'xbox': 'ксбокс',\n 'windows': 'виндоус', 'android': 'андроид', 'samsung': 'самсунг',\n 'nokia': 'нокиа', 'lenovo': 'леново', 'huawei': 'хуавей',\n 'xiaomi': 'сяоми', 'oppo': 'оппо', 'vivo': 'виво', 'realme': 'рилми',\n 'oneplus': 'ванплас', 'google': 'гугл', 'sony': 'сони',\n 'lg': 'элджи', 'asus': 'асус', 'acer': 'асер', 'dell': 'делл',\n 'hp': 'хп', 'canon': 'канон', 'nikon': 'никон', 'band': 'бенд',\n 'playstation': 'плейстейшн', 'nintendo': 'нинтендо',\n 'watch': 'вотч', 'nfc': 'нфс', 'plus': 'плюс', 'synergetic': 'синергетик', \n \n # Основные цвета\n 'red': 'красный', 'blue': 'синий', 'green': 'зеленый', 'yellow': 'желтый', 'black': 'черный', 'white': 'белый',\n \n # Фиолетовые\n 'purple': 'фиолетовый', 'violet': 'фиолетовый', 'lavender': 'лавандовый', 'lilac': 'сиреневый', 'plum': 'сливовый',\n 'magenta': 'пурпурный', 'orchid': 'орхидея', 'mauve': 'розовато-лиловый', 'indigo': 'индиго', \n \n # Розовые\n 'pink': 'розовый', 'rose': 'розовый', 'coral': 'коралловый', 'salmon': 'лососевый', 'fuchsia': 'фуксия',\n \n # Оранжевые и коричневые\n 'orange': 'оранжевый', 'brown': 'коричневый', 'tan': 'желто-коричневый', 'beige': 'бежевый', 'cream': 'кремовый',\n 'peach': 'персиковый', 'apricot': 'абрикосовый', 'caramel': 'карамельный', 'chocolate': 'шоколадный',\n \n # Серые и металлические\n 'gray': 'серый', 'silver': 'серебристый', 'charcoal': 'угольный', 'graphite': 'графитовый', 'slate': 'аспидный',\n 'steel': 'стальной', 'gunmetal': 'оружейный металл', 'pewter': 'оловянный', 'ash': 'пепельный',\n \n # Синие оттенки\n 'navy': 'темно-синий', 'cyan': 'голубой', 'turquoise': 'бирюзовый', 'teal': 'сине-зеленый', 'aqua': 'аква',\n 'sky': 'небесный', 'royal': 'королевский', 'cobalt': 'кобальтовый', 'cerulean': 'лазоревый',\n \n # Золотые и желтые\n 'gold': 'золотой', 'bronze': 'бронзовый', 'copper': 'медный', 'amber': 'янтарный', 'mustard': 'горчичный',\n 'lemon': 'лимонный', 'honey': 'медовый', 'vanilla': 'ванильный',\n \n # Темные\n 'midnight': 'полуночный', 'ebony': 'эбеновый', 'espresso': 'эспрессо', 'mahogany': 'красное дерево',\n 'maroon': 'бордовый', 'burgundy': 'бургундский', 'chestnut': 'каштановый',\n \n # Светлые и пастельные\n 'ivory': 'слоновая кость', 'pearl': 'жемчужный', 'alabaster': 'алебастровый', 'porcelain': 'фарфоровый',\n 'champagne': 'шампань', 'butter': 'сливочный', 'mist': 'туманный', 'frost': 'морозный',\n \n # Яркие и неоновые\n 'neon': 'неоновый', 'electric': 'электрический', 'fluorescent': 'флуоресцентный', 'crimson': 'малиновый',\n 'scarlet': 'алый', 'ruby': 'рубиновый', 'vermillion': 'киноварь',\n \n # Природные\n 'olive': 'оливковый', 'khaki': 'хаки', 'mint': 'мятный', 'lime': 'лаймовый', 'forest': 'лесной',\n 'jade': 'нефритовый', 'emerald': 'изумрудный', 'malachite': 'малахитовый', 'pine': 'сосновый',\n 'sea': 'морской', 'ocean': 'океанский', 'lagoon': 'лагуна', 'arctic': 'арктический', 'glacier': 'ледниковый'\n}\n\n\n# ===== ПРАВИЛА ТРАНСЛИТЕРАЦИИ =====\n\nTRANSLITERATION_RULES = [\n ('sch', 'щ'), ('ch', 'ч'), ('sh', 'ш'), ('zh', 'ж'), \n ('th', 'т'), ('ph', 'ф'), ('kh', 'х'),\n ('you', 'ю'), ('ya', 'я'), ('ye', 'ье'), ('yo', 'йо'),\n ('ia', 'ия'), ('io', 'ио'), ('iu', 'ю'), ('yu', 'ю'),\n ('a', 'а'), ('b', 'б'), ('c', 'к'), ('d', 'д'), \n ('e', 'е'), ('f', 'ф'), ('g', 'г'), ('h', 'х'),\n ('i', 'и'), ('j', 'дж'), ('k', 'к'), ('l', 'л'),\n ('m', 'м'), ('n', 'н'), ('o', 'о'), ('p', 'п'),\n ('q', 'к'), ('r', 'р'), ('s', 'с'), ('t', 'т'),\n ('u', 'у'), ('v', 'в'), ('w', 'в'), ('x', 'кс'),\n ('y', 'й'), ('z', 'з')\n]\n\n\n# ===== РАСКЛАДКИ КЛАВИАТУРЫ =====\n\nKEYBOARD_LAYOUTS = {\n 'ru_to_en': {\n 'й': 'q', 'ц': 'w', 'у': 'e', 'к': 'r', 'е': 't', 'н': 'y', 'г': 'u', \n 'ш': 'i', 'щ': 'o', 'з': 'p', 'х': '[', 'ъ': ']', 'ф': 'a', 'ы': 's', \n 'в': 'd', 'а': 'f', 'п': 'g', 'р': 'h', 'о': 'j', 'л': 'k', 'д': 'l', \n 'ж': ';', 'э': \"'\", 'я': 'z', 'ч': 'x', 'с': 'c', 'м': 'v', 'и': 'b', \n 'т': 'n', 'ь': 'm', 'б': ',', 'ю': '.', 'ё': '`'\n },\n 'en_to_ru': {\n 'q': 'й', 'w': 'ц', 'e': 'у', 'r': 'к', 't': 'е', 'y': 'н', 'u': 'г',\n 'i': 'ш', 'o': 'щ', 'p': 'з', '[': 'х', ']': 'ъ', 'a': 'ф', 's': 'ы',\n 'd': 'в', 'f': 'а', 'g': 'п', 'h': 'р', 'j': 'о', 'k': 'л', 'l': 'д',\n ';': 'ж', \"'\": 'э', 'z': 'я', 'x': 'ч', 'c': 'с', 'v': 'м', 'b': 'и',\n 'n': 'т', 'm': 'ь', ',': 'б', '.': 'ю', '`': 'ё'\n }\n}\n\nCYRILLIC_CHARS = set(KEYBOARD_LAYOUTS['ru_to_en'].keys())\nLATIN_CHARS = set(KEYBOARD_LAYOUTS['en_to_ru'].keys())","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.110873Z","iopub.execute_input":"2025-10-25T02:41:26.111224Z","iopub.status.idle":"2025-10-25T02:41:26.140441Z","shell.execute_reply.started":"2025-10-25T02:41:26.111195Z","shell.execute_reply":"2025-10-25T02:41:26.139375Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# ===== СПИСОК КОЛОНОК ДЛЯ УДАЛЕНИЯ =====\n\nCOLUMNS_TO_DROP = [\n # Исходные числовые колонки\n 'price', 'price_discount', 'browse_time', 'response_time', 'product_position',\n \n # Временные колонки\n 'date_of_create', 'date', \n \n # Идентификаторы\n 'selected_category_id', 'user_number', 'search_number', 'session_product_id', 'actual_product_id',\n \n # Текстовые колонки\n 'query', 'category_name_1', 'category_name_2', 'category_name_3', 'category_name_4',\n 'query_model', 'query_general', 'found_brand',\n 'brand', 'model', 'country', 'type', 'device',\n 'available', 'price_is_discounted',\n \n # Временные метрики продуктов\n 'cumulative_views', 'cumulative_clicks', 'cumulative_carts', \n 'cumulative_purchases', 'cumulative_relevance_sum',\n 'cumulative_user_clicks', 'cumulative_user_purchases',\n 'cumulative_user_sessions', 'cumulative_filters_sum',\n \n # Целевая переменная\n 'relevance',\n \n # Временные фичи\n 'price_is_discounted', 'brand_category_count',\n 'query_global_frequency', 'query_word_count', 'query_avg_word_length',\n 'cumulative_sessions_count', 'cumulative_complexity_sum',\n 'prev_relevance', 'prev_brand'\n]","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.141587Z","iopub.execute_input":"2025-10-25T02:41:26.141909Z","iopub.status.idle":"2025-10-25T02:41:26.162853Z","shell.execute_reply.started":"2025-10-25T02:41:26.141880Z","shell.execute_reply":"2025-10-25T02:41:26.161657Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"markdown","source":"## Базовый анализ","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(config.DATA_PATHS['train'])\ntest_df = pd.read_csv(config.DATA_PATHS['test'])\nproducts_df = pd.read_csv(config.DATA_PATHS['products'])","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.164094Z","iopub.execute_input":"2025-10-25T02:41:26.164498Z","iopub.status.idle":"2025-10-25T02:41:26.425976Z","shell.execute_reply.started":"2025-10-25T02:41:26.164468Z","shell.execute_reply":"2025-10-25T02:41:26.425087Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train_df.copy()\ntest = test_df.copy()\nproducts = products_df.copy()\ntrain = train.merge(products, on='product_id', how='left')\ntrain['date'] = pd.to_datetime(train['date'])","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.427282Z","iopub.execute_input":"2025-10-25T02:41:26.427559Z","iopub.status.idle":"2025-10-25T02:41:26.461218Z","shell.execute_reply.started":"2025-10-25T02:41:26.427538Z","shell.execute_reply":"2025-10-25T02:41:26.460182Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Размеры данных:\")\nprint(f\"Train: {train.shape}, Test: {test.shape}, Products: {products.shape}\")","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.462228Z","iopub.execute_input":"2025-10-25T02:41:26.462534Z","iopub.status.idle":"2025-10-25T02:41:26.468371Z","shell.execute_reply.started":"2025-10-25T02:41:26.462513Z","shell.execute_reply":"2025-10-25T02:41:26.467216Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.472308Z","iopub.execute_input":"2025-10-25T02:41:26.472580Z","iopub.status.idle":"2025-10-25T02:41:26.494836Z","shell.execute_reply.started":"2025-10-25T02:41:26.472558Z","shell.execute_reply":"2025-10-25T02:41:26.493663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"test.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.496010Z","iopub.execute_input":"2025-10-25T02:41:26.496812Z","iopub.status.idle":"2025-10-25T02:41:26.520171Z","shell.execute_reply.started":"2025-10-25T02:41:26.496774Z","shell.execute_reply":"2025-10-25T02:41:26.518993Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"products.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.521368Z","iopub.execute_input":"2025-10-25T02:41:26.521664Z","iopub.status.idle":"2025-10-25T02:41:26.565743Z","shell.execute_reply.started":"2025-10-25T02:41:26.521632Z","shell.execute_reply":"2025-10-25T02:41:26.564467Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.566691Z","iopub.execute_input":"2025-10-25T02:41:26.566935Z","iopub.status.idle":"2025-10-25T02:41:26.588419Z","shell.execute_reply.started":"2025-10-25T02:41:26.566916Z","shell.execute_reply":"2025-10-25T02:41:26.587291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"products.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:26.589650Z","iopub.execute_input":"2025-10-25T02:41:26.589986Z","iopub.status.idle":"2025-10-25T02:41:26.614854Z","shell.execute_reply.started":"2025-10-25T02:41:26.589957Z","shell.execute_reply":"2025-10-25T02:41:26.613651Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_missing_and_outliers(train, products):\n print(\"\\n=== АНАЛИЗ ПРОПУСКОВ И ВЫБРОСОВ ===\")\n \n print(\"Пропуски в train:\")\n for col in train.columns:\n if train[col].isna().sum() > 0:\n print(f\" {col}: {train[col].isna().sum():,} ({train[col].isna().mean():.1%})\")\n \n print(\"\\nПропуски в products:\")\n for col in products.columns:\n if products[col].isna().sum() > 0:\n print(f\" {col}: {products[col].isna().sum():,} ({products[col].isna().mean():.1%})\")\n\n browse_outliers = train[train['browse_time'] > train['browse_time'].quantile(0.99)]\n print(f\"\\nВыбросы browse_time (>99%): {len(browse_outliers):,} записей\")\n print(f\"Конверсия выбросов: {(browse_outliers['relevance'] >= 1).mean():.3f}\")\n\n product_position_outliers = train[train['product_position'] > train['product_position'].quantile(0.99)]\n print(f\"\\nВыбросы product_position (>99%): {len(product_position_outliers):,} записей\")\n print(f\"Конверсия выбросов: {(product_position_outliers['relevance'] >= 1).mean():.3f}\")\n\n price_outliers = train[train['price'] > train['price'].quantile(0.99)]\n print(f\"\\nВыбросы price (>99%): {len(price_outliers):,} записей\")\n print(f\"Конверсия выбросов: {(price_outliers['relevance'] >= 1).mean():.3f}\")\n\n response_time_outliers = train[train['response_time'] > train['response_time'].quantile(0.99)]\n print(f\"\\nВыбросы response_time (>99%): {len(response_time_outliers):,} записей\")\n print(f\"Конверсия выбросов: {(response_time_outliers['relevance'] >= 1).mean():.3f}\")\n\nanalyze_missing_and_outliers(train, products)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.616100Z","iopub.execute_input":"2025-10-25T02:41:26.616546Z","iopub.status.idle":"2025-10-25T02:41:26.665072Z","shell.execute_reply.started":"2025-10-25T02:41:26.616510Z","shell.execute_reply":"2025-10-25T02:41:26.664204Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Сравнение train и test","metadata":{}},{"cell_type":"code","source":"print(\"=== БАЗОВОЕ СРАВНЕНИЕ TRAIN/TEST ===\")\n\nprint(f\"\\n--- Основные метрики ---\")\nprint(f\"Уникальные пользователи: train={train['user_number'].nunique():,}, test={test['user_number'].nunique():,}\")\nprint(f\"Уникальные сессии: train={train['search_number'].nunique():,}, test={test['search_number'].nunique():,}\")\nprint(f\"Уникальные товары: train={train['product_id'].nunique():,}, test={test['product_id'].nunique():,}\")\n","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.666076Z","iopub.execute_input":"2025-10-25T02:41:26.666763Z","iopub.status.idle":"2025-10-25T02:41:26.673985Z","shell.execute_reply.started":"2025-10-25T02:41:26.666739Z","shell.execute_reply":"2025-10-25T02:41:26.672786Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"=== СРАВНЕНИЕ ДАТ ===\")\n\ntrain['date'] = pd.to_datetime(train['date'])\ntest['date'] = pd.to_datetime(test['date'])\n\nprint(f\"Диапазон дат train: {train['date'].min()} - {train['date'].max()}\")\nprint(f\"Диапазон дат test: {test['date'].min()} - {test['date'].max()}\")\nprint(f\"Разрыв между выборками: {(test['date'].min() - train['date'].max()).days} день\")","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.675030Z","iopub.execute_input":"2025-10-25T02:41:26.675324Z","iopub.status.idle":"2025-10-25T02:41:26.704342Z","shell.execute_reply.started":"2025-10-25T02:41:26.675293Z","shell.execute_reply":"2025-10-25T02:41:26.703226Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(16, 6))\n\ntrain_daily = pd.to_datetime(train['date']).dt.date.value_counts().sort_index() / train['date'].nunique()\ntest_daily = pd.to_datetime(test['date']).dt.date.value_counts().sort_index() / test['date'].nunique()\n\nax.plot(train_daily.index, train_daily.values, label='Train', linewidth=2)\nax.plot(test_daily.index, test_daily.values, label='Test', linewidth=2)\n\nax.set_title('Средняя активность по дням', fontsize=16)\nax.set_ylabel('Среднее количество записей в день', fontsize=12)\nax.legend(fontsize=12)\nax.tick_params(labelsize=10)\nplt.xticks(rotation=45)\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:41:26.705664Z","iopub.execute_input":"2025-10-25T02:41:26.706026Z","iopub.status.idle":"2025-10-25T02:41:27.052319Z","shell.execute_reply.started":"2025-10-25T02:41:26.705993Z","shell.execute_reply":"2025-10-25T02:41:27.050958Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"numeric_cols = ['product_position', 'response_products', 'response_time', 'browse_time', 'price', 'price_discount']\n \n\ndef detailed_distribution_comparison(train, test):\n print(\"=== СРАВНЕНИЕ РАСПРЕДЕЛЕНИЙ ===\")\n \n comparison_results = []\n \n for col in numeric_cols:\n if col in train.columns and col in test.columns:\n train_stats = {\n 'mean': train[col].mean(),\n 'std': train[col].std(),\n 'min': train[col].min(),\n 'max': train[col].max(),\n 'median': train[col].median(),\n 'q95': train[col].quantile(0.95)\n }\n \n test_stats = {\n 'mean': test[col].mean(),\n 'std': test[col].std(),\n 'min': test[col].min(),\n 'max': test[col].max(),\n 'median': test[col].median(),\n 'q95': test[col].quantile(0.95)\n }\n \n mean_shift = (test_stats['mean'] - train_stats['mean']) / train_stats['mean'] * 100\n median_shift = (test_stats['median'] - train_stats['median']) / train_stats['median'] * 100\n \n comparison_results.append({\n 'feature': col,\n 'train_mean': train_stats['mean'],\n 'test_mean': test_stats['mean'],\n 'mean_shift_pct': mean_shift,\n 'train_median': train_stats['median'],\n 'test_median': test_stats['median'],\n 'median_shift_pct': median_shift,\n 'train_std': train_stats['std'],\n 'test_std': test_stats['std']\n })\n\n comp_df = pd.DataFrame(comparison_results)\n \n print(\"\\n--- Сравнение числовых признаков ---\")\n for _, row in comp_df.iterrows():\n print(f\"\\n{row['feature']}:\")\n print(f\" Среднее: train={row['train_mean']:.2f}, test={row['test_mean']:.2f} (смещение: {row['mean_shift_pct']:+.1f}%)\")\n print(f\" Медиана: train={row['train_median']:.2f}, test={row['test_median']:.2f} (смещение: {row['median_shift_pct']:+.1f}%)\")\n print(f\" Std: train={row['train_std']:.2f}, test={row['test_std']:.2f}\")\n \n return comp_df\n\ndistribution_comparison = detailed_distribution_comparison(train, test)","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:41:27.053572Z","iopub.execute_input":"2025-10-25T02:41:27.053853Z","iopub.status.idle":"2025-10-25T02:41:27.170794Z","shell.execute_reply.started":"2025-10-25T02:41:27.053832Z","shell.execute_reply":"2025-10-25T02:41:27.167028Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def statistical_tests(train, test):\n print(\"\\n=== СТАТИСТИЧЕСКИЕ ТЕСТЫ РАЗЛИЧИЙ ===\")\n\n for col in numeric_cols:\n t_stat, p_value = stats.ttest_ind(train[col].dropna(), test[col].dropna(), equal_var=False)\n ks_stat, ks_pvalue = stats.ks_2samp(train[col].dropna(), test[col].dropna())\n \n print(f\"\\n{col}:\")\n print(f\" t-тест: p-value={p_value:.6f} {'***' if p_value < 0.001 else '**' if p_value < 0.01 else '*' if p_value < 0.05 else 'NS'}\")\n print(f\" KS-тест: p-value={ks_pvalue:.6f} {'***' if ks_pvalue < 0.001 else '**' if ks_pvalue < 0.01 else '*' if ks_pvalue < 0.05 else 'NS'}\")\n\nstatistical_tests(train, test)","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:07.312642Z","iopub.execute_input":"2025-10-25T02:42:07.313233Z","iopub.status.idle":"2025-10-25T02:42:07.401697Z","shell.execute_reply.started":"2025-10-25T02:42:07.313193Z","shell.execute_reply":"2025-10-25T02:42:07.400729Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Обучающая и тестовая выборки статистически значимо различаются по всем числовым параметрам. ","metadata":{}},{"cell_type":"code","source":"def plot_distributions(train, test):\n\n fig, axes = plt.subplots(6, 1, figsize=(12, 30))\n \n for i, col in enumerate(numeric_cols):\n train_clean = train[col].dropna()\n test_clean = test[col].dropna()\n \n axes[i].hist(train_clean, alpha=0.7, label='Train', bins=50, density=True)\n axes[i].hist(test_clean, alpha=0.7, label='Test', bins=50, density=True)\n axes[i].set_title(f'{col}')\n axes[i].legend()\n axes[i].grid(True, alpha=0.3)\n \n plt.tight_layout()\n plt.show()\n\nprint(\"\\n=== ВИЗУАЛИЗАЦИЯ РАСПРЕДЕЛЕНИЙ НА TRAIN/TEST ===\")\nplot_distributions(train, test)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:07.403353Z","iopub.execute_input":"2025-10-25T02:42:07.403708Z","iopub.status.idle":"2025-10-25T02:42:09.786997Z","shell.execute_reply.started":"2025-10-25T02:42:07.403678Z","shell.execute_reply":"2025-10-25T02:42:09.785900Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_user_overlap(train, test):\n print(\"\\n=== АНАЛИЗ ПЕРЕСЕЧЕНИЯ ПОЛЬЗОВАТЕЛЕЙ TRAIN/TEST ===\")\n \n train_users = set(train['user_number'].unique())\n test_users = set(test['user_number'].unique())\n \n common_users = train_users & test_users\n unique_train_users = train_users - test_users\n unique_test_users = test_users - train_users\n \n print(f\"Уникальных пользователей в train: {len(train_users):,}\")\n print(f\"Уникальных пользователей в test: {len(test_users):,}\")\n print(f\"Общих пользователей: {len(common_users):,}\")\n print(f\"Уникальных в train: {len(unique_train_users):,}\")\n print(f\"Уникальных в test: {len(unique_test_users):,}\")\n print(f\"Покрытие test пользователей: {len(common_users)/len(test_users)*100:.1f}%\")\n \n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n \n overlap_sizes = [len(common_users), len(unique_train_users), len(unique_test_users)]\n overlap_labels = ['Общие пользователи', 'Только в train', 'Только в test']\n \n ax1.pie(overlap_sizes, labels=overlap_labels, autopct='%1.1f%%', startangle=90)\n ax1.set_title('Пересечение пользователей между train и test')\n\n train_user_activity = train.groupby('user_number').size()\n test_user_activity = test.groupby('user_number').size()\n \n activity_bins = [1, 2, 5, 10, 20, 50, 100, 1000]\n train_activity_dist = pd.cut(train_user_activity, bins=activity_bins).value_counts().sort_index()\n test_activity_dist = pd.cut(test_user_activity, bins=activity_bins).value_counts().sort_index()\n \n x = np.arange(len(train_activity_dist))\n width = 0.35\n \n ax2.bar(x - width/2, train_activity_dist.values, width, label='Train', alpha=0.7)\n ax2.bar(x + width/2, test_activity_dist.values, width, label='Test', alpha=0.7)\n ax2.set_xlabel('Количество сессий на пользователя')\n ax2.set_ylabel('Количество пользователей')\n ax2.set_title('Распределение пользователей по активности')\n ax2.set_xticks(x)\n ax2.set_xticklabels([str(bin_range) for bin_range in train_activity_dist.index])\n ax2.legend()\n ax2.tick_params(axis='x', rotation=45)\n \n plt.tight_layout()\n plt.show()\n\n if len(common_users) > 0:\n common_users_train_activity = train[train['user_number'].isin(common_users)].groupby('user_number').size()\n common_users_test_activity = test[test['user_number'].isin(common_users)].groupby('user_number').size()\n \n print(f\"\\nАктивность общих пользователей:\")\n print(f\" В train: mean={common_users_train_activity.mean():.1f}, max={common_users_train_activity.max()}\")\n print(f\" В test: mean={common_users_test_activity.mean():.1f}, max={common_users_test_activity.max()}\")\n\nanalyze_user_overlap(train, test)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:09.788081Z","iopub.execute_input":"2025-10-25T02:42:09.788392Z","iopub.status.idle":"2025-10-25T02:42:10.207358Z","shell.execute_reply.started":"2025-10-25T02:42:09.788370Z","shell.execute_reply":"2025-10-25T02:42:10.206227Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_product_overlap(train, test):\n print(\"\\n=== АНАЛИЗ ПЕРЕСЕЧЕНИЯ ТОВАРОВ TRAIN/TEST ===\")\n \n train_products = set(train['product_id'].unique())\n test_products = set(test['product_id'].unique())\n \n common_products = train_products & test_products\n unique_train_products = train_products - test_products\n unique_test_products = test_products - train_products\n \n print(f\"Уникальных товаров в train: {len(train_products):,}\")\n print(f\"Уникальных товаров в test: {len(test_products):,}\")\n print(f\"Общих товаров: {len(common_products):,}\")\n print(f\"Уникальных в train: {len(unique_train_products):,}\")\n print(f\"Уникальных в test: {len(unique_test_products):,}\")\n print(f\"Покрытие test товаров: {len(common_products)/len(test_products)*100:.1f}%\")\n \n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n \n overlap_sizes = [len(common_products), len(unique_train_products), len(unique_test_products)]\n overlap_labels = ['Общие товары', 'Только в train', 'Только в test']\n \n ax1.pie(overlap_sizes, labels=overlap_labels, autopct='%1.1f%%', startangle=90)\n ax1.set_title('Пересечение товаров между train и test')\n\n train_product_popularity = train.groupby('product_id').size()\n test_product_popularity = test.groupby('product_id').size()\n \n popularity_bins = [1, 2, 5, 10, 20, 50, 100, 1000, 10000]\n train_popularity_dist = pd.cut(train_product_popularity, bins=popularity_bins).value_counts().sort_index()\n test_popularity_dist = pd.cut(test_product_popularity, bins=popularity_bins).value_counts().sort_index()\n \n x = np.arange(len(train_popularity_dist))\n width = 0.35\n \n ax2.bar(x - width/2, train_popularity_dist.values, width, label='Train', alpha=0.7)\n ax2.bar(x + width/2, test_popularity_dist.values, width, label='Test', alpha=0.7)\n ax2.set_xlabel('Количество взаимодействий с товаром')\n ax2.set_ylabel('Количество товаров')\n ax2.set_title('Распределение товаров по популярности')\n ax2.set_xticks(x)\n ax2.set_xticklabels([str(bin_range) for bin_range in train_popularity_dist.index])\n ax2.legend()\n ax2.tick_params(axis='x', rotation=45)\n \n plt.tight_layout()\n plt.show()\n\n if len(common_products) > 0:\n common_products_train_popularity = train[train['product_id'].isin(common_products)].groupby('product_id').size()\n common_products_test_popularity = test[test['product_id'].isin(common_products)].groupby('product_id').size()\n \n print(f\"\\nПопулярность общих товаров:\")\n print(f\" В train: mean={common_products_train_popularity.mean():.1f}, max={common_products_train_popularity.max()}\")\n print(f\" В test: mean={common_products_test_popularity.mean():.1f}, max={common_products_test_popularity.max()}\")\n\n# Вызов функции\nanalyze_product_overlap(train, test)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:10.210561Z","iopub.execute_input":"2025-10-25T02:42:10.210948Z","iopub.status.idle":"2025-10-25T02:42:10.623905Z","shell.execute_reply.started":"2025-10-25T02:42:10.210916Z","shell.execute_reply":"2025-10-25T02:42:10.622613Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_product_positions_with_plot(train, test):\n print(\"\\n=== АНАЛИЗ ПОЗИЦИЙ ТОВАРОВ И ГЛУБИНЫ ВЫДАЧИ ===\")\n \n position_stats_train = train.groupby('search_number')['product_position'].max().describe()\n position_stats_test = test.groupby('search_number')['product_position'].max().describe()\n \n print(\"Максимальные позиции в сессиях:\")\n print(f\"Train - mean: {position_stats_train['mean']:.1f}, std: {position_stats_train['std']:.1f}, max: {position_stats_train['max']}\")\n print(f\"Test - mean: {position_stats_test['mean']:.1f}, std: {position_stats_test['std']:.1f}, max: {position_stats_test['max']}\")\n \n fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))\n \n position_bins = np.arange(0, 251, 5)\n ax1.hist(train['product_position'], bins=position_bins, alpha=0.7, label='Train', density=True)\n ax1.hist(test['product_position'], bins=position_bins, alpha=0.7, label='Test', density=True)\n ax1.set_xlabel('Позиция товара')\n ax1.set_ylabel('Плотность распределения')\n ax1.set_title('Распределение позиций товаров')\n ax1.legend()\n ax1.grid(True, alpha=0.3)\n\n train_sorted = np.sort(train['product_position'])\n test_sorted = np.sort(test['product_position'])\n train_cdf = np.arange(1, len(train_sorted)+1) / len(train_sorted)\n test_cdf = np.arange(1, len(test_sorted)+1) / len(test_sorted)\n \n ax2.plot(train_sorted, train_cdf, label='Train', linewidth=2)\n ax2.plot(test_sorted, test_cdf, label='Test', linewidth=2)\n ax2.set_xlabel('Позиция товара')\n ax2.set_ylabel('CDF')\n ax2.set_title('CDF распределения позиций товаров')\n ax2.legend()\n ax2.grid(True, alpha=0.3)\n ax2.set_xlim(0, 250)\n \n plt.tight_layout()\n plt.show()\n\n quantiles = [0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]\n train_quantiles = train['product_position'].quantile(quantiles)\n test_quantiles = test['product_position'].quantile(quantiles)\n \n print(\"\\nКвантили распределения позиций:\")\n for q in quantiles:\n print(f\" {q:.0%}: train={train_quantiles[q]:.1f}, test={test_quantiles[q]:.1f}\")\n\nanalyze_product_positions_with_plot(train, test)","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:10.624947Z","iopub.execute_input":"2025-10-25T02:42:10.625236Z","iopub.status.idle":"2025-10-25T02:42:11.286420Z","shell.execute_reply.started":"2025-10-25T02:42:10.625215Z","shell.execute_reply":"2025-10-25T02:42:11.285238Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Анализ целевой переменной","metadata":{}},{"cell_type":"code","source":"sessions_with_engagement = train.groupby('search_number')['relevance'].max()\nprint(f\"\\nСессии с взаимодействиями: {(sessions_with_engagement >= 1).mean():.1%}\")\nprint(f\"Сессии с корзинами: {(sessions_with_engagement >= 2).mean():.1%}\")\nprint(f\"Сессии с покупками: {(sessions_with_engagement == 3).mean():.1%}\")\n\nplt.figure(figsize=(10, 6))\nsns.countplot(data=train, x='relevance')\nplt.title('Распределение релевантности', fontsize=16)\nplt.xlabel('Релевантность')\nplt.ylabel('Количество')\nplt.show()","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:11.287546Z","iopub.execute_input":"2025-10-25T02:42:11.287900Z","iopub.status.idle":"2025-10-25T02:42:11.469592Z","shell.execute_reply.started":"2025-10-25T02:42:11.287869Z","shell.execute_reply":"2025-10-25T02:42:11.468500Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pos_relevance = train.groupby('product_position')['relevance'].mean()\nplt.figure(figsize=(12, 6))\npos_relevance.plot()\nplt.title('Средняя релевантность по позициям товаров')\nplt.xlabel('Позиция товара')\nplt.ylabel('Средняя relevance')\nplt.show()","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:11.470665Z","iopub.execute_input":"2025-10-25T02:42:11.471015Z","iopub.status.idle":"2025-10-25T02:42:11.683056Z","shell.execute_reply.started":"2025-10-25T02:42:11.470987Z","shell.execute_reply":"2025-10-25T02:42:11.682034Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"relevant_items = train[train['relevance'] > 0]\nprint(f\"\\nПозиции релевантных товаров:\")\nprint(relevant_items['product_position'].describe().round(1).to_frame().T)\nprint('\\n')\n\nfirst_ten_positions = len(relevant_items[relevant_items['product_position'] <= 10])\nten_to_twelve_positions = len(relevant_items[relevant_items['product_position'] <= 20]) - first_ten_positions\nlater_positions = len(relevant_items[relevant_items['product_position'] > 20])\n\n\nprint(f\"Релевантных товаров в первых 10 позициях: {first_ten_positions} ({first_ten_positions/len(relevant_items):.1%})\")\nprint(f\"Релевантных товаров с 10 до 20 позиции: {ten_to_twelve_positions} ({ten_to_twelve_positions/len(relevant_items):.1%})\")\nprint(f\"Релевантных товаров после 20 позиции: {later_positions} ({later_positions/len(relevant_items):.1%})\")","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:11.684012Z","iopub.execute_input":"2025-10-25T02:42:11.684299Z","iopub.status.idle":"2025-10-25T02:42:11.705230Z","shell.execute_reply.started":"2025-10-25T02:42:11.684278Z","shell.execute_reply":"2025-10-25T02:42:11.704217Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Анализ пользовательских паттернов","metadata":{}},{"cell_type":"code","source":"train['day_of_week'] = train['date'].dt.dayofweek\ntrain['day_of_month'] = train['date'].dt.day\n\ndef analyze_user_weekday_patterns(df, top_users=10000):\n top_users_list = df['user_number'].value_counts().head(top_users).index\n results = []\n \n for user in top_users_list:\n user_data = df[df['user_number'] == user]\n weekday_relevance = user_data.groupby(['day_of_week', 'relevance']).size().unstack(fill_value=0)\n \n for rel in [0,1,2,3]:\n if rel not in weekday_relevance.columns:\n weekday_relevance[rel] = 0\n \n for day in range(7):\n if day in weekday_relevance.index:\n day_row = weekday_relevance.loc[day]\n total = day_row.sum()\n if total > 0:\n results.append({\n 'user_number': user,\n 'day_of_week': day,\n 'purchase_rate': day_row.get(3, 0) / total,\n 'click_rate': day_row.get(1, 0) / total,\n 'cart_rate': day_row.get(2, 0) / total,\n 'view_rate': day_row.get(0, 0) / total,\n 'total_actions': total,\n 'day_name': ['Пн','Вт','Ср','Чт','Пт','Сб','Вс'][day]\n })\n \n return pd.DataFrame(results)\n\ndef analyze_user_monthday_patterns(df, top_users=10000):\n top_users_list = df['user_number'].value_counts().head(top_users).index\n results = []\n \n for user in top_users_list:\n user_data = df[df['user_number'] == user]\n monthday_relevance = user_data.groupby(['day_of_month', 'relevance']).size().unstack(fill_value=0)\n \n for rel in [0,1,2,3]:\n if rel not in monthday_relevance.columns:\n monthday_relevance[rel] = 0\n \n for day in range(1, 32):\n if day in monthday_relevance.index:\n day_row = monthday_relevance.loc[day]\n total = day_row.sum()\n if total > 0:\n results.append({\n 'user_number': user,\n 'day_of_month': day,\n 'purchase_rate': day_row.get(3, 0) / total,\n 'click_rate': day_row.get(1, 0) / total,\n 'cart_rate': day_row.get(2, 0) / total,\n 'view_rate': day_row.get(0, 0) / total,\n 'total_actions': total\n })\n \n return pd.DataFrame(results)\n\nweekday_analysis = analyze_user_weekday_patterns(train)\nmonthday_analysis = analyze_user_monthday_patterns(train)\n\nweekday_order = ['Пн', 'Вт', 'Ср', 'Чт', 'Пт', 'Сб', 'Вс']\nmetrics = ['purchase_rate', 'cart_rate', 'click_rate']\ntitles = ['Доля покупок', 'Доля корзин', 'Доля кликов']\n\nfig, axes = plt.subplots(2, 3, figsize=(18, 10))\n\nfor i, metric in enumerate(metrics):\n data = weekday_analysis.groupby('day_name')[metric].mean().reindex(weekday_order)\n axes[0,i].bar(data.index, data.values, alpha=0.7)\n axes[0,i].set_title(f'{titles[i]} по дням недели')\n axes[0,i].set_ylabel('Доля')\n axes[0,i].grid(axis='y', alpha=0.3)\n\nfor i, metric in enumerate(metrics):\n data = monthday_analysis.groupby('day_of_month')[metric].mean()\n axes[1,i].bar(data.index, data.values, alpha=0.7)\n axes[1,i].set_title(f'{titles[i]} по дням месяца')\n axes[1,i].set_xlabel('День месяца')\n axes[1,i].set_ylabel('Доля')\n axes[1,i].grid(axis='y', alpha=0.3)\n\nplt.tight_layout()\nplt.show()\n\ntrain = train.drop(columns=['day_of_week', 'day_of_month'])","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:11.706536Z","iopub.execute_input":"2025-10-25T02:42:11.707083Z","iopub.status.idle":"2025-10-25T02:42:13.093902Z","shell.execute_reply.started":"2025-10-25T02:42:11.707046Z","shell.execute_reply":"2025-10-25T02:42:13.092945Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"user_actions = train['user_number'].value_counts()\n\nplt.figure(figsize=(12, 6))\n\nplt.subplot(1, 2, 1)\nplt.hist(user_actions, bins=50, alpha=0.7, edgecolor='black')\nplt.title('Распределение количества действий по пользователям')\nplt.xlabel('Количество действий')\nplt.ylabel('Количество пользователей')\nplt.grid(alpha=0.3)\n\nplt.subplot(1, 2, 2)\nplt.boxplot(user_actions, vert=False)\nplt.title('Боксплот распределения действий')\nplt.xlabel('Количество действий')\nplt.grid(alpha=0.3)\n\nplt.tight_layout()\nplt.show()\n\nprint(f\"Всего пользователей: {len(user_actions)}\")\nprint(f\"Медиана действий на пользователя: {user_actions.median():.1f}\")\nprint(f\"Среднее действий на пользователя: {user_actions.mean():.1f}\")\nprint(f\"Максимум действий: {user_actions.max()}\")\nprint(f\"Минимум действий: {user_actions.min()}\")\nprint(f\"Пользователей с >100 действиями: {len(user_actions[user_actions > 100])}\")\nprint(f\"Пользователей с >30 действиями: {len(user_actions[user_actions > 30])}\")","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:13.097184Z","iopub.execute_input":"2025-10-25T02:42:13.097929Z","iopub.status.idle":"2025-10-25T02:42:13.601060Z","shell.execute_reply.started":"2025-10-25T02:42:13.097899Z","shell.execute_reply":"2025-10-25T02:42:13.600037Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Анализ поисковых запросов","metadata":{}},{"cell_type":"code","source":"def analyze_queries_overlap(train_sessions, test_sessions):\n \"\"\"Анализ пересечения уникальных запросов между train и test\"\"\"\n\n train_queries = set(train_sessions['query'].dropna().unique())\n test_queries = set(test_sessions['query'].dropna().unique())\n\n print(\"=== АНАЛИЗ ПЕРЕСЕЧЕНИЯ ЗАПРОСОВ ===\")\n print(f\"Уникальных запросов в train: {len(train_queries):,}\")\n print(f\"Уникальных запросов в test: {len(test_queries):,}\")\n print(f\"Всего уникальных запросов всего: {len(train_queries | test_queries):,}\")\n\n common_queries = train_queries & test_queries\n unique_test_queries = test_queries - train_queries\n unique_train_queries = train_queries - test_queries\n \n print(f\"\\n--- Пересечение запросов ---\")\n print(f\"Общих запросов (train ∩ test): {len(common_queries):,}\")\n print(f\"Уникальных запросов в test (не в train): {len(unique_test_queries):,}\")\n\n plt.figure(figsize=(5, 4))\n overlap_sizes = [len(common_queries), len(train_queries - test_queries), len(unique_test_queries)]\n overlap_labels = ['Общие запросы', 'Только в train', 'Только в test']\n \n plt.pie(overlap_sizes, autopct=lambda p: f'{p:.1f}%' if p > 5 else '', \n startangle=90, textprops={'fontsize': 12})\n plt.title('Пересечение запросов между train и test', fontsize=12)\n plt.legend(overlap_labels, title=\"Категории\", loc=\"center left\", bbox_to_anchor=(1, 0, 0.5, 1))\n plt.tight_layout()\n plt.show()\n\n if len(test_queries) > 0:\n coverage_percent = len(common_queries) / len(test_queries) * 100\n novelty_percent = len(unique_test_queries) / len(test_queries) * 100\n print(f\"\\n--- Процент покрытия ---\")\n print(f\"Запросов из test, которые есть в train: {coverage_percent:.2f}%\")\n print(f\"Новых запросов в test (неизвестных из train): {novelty_percent:.2f}%\")\n \n if len(unique_test_queries) > 0:\n print(f\"Примеры новых запросов в test (первые 10): {list(unique_test_queries)[:10]}\")\n \n train_query_lengths = [len(str(q)) for q in train_queries]\n test_query_lengths = [len(str(q)) for q in test_queries]\n \n print(f\"\\n--- Статистика длины запросов ---\")\n print(f\"Train - Средняя длина: {np.mean(train_query_lengths):.1f}, \"\n f\"Мин: {np.min(train_query_lengths)}, Макс: {np.max(train_query_lengths)}\")\n print(f\"Test - Средняя длина: {np.mean(test_query_lengths):.1f}, \"\n f\"Мин: {np.min(test_query_lengths)}, Макс: {np.max(test_query_lengths)}\")\n \n return {\n 'train_queries': train_queries,\n 'test_queries': test_queries,\n 'common_queries': common_queries,\n 'unique_test_queries': unique_test_queries,\n 'coverage_percent': coverage_percent if len(test_queries) > 0 else 0,\n 'novelty_percent': novelty_percent if len(test_queries) > 0 else 0\n }\n\nresults = analyze_queries_overlap(train, test)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:13.602096Z","iopub.execute_input":"2025-10-25T02:42:13.602581Z","iopub.status.idle":"2025-10-25T02:42:13.747362Z","shell.execute_reply.started":"2025-10-25T02:42:13.602546Z","shell.execute_reply":"2025-10-25T02:42:13.746463Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_query_quality(train):\n print(\"\\n=== АНАЛИЗ КАЧЕСТВА ЗАПРОСОВ И ОШИБОК ===\")\n\n all_words = []\n for query in train['query'].dropna():\n words = query.lower().split()\n all_words.extend(words)\n \n word_freq = Counter(all_words)\n\n low_freq_words = [word for word, count in word_freq.items() if count <= 3 and len(word) > 4]\n \n print(f\"Потенциальных опечаток (слово встречается <= 3 раз): {len(low_freq_words):,}\")\n print(f\"Примеры потенциальных опечаток: {low_freq_words[:20]}\")\n\n query_stats = train.groupby('query').agg({\n 'relevance': ['count', lambda x: (x >= 2).mean()]\n }).round(4)\n query_stats.columns = ['count', 'conversion_rate']\n \n low_conversion_queries = query_stats[(query_stats['count'] >= 10) & (query_stats['conversion_rate'] == 0)]\n high_conversion_queries = query_stats[(query_stats['count'] >= 10) & (query_stats['conversion_rate'] >= 0.1)]\n \n print(f\"\\nЗапросы с нулевой конверсией (>=10 показов): {len(low_conversion_queries):,}\")\n print(f\"Запросы с высокой конверсией (>=10%): {len(high_conversion_queries):,}\")\n \n short_queries = train[train['query'].str.len() <= 2]\n long_queries = train[train['query'].str.len() >= 50]\n \n print(f\"\\nЭкстремальные запросы:\")\n print(f\" Очень короткие (<=2 символа): {len(short_queries):,}\")\n print(f\" Очень длинные (>=50 символов): {len(long_queries):,}\")\n \n if len(short_queries) > 0:\n print(f\" Примеры коротких: {short_queries['query'].unique()[:10]}\")\n if len(long_queries) > 0:\n print(f\" Примеры длинных: {[q[:50] + '...' for q in long_queries['query'].unique()[:5]]}\")\n\nanalyze_query_quality(train)","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:13.748490Z","iopub.execute_input":"2025-10-25T02:42:13.748911Z","iopub.status.idle":"2025-10-25T02:42:13.781072Z","shell.execute_reply.started":"2025-10-25T02:42:13.748881Z","shell.execute_reply":"2025-10-25T02:42:13.780045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_query_semantics(train):\n all_queries = train['query'].dropna().str.lower()\n stop_words = ['купить', 'цена', 'отзывы', 'характеристики', 'фото']\n \n print(f\"Запросы со стоп-словами:\")\n for stop_word in stop_words:\n matches = all_queries[all_queries.str.contains(stop_word, regex=False)]\n print(f\" '{stop_word}': {len(matches):,}\")\n\nanalyze_query_semantics(train)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:13.782139Z","iopub.execute_input":"2025-10-25T02:42:13.782528Z","iopub.status.idle":"2025-10-25T02:42:13.794037Z","shell.execute_reply.started":"2025-10-25T02:42:13.782497Z","shell.execute_reply":"2025-10-25T02:42:13.792970Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_all_special_chars(queries):\n pattern = r'[^\\w\\s]'\n \n all_chars = []\n for query in queries:\n special_chars = re.findall(pattern, str(query))\n all_chars.extend(special_chars)\n \n return Counter(all_chars)\n\nchar_counter = get_all_special_chars(train['query'])\n\nprint(\"=== СПЕЦСИМВОЛЫ В ЗАПРОСАХ === \")\nfor char, count in char_counter.most_common():\n print(f\"'{char}': {count} раз\")\n\nprint(f\"\\nВСЕГО УНИКАЛЬНЫХ СПЕЦСИМВОЛОВ: {len(char_counter)}\")","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:13.795171Z","iopub.execute_input":"2025-10-25T02:42:13.795482Z","iopub.status.idle":"2025-10-25T02:42:13.819210Z","shell.execute_reply.started":"2025-10-25T02:42:13.795448Z","shell.execute_reply":"2025-10-25T02:42:13.818212Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Анализ товаров","metadata":{}},{"cell_type":"code","source":"def analyze_products_and_replacements(products):\n print(\"\\n=== АНАЛИЗ ТОВАРНЫХ ХАРАКТЕРИСТИК ===\")\n \n print(f\"Уникальные бренды: {products['brand'].nunique()}\")\n print(f\"Уникальные типы товаров: {products['type'].nunique()}\")\n print(f\"Уникальные модели: {products['model'].nunique()}\")\n \n replacements = products[products['replaced_by'] > 0]\n print(f\"\\nТоваров с заменой: {len(replacements)} ({len(replacements)/len(products)*100:.1f}%)\")\n\nanalyze_products_and_replacements(products)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:13.820171Z","iopub.execute_input":"2025-10-25T02:42:13.820463Z","iopub.status.idle":"2025-10-25T02:42:13.865419Z","shell.execute_reply.started":"2025-10-25T02:42:13.820442Z","shell.execute_reply":"2025-10-25T02:42:13.864388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_products_per_search(df):\n products_per_search = df.groupby('search_number').size()\n total_searches = len(products_per_search)\n \n ranges = [\n (1, 5),\n (5, 10),\n (10, 15),\n (15, 19),\n (20, 20),\n (21, 40),\n (40, 100),\n (100, float('inf'))\n ]\n \n labels = [\n '1-5 товаров в выдаче',\n '5-10 товаров в выдаче', \n '10-15 товаров в выдаче',\n '15-19 товаров',\n 'ровно 20 товаров в выдаче',\n '21-40 товаров',\n '40+ товаров',\n '100+ товаров'\n ]\n \n distribution_stats = []\n \n for i, ((start, end), label) in enumerate(zip(ranges, labels)):\n if start == end:\n count = (products_per_search == start).sum()\n else:\n count = ((products_per_search >= start) & (products_per_search < end)).sum()\n \n percentage = (count / total_searches) * 100\n distribution_stats.append({\n 'range_label': label,\n 'count': count,\n 'percentage': percentage\n })\n \n return products_per_search, distribution_stats, total_searches\n\nproducts_per_search, distribution_stats, total_searches = analyze_products_per_search(train)\n\nplt.figure(figsize=(12, 6))\n\nplt.subplot(1, 2, 1)\nplt.hist(products_per_search, bins=50, alpha=0.7, color='skyblue', edgecolor='black')\nplt.title('Распределение товаров по поисковым выдачам')\nplt.xlabel('Количество товаров в search_number')\nplt.ylabel('Количество выдачей')\nplt.grid(alpha=0.3)\n\nplt.axvline(products_per_search.mean(), color='red', linestyle='--', linewidth=2, \n label=f'Среднее: {products_per_search.mean():.1f}')\nplt.axvline(products_per_search.median(), color='green', linestyle='--', linewidth=2, \n label=f'Медиана: {products_per_search.median()}')\nplt.legend()\n\nplt.subplot(1, 2, 2)\nplt.boxplot(products_per_search, vert=False)\nplt.title('Боксплот распределения товаров по выдачам')\nplt.xlabel('Количество товаров')\nplt.grid(alpha=0.3)\n\nplt.tight_layout()\nplt.show()\n\nprint(\"\\n=== СТАТИСТИКА РАСПРЕДЕЛЕНИЯ ТОВАРОВ ПО ВЫДАЧАМ ===\")\n\nfor stat in distribution_stats:\n print(f\"{stat['range_label']}: {stat['percentage']:.2f}% ({stat['count']} шт.)\")\n\nprint(\"=\" * 50)\nprint(f\"Всего уникальных сессий: {total_searches}\")\nprint(f\"Медиана товаров: {products_per_search.median():.1f}\")\nprint(f\"Среднее товаров: {products_per_search.mean():.1f}\")\nprint(f\"Максимум товаров: {products_per_search.max()}\")\nprint(f\"Минимум товаров: {products_per_search.min()}\")\nprint(f\"Выдачей с >20 товарами: {len(products_per_search[products_per_search > 20])}\")\nprint(f\"Выдачей с >40 товарами: {len(products_per_search[products_per_search > 40])}\")","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:13.866485Z","iopub.execute_input":"2025-10-25T02:42:13.866836Z","iopub.status.idle":"2025-10-25T02:42:14.355941Z","shell.execute_reply.started":"2025-10-25T02:42:13.866806Z","shell.execute_reply":"2025-10-25T02:42:14.355037Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_replacement_graph(products):\n print(\"\\n=== АНАЛИЗ ТОВАРНОГО ГРАФА ЗАМЕН ===\")\n \n replacements = products[products['replaced_by'].notna()].copy()\n \n if len(replacements) > 0:\n replacement_chains = {}\n for product_id in replacements['product_id']:\n chain_length = 0\n current_id = product_id\n \n while current_id in replacements['product_id'].values:\n current_id = replacements[replacements['product_id'] == current_id]['replaced_by'].iloc[0]\n chain_length += 1\n if chain_length > 10:\n break\n \n replacement_chains[product_id] = chain_length\n \n chain_lengths = list(replacement_chains.values())\n print(f\"Максимальная длина цепочки замен: {max(chain_lengths) if chain_lengths else 0}\")\n print(f\"Средняя длина цепочки: {np.mean(chain_lengths):.2f}\")\n \n products_with_dates = products[products['date_of_create'].notna()].copy()\n products_with_dates['date_of_create'] = pd.to_datetime(products_with_dates['date_of_create'])\n \n replacement_pairs = products_with_dates.merge(\n products_with_dates[['product_id', 'date_of_create']],\n left_on='replaced_by',\n right_on='product_id',\n suffixes=('_old', '_new')\n )\n \n if len(replacement_pairs) > 0:\n replacement_pairs['days_between'] = (replacement_pairs['date_of_create_new'] - replacement_pairs['date_of_create_old']).dt.days\n print(f\"Среднее время между заменами: {replacement_pairs['days_between'].mean():.1f} дней\")\n\nanalyze_replacement_graph(products)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:14.357205Z","iopub.execute_input":"2025-10-25T02:42:14.357485Z","iopub.status.idle":"2025-10-25T02:42:28.915201Z","shell.execute_reply.started":"2025-10-25T02:42:14.357464Z","shell.execute_reply":"2025-10-25T02:42:28.914297Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_merged = products.merge(\n products,\n left_on='replaced_by',\n right_on='product_id',\n suffixes=('_old', '_new')\n)\n\ncomparison_columns = ['type', 'model', 'brand', 'country'] + \\\n [f'category_name_{i}' for i in range(1, 5)]\n\ndetailed_mismatches = []\n\nfor col in comparison_columns:\n col_old = f'{col}_old'\n col_new = f'{col}_new'\n \n mismatches = df_merged[\n df_merged[col_old].astype(str).str.lower() != df_merged[col_new].astype(str).str.lower()\n ]\n \n for _, row in mismatches.iterrows():\n detailed_mismatches.append({\n 'column': col,\n 'product_id_old': row['product_id_old'],\n 'product_id_new': row['product_id_new'],\n 'old_value': row[col_old],\n 'new_value': row[col_new]\n })\n\ndetailed_mismatch_df = pd.DataFrame(detailed_mismatches)\n\nprint(\"=== ИЗМЕНЕНИЯ ДЛЯ КАЖДОГО ВИДА ЗАМЕН ===\")\n\nif not detailed_mismatch_df.empty:\n for column in detailed_mismatch_df['column'].unique():\n column_data = detailed_mismatch_df[detailed_mismatch_df['column'] == column]\n \n print(f\"\\n┌── {column.upper()} ── ({len(column_data)} изменений)\")\n print(f\"├{'─' * 78}\")\n \n for i, (_, row) in enumerate(column_data.head(10).iterrows()):\n print(f\"│ {row['product_id_old']} → {row['product_id_new']}: '{row['old_value']}' → '{row['new_value']}'\")\n \n if len(column_data) > 10:\n print(f\"│ ... и еще {len(column_data) - 10} изменений\")\n \n print(f\"└{'─' * 78}\")\n\n print(f\"\\nОБЩАЯ СТАТИСТИКА:\")\n print(\"-\" * 50)\n summary = detailed_mismatch_df.groupby('column').size()\n for col, count in summary.items():\n print(f\" {col}: {count} изменений\")\n \n print(f\"\\nВсего найдено: {len(detailed_mismatch_df)} изменений\")","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:28.916168Z","iopub.execute_input":"2025-10-25T02:42:28.916428Z","iopub.status.idle":"2025-10-25T02:42:29.001744Z","shell.execute_reply.started":"2025-10-25T02:42:28.916408Z","shell.execute_reply":"2025-10-25T02:42:29.000702Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def check_sequence_percent(df):\n total_groups = df['search_number'].nunique()\n \n problem_groups = 0\n for _, group in df.groupby('search_number'):\n positions = sorted(group['product_position'])\n if positions != list(range(positions[0], positions[0] + len(positions))):\n problem_groups += 1\n \n percent = (problem_groups / total_groups) * 100\n print(f'Групп со скачками позиций товаров в выдаче (на тесте): {percent:.1f}%')\n\nresult_df = check_sequence_percent(test)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:29.002678Z","iopub.execute_input":"2025-10-25T02:42:29.002952Z","iopub.status.idle":"2025-10-25T02:42:29.016271Z","shell.execute_reply.started":"2025-10-25T02:42:29.002932Z","shell.execute_reply":"2025-10-25T02:42:29.015230Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Анализ других переменных","metadata":{}},{"cell_type":"code","source":"train_device = train['device'].value_counts(normalize=True)\ntest_device = test['device'].value_counts(normalize=True)\n\nfig, ax = plt.subplots(figsize=(10, 6))\n\ndevices = train_device.index\nx = range(len(devices))\nwidth = 0.35\n\nax.bar([i - width/2 for i in x], train_device.values, width, label='Train', alpha=0.8)\nax.bar([i + width/2 for i in x], [test_device.get(dev, 0) for dev in devices], width, label='Test', alpha=0.8)\n\nax.set_xlabel('Устройства')\nax.set_ylabel('Доля (%)')\nax.set_title('Распределение по устройствам: Train vs Test')\nax.set_xticks(x)\nax.set_xticklabels(devices, rotation=45)\nax.legend()\nax.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:29.017316Z","iopub.execute_input":"2025-10-25T02:42:29.017548Z","iopub.status.idle":"2025-10-25T02:42:29.266349Z","shell.execute_reply.started":"2025-10-25T02:42:29.017530Z","shell.execute_reply":"2025-10-25T02:42:29.265350Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_feature_interactions(train):\n print(\"\\n=== АНАЛИЗ ВЛИЯНИЯ ПРИЗНАКОВ ===\")\n\n filters_effect = train.groupby('filters_applied')['relevance'].apply(lambda x: (x >= 2).mean())\n print(\"Влияние фильтров на конверсию:\")\n print(f\" Без фильтров: {filters_effect[0]:.3f}\")\n print(f\" С фильтрами: {filters_effect[1]:.3f}\")\n\n device_effect = train.groupby('device')['relevance'].apply(lambda x: (x >= 1).mean())\n print(\"\\nCTR по устройствам:\")\n for device, ctr in device_effect.items():\n print(f\" {device}: {ctr:.3f}\")\n\n print(\"\\nВзаимодействия в зависимости от цены:\")\n train['price_byn'] = train['price'] / 100\n price_bins = pd.qcut(train['price_byn'], q=20, duplicates='drop')\n price_clicks = train.groupby(price_bins)['relevance'].apply(lambda x: (x >= 1).mean())\n price_carts = train.groupby(price_bins)['relevance'].apply(lambda x: (x >= 2).mean())\n price_conversion = train.groupby(price_bins)['relevance'].apply(lambda x: (x == 3).mean())\n\n fig, axes = plt.subplots(3, 1, figsize=(12, 12))\n \n price_clicks.plot(kind='bar', ax=axes[0], title='Клики (relevance ≥ 1)')\n price_carts.plot(kind='bar', ax=axes[1], title='Добавления в корзину (relevance ≥ 2)')\n price_conversion.plot(kind='bar', ax=axes[2], title='Конверсия (relevance = 3)')\n \n for ax in axes:\n ax.tick_params(axis='x', rotation=45)\n \n plt.tight_layout()\n plt.show()\n train = train.drop(columns=['price_byn'])\n\nanalyze_feature_interactions(train)","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:29.267588Z","iopub.execute_input":"2025-10-25T02:42:29.267956Z","iopub.status.idle":"2025-10-25T02:42:30.119437Z","shell.execute_reply.started":"2025-10-25T02:42:29.267929Z","shell.execute_reply":"2025-10-25T02:42:30.118470Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def analyze_query_frequency_and_conversion(train):\n print(\"\\n=== АНАЛИЗ ЧАСТОТНОСТИ ЗАПРОСОВ И КОНВЕРСИИ ===\")\n \n query_freq = train.groupby('query')['search_number'].nunique()\n \n print(\"Статистики частотности запросов:\")\n print(f\" Самый частый запрос: '{query_freq.idxmax()}' - {query_freq.max()} раз\")\n print(f\" Медианная частота: {query_freq.median()}\")\n print(f\" Запросы с частотой 1: {sum(query_freq == 1):,} ({sum(query_freq == 1)/len(query_freq)*100:.1f}%)\")\n\n freq_bins = [1, 2, 5, 10, 50, 100, 1000, float('inf')]\n freq_labels = ['1', '2-5', '6-10', '11-50', '51-100', '101-1000', '1000+']\n \n train['query_freq'] = train['query'].map(query_freq)\n train['freq_group'] = pd.cut(train['query_freq'], bins=freq_bins, labels=freq_labels)\n \n freq_clicks = train.groupby('freq_group')['relevance'].apply(lambda x: (x >= 1).mean())\n freq_carts = train.groupby('freq_group')['relevance'].apply(lambda x: (x >= 2).mean())\n freq_conversion = train.groupby('freq_group')['relevance'].apply(lambda x: (x == 3).mean())\n \n fig, axes = plt.subplots(3, 1, figsize=(12, 12))\n \n freq_clicks.plot(kind='bar', ax=axes[0], title='Клики по частотности запросов (relevance ≥ 1)')\n freq_carts.plot(kind='bar', ax=axes[1], title='Добавления в корзину по частотности запросов (relevance ≥ 2)')\n freq_conversion.plot(kind='bar', ax=axes[2], title='Конверсия по частотности запросов (relevance = 3)')\n\n for ax in axes:\n ax.tick_params(axis='x', rotation=45)\n ax.set_ylabel('Доля')\n \n plt.tight_layout()\n plt.show()\n\n train = train.drop(columns=['query_freq', 'freq_group'])\n \n \nanalyze_query_frequency_and_conversion(train)","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:30.120558Z","iopub.execute_input":"2025-10-25T02:42:30.120835Z","iopub.status.idle":"2025-10-25T02:42:30.737586Z","shell.execute_reply.started":"2025-10-25T02:42:30.120814Z","shell.execute_reply":"2025-10-25T02:42:30.736589Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"corr_matrix = train[numeric_cols + ['relevance']].corr()\n\nplt.figure(figsize=(10, 8))\nsns.heatmap(corr_matrix, annot=True, cmap='coolwarm', center=0)\nplt.title('Корреляционная матрица')\nplt.show()","metadata":{"trusted":true,"_kg_hide-input":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:30.738801Z","iopub.execute_input":"2025-10-25T02:42:30.739094Z","iopub.status.idle":"2025-10-25T02:42:31.110047Z","shell.execute_reply.started":"2025-10-25T02:42:30.739056Z","shell.execute_reply":"2025-10-25T02:42:31.109201Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def mutual_info_matrix(df, target_col):\n X = df.drop(columns=[target_col, 'date'])\n y = df[target_col]\n X_encoded = X.copy()\n\n object_cols = X.select_dtypes(include=['object']).columns\n for col in object_cols:\n X_encoded[col] = X[col].astype('category')\n\n categorical_features = X_encoded.select_dtypes(include=['category']).columns\n numeric_features = X_encoded.select_dtypes(include=[np.number]).columns\n\n for col in categorical_features:\n X_encoded[col] = X_encoded[col].cat.codes\n\n discrete_mask = [col in categorical_features for col in X_encoded.columns]\n\n mi_scores = mutual_info_classif(\n X_encoded, \n y, \n discrete_features=discrete_mask,\n random_state=config.SEED\n )\n \n mi_df = pd.DataFrame({'feature': X_encoded.columns, 'mutual_info': mi_scores})\n mi_df = mi_df.sort_values('mutual_info', ascending=False)\n \n return mi_df\n\nprint(\"\\n=== MUTUAL INFO С ЦЕЛЕВОЙ ПЕРЕМЕННОЙ ===\")\n#mi_results = mutual_info_matrix(train, 'relevance')\n#mi_results","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.111274Z","iopub.execute_input":"2025-10-25T02:42:31.111594Z","iopub.status.idle":"2025-10-25T02:42:31.122492Z","shell.execute_reply.started":"2025-10-25T02:42:31.111568Z","shell.execute_reply":"2025-10-25T02:42:31.121498Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"ВЫСОКАЯ ВАЖНОСТЬ (MI > 0.02)\n1. response_products - 0.181847\n2. query_freq - 0.029241\n3. price_is_super - 0.027395\n4. selected_category_id - 0.025361\n5. filters_applied - 0.023760\n6. query - 0.023155\n7. user_number - 0.022451\n8. search_number - 0.021968\n9. price_discount - 0.020071\n\nСРЕДНЯЯ ВАЖНОСТЬ (MI 0.01-0.02)\n1. product_position - 0.013504\n2. price - 0.011716\n3. price_byn - 0.011529\n4. model - 0.010364\n5. product_id - 0.009871\n\nНИЗКАЯ ВАЖНОСТЬ (MI < 0.01)\n1. date_of_create - 0.004392\n2. brand - 0.004319\n3. response_time - 0.003858\n4. type - 0.003265\n5. browse_time - 0.003194\n6. category_name_4 - 0.002646\n7. category_name_3 - 0.002646\n8. category_name_2 - 0.002472\n9. country - 0.002405\n10. category_name_1 - 0.002204\n11. freq_group - 0.000848\n12. device - 0.000692\n13. replaced_by - 0.000074\n14. available - 0.000000\n15. price_is_discounted - 0.000000","metadata":{}},{"cell_type":"markdown","source":"# Созданные для предобработки классы","metadata":{}},{"cell_type":"markdown","source":"## Предобработка таблиц","metadata":{}},{"cell_type":"markdown","source":"### ProductFeaturesProcessor\n**Назначение:** Обработка товарных замен и построение графа товаров\n\n**Методы:**\n- `prepare_products()` - подготовка данных товаров\n- `_build_product_mappings()` - построение маппингов замен\n- `_add_product_features()` - добавление характеристик к сессиям\n\n**Принципы работы:**\n- Поиск корневых товаров в цепочках замен\n- Определение самой ранней даты создания в цепочке\n- Привязка актуальных характеристик товаров к сессиям\n","metadata":{}},{"cell_type":"code","source":"class ProductFeaturesProcessor:\n \"\"\"Добавление характеристик товаров к сессиям\"\"\"\n \n def __init__(self):\n self.mappings = None\n self.products = None\n \n def prepare_products(self, products):\n \"\"\"Подготовка данных о товарах\"\"\"\n self.products = products.drop_duplicates(subset=['product_id'], keep='first')\n self.products['date_of_create'] = pd.to_datetime(self.products['date_of_create'])\n self.mappings = self._build_product_mappings()\n return self.products\n \n def _build_product_mappings(self):\n \"\"\"Построение маппингов товаров - находит самый поздний товар в цепочке для каждого product_id\"\"\"\n product_dict = self.products.set_index('product_id').to_dict('index')\n \n replacement_mapping = {}\n \n for product_id in self.products['product_id']:\n current_id = product_id\n visited = set()\n\n while (current_id in product_dict and \n pd.notna(product_dict[current_id].get('replaced_by')) and\n product_dict[current_id]['replaced_by'] in product_dict and\n product_dict[current_id]['replaced_by'] not in visited and\n product_dict[current_id]['replaced_by'] > 0):\n \n visited.add(current_id)\n current_id = product_dict[current_id]['replaced_by']\n\n replacement_mapping[product_id] = current_id\n \n return {\n 'product_dict': product_dict,\n 'replacement_mapping': replacement_mapping\n }\n \n \n def _add_product_features(self, sessions):\n sessions = sessions.copy()\n product_dict = self.mappings['product_dict']\n replacement_mapping = self.mappings['replacement_mapping']\n \n earliest_date_mapping = {}\n groups = {}\n \n for product_id, root_id in replacement_mapping.items():\n if root_id not in groups:\n groups[root_id] = []\n groups[root_id].append(product_id)\n \n for root_id, product_ids in groups.items():\n earliest_date = product_dict[root_id]['date_of_create']\n for pid in product_ids:\n current_date = product_dict[pid]['date_of_create']\n if current_date < earliest_date:\n earliest_date = current_date\n for pid in product_ids:\n earliest_date_mapping[pid] = earliest_date\n\n sessions['actual_product_id'] = sessions['product_id'].map(replacement_mapping).fillna(sessions['product_id'])\n \n products_df = pd.DataFrame.from_dict(product_dict, orient='index').reset_index()\n products_df = products_df.rename(columns={'index': 'product_id'})\n \n replacements_info = products_df[['product_id', 'replaced_by', 'date_of_create']].copy()\n replacements_info = replacements_info[replacements_info['replaced_by'].notna() & (replacements_info['replaced_by'] > 0)]\n \n sessions_with_replacements = sessions.merge(replacements_info, on='product_id', how='left')\n \n replacement_products = products_df[['product_id', 'date_of_create']].rename(\n columns={'product_id': 'replacement_id', 'date_of_create': 'date_of_create_replacement'}\n )\n \n sessions_with_replacements = sessions_with_replacements.merge(\n replacement_products, left_on='replaced_by', right_on='replacement_id', how='left'\n )\n \n condition_replacement = (\n sessions_with_replacements['replaced_by'].notna() & \n sessions_with_replacements['date_of_create_replacement'].notna() &\n (sessions_with_replacements['date_of_create_replacement'] <= sessions_with_replacements['date'])\n )\n \n sessions_with_replacements['product_id_for_features'] = np.where(\n condition_replacement,\n sessions_with_replacements['replaced_by'],\n sessions_with_replacements['product_id']\n )\n \n feature_columns = ['type', 'model', 'brand', 'country', 'category_name_1', 'category_name_2', 'category_name_3', 'category_name_4']\n \n all_product_features = []\n for product_id, product_data in product_dict.items():\n features = {'product_id': product_id}\n for col in feature_columns:\n features[col] = product_data.get(col, 'unknown')\n all_product_features.append(features)\n \n features_df_all = pd.DataFrame(all_product_features)\n \n sessions_with_features = sessions_with_replacements.merge(\n features_df_all, \n left_on='product_id_for_features', \n right_on='product_id', \n how='left',\n suffixes=('', '_features')\n )\n \n sessions_with_features['earliest_date_of_create'] = sessions_with_features['actual_product_id'].map(earliest_date_mapping)\n \n missing_mask = sessions_with_features['type'].isna()\n \n if missing_mask.any():\n sessions_missing = sessions_with_features[missing_mask].copy()\n sessions_missing = sessions_missing.drop(feature_columns, axis=1, errors='ignore')\n sessions_missing = sessions_missing.merge(\n features_df_all, \n left_on='actual_product_id',\n right_on='product_id', \n how='left', \n suffixes=('', '_backup')\n )\n sessions_with_features = pd.concat([sessions_with_features[~missing_mask], sessions_missing], ignore_index=True)\n \n for col in feature_columns:\n if col in sessions_with_features.columns:\n sessions_with_features[col] = sessions_with_features[col].fillna('unknown')\n \n columns_to_drop = [\n 'product_id', 'product_id_features', 'replaced_by', \n 'date_of_create_replacement', 'replacement_id', 'product_id_for_features',\n 'date_of_create'\n ]\n columns_to_drop = [col for col in columns_to_drop if col in sessions_with_features.columns]\n sessions_with_features = sessions_with_features.drop(columns=columns_to_drop)\n \n sessions_with_features = sessions_with_features.rename(\n columns={'earliest_date_of_create': 'date_of_create'}\n )\n \n return sessions_with_features\n \n def process(self, dfs):\n print(\"Добавление характеристик товаров...\")\n\n for i, df in enumerate(dfs):\n dfs[i] = self._add_product_features(df)\n print(\"Обработка завершена!\")\n return dfs","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.123556Z","iopub.execute_input":"2025-10-25T02:42:31.123829Z","iopub.status.idle":"2025-10-25T02:42:31.147285Z","shell.execute_reply.started":"2025-10-25T02:42:31.123810Z","shell.execute_reply":"2025-10-25T02:42:31.145934Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### DataSampler\n**Назначение:** Семплирование и разделение данных\n\n**Методы:**\n- `date_to_datetime()` - перевод столбца даты в datetime\n- `group_sampling()` - ограничение размера поисковых выдач\n- `filter_short_groups()` - фильтрация коротких выдач\n- `create_validation_split()` - создание валидационной выборки\n\n**Принципы работы:**\n- Валидация по временному сплиту (последние 2 месяца)\n- Удаление поисковых выдач с малым количеством товаров\n- Отбор N самых релевантных товаров для каждой выдачи","metadata":{}},{"cell_type":"code","source":"class DataSampler:\n \"\"\"Сэмплирование и разделение данных\"\"\"\n \n def __init__(self):\n self.min_group_size = config.SAMPLING_CONFIG['min_group_size']\n self.validation_group_size = config.SAMPLING_CONFIG['validation_group_size']\n self.validation_months = config.SAMPLING_CONFIG['validation_months']\n \n def date_to_datetime(self, dfs):\n for i, df in enumerate(dfs):\n dfs[i]['date'] = pd.to_datetime(df['date'])\n return dfs\n\n def group_sampling(self, df, group_size):\n \"\"\"Сэмплирование групп поисковых запросов\"\"\"\n df_sorted = df.sort_values(['search_number', 'relevance'], ascending=[True, False])\n result = df_sorted.groupby('search_number').head(group_size)\n \n print(\"Обработка больших групп.\")\n print(f\"Итоговый размер: {len(result):,} записей\")\n \n final_sizes = result.groupby('search_number').size()\n deep_pos_count = len(result[result['product_position'] >= group_size])\n \n print(f\"Групп: {len(final_sizes):,}\")\n print(f\"Позиции ≥{group_size}: {deep_pos_count:,} ({deep_pos_count/len(result)*100:.1f}%)\")\n print(f\"Максимальный размер группы: {final_sizes.max()}\")\n \n return result.reset_index(drop=True)\n \n def filter_short_groups(self, df, size=None):\n \"\"\"Фильтрация коротких выдач без создания валидационной выборки\"\"\"\n size = self.min_group_size if size is None else size\n df_filtered = df.groupby('search_number').filter(lambda x: len(x) > size)\n print(f\"После фильтрации коротких выдач: {len(df_filtered):,} записей\")\n return df_filtered\n \n def create_validation_split(self, train_df, months):\n \"\"\"Создание валидационной выборки из тренировочной\"\"\"\n latest_date = train_df['date'].max()\n validation_df = train_df[train_df['date'] >= latest_date - pd.DateOffset(months=months)].copy()\n validation_df = data_sampler.group_sampling(validation_df, self.validation_group_size)\n validation_df = data_sampler.filter_short_groups(validation_df, self.validation_group_size) \n train_df = train_df[train_df['date'] < latest_date - pd.DateOffset(months=months)]\n \n overall_length = len(train_df) + len(validation_df)\n print(f\"Размер обучающей выборки: {len(train_df):,} записей, {len(train_df) / overall_length * 100:.1f}%\")\n print(f\"Размер валидационной выборки: {len(validation_df):,} записей, {len(validation_df) / overall_length * 100:.1f}%\")\n \n return train_df, validation_df\n","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:31.148623Z","iopub.execute_input":"2025-10-25T02:42:31.148861Z","iopub.status.idle":"2025-10-25T02:42:31.176951Z","shell.execute_reply.started":"2025-10-25T02:42:31.148843Z","shell.execute_reply":"2025-10-25T02:42:31.175982Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Обработка текста","metadata":{}},{"cell_type":"markdown","source":"### TextPreprocessor\n**Назначение:** Предобработка текстовых данных\n\n**Компоненты:**\n- **Лемматизация** через pymorphy3 с кэшированием\n- **Нормализация** текста (нижний регистр, замена символов)\n- **Транслитерация** латинских названий в кириллицу\n\n**Принципы работы:**\n- Построение справочников брендов, моделей, типов товаров, категорий и стран по значениям из таблицы products для дальнейшей обработки спелл-чекером\n- Транслитерация по словарю специальных случаев: некоторых брендов и цветов\n- Обработка специальных паттернов: процентов, чисел с запятыми, размеров, плюсов\n- Разделение букв и чисел пробелом\n- Удаление всех не-буквенных и не-цифровых символов кроме \"+.-\"\n- Кэширование результатов для производительности","metadata":{}},{"cell_type":"code","source":"class TextPreprocessor:\n \"\"\"Предобработка текстовых данных\"\"\"\n \n def __init__(self):\n self.morph = pymorphy3.MorphAnalyzer()\n self._normalize_cache = {}\n self._transliterate_cache = {}\n\n \n @lru_cache(maxsize=10000)\n def _lemmatize_word(self, word):\n \"\"\"Лемматизация одного слова с кэшированием\"\"\"\n if WORD_PATTERNS['word'].match(word):\n return word\n parsed = self.morph.parse(word)[0]\n return parsed.normal_form\n \n def lemmatize_text(self, text):\n \"\"\"Лемматизация текста\"\"\"\n if not isinstance(text, str) or not text.strip():\n return text\n \n words = text.split()\n lemmas = []\n \n for word in words:\n if len(word) > config.TEXT_PROCESSING['lemmatize_min_length'] and WORD_PATTERNS['word'].match(word):\n lemmatized_word = self._lemmatize_word(word)\n lemmatized_word = lemmatized_word.replace('ё', 'е')\n lemmas.append(lemmatized_word)\n else:\n lemmas.append(word)\n \n return ' '.join(lemmas)\n \n @lru_cache(maxsize=10000)\n def normalize_text(self, text):\n \"\"\"Нормализация текста\"\"\"\n if not isinstance(text, str):\n return \"\" \n \n if not text.strip():\n return \"\" \n \n text = str(text).lower().strip() \n \n text = text.replace('ё', 'е').replace('і', 'и')\n text = WORD_PATTERNS['comma_numbers'].sub(r'\\1.\\2', text)\n text = WORD_PATTERNS['percent'].sub(r'\\1 процент', text)\n text = WORD_PATTERNS['size'].sub(r'\\1 x \\2', text)\n text = WORD_PATTERNS['plus'].sub(r'\\1 plus', text)\n text = WORD_PATTERNS['letters_digits'].sub(r'\\1 \\2', text)\n text = WORD_PATTERNS['digits_letters'].sub(r'\\1 \\2', text)\n text = WORD_PATTERNS['non_word'].sub(' ', text)\n text = WORD_PATTERNS['multi_space'].sub(' ', text) \n \n return text.strip()\n \n @lru_cache(maxsize=5000)\n def _transliterate_word(self, word):\n \"\"\"Транслитерация слова в кириллицу\"\"\"\n if not word:\n return word\n \n if word in SPECIAL_CASES:\n return SPECIAL_CASES[word]\n\n result = []\n i = 0\n while i < len(word):\n applied = False\n for pattern, replacement in TRANSLITERATION_RULES:\n pattern_len = len(pattern)\n if i + pattern_len <= len(word) and word[i:i+pattern_len] == pattern:\n result.append(replacement)\n i += pattern_len\n applied = True\n break\n \n if not applied:\n result.append(word[i])\n i += 1\n \n return ''.join(result)\n \n def process_column(self, df, column_name, lemmatize=False):\n \"\"\"Предобработка текстовой колонки\"\"\"\n print(f\"Предобработка колонки {column_name}...\")\n \n unique_values = df[column_name].dropna().unique()\n mapping_dict = {}\n \n for value in unique_values:\n if not isinstance(value, str) or not value.strip():\n mapping_dict[value] = value\n continue\n\n processed_words = []\n for word in value.split():\n if word.isdigit():\n processed_words.append(word)\n continue\n\n has_cyrillic = any('а' <= char <= 'я' or char == 'ё' for char in word.lower())\n has_latin = any('a' <= char <= 'z' for char in word.lower())\n\n if lemmatize and has_cyrillic and len(word) > config.TEXT_PROCESSING['lemmatize_min_length']:\n processed_word = self._lemmatize_word(word)\n else:\n processed_word = word\n\n if has_latin:\n processed_word = self._transliterate_word(processed_word)\n \n processed_words.append(processed_word)\n \n mapping_dict[value] = ' '.join(processed_words)\n\n df[column_name] = df[column_name].map(mapping_dict)\n return df\n \n def normalize_products(self, df):\n \"\"\"Нормализация таблицы товаров\"\"\"\n print(\"Нормализация таблицы товаров...\")\n \n text_columns = ['type', 'model', 'brand', 'country']\n text_columns += [f'category_name_{i}' for i in range(1, 5)]\n \n for col in text_columns:\n if col in df.columns:\n df[col] = df[col].astype(str).apply(self.normalize_text)\n\n brands = set(df['brand'].dropna().unique()) if 'brand' in df.columns else set()\n models = set(df['model'].dropna().unique()) if 'model' in df.columns else set()\n types = set(df['type'].dropna().unique()) if 'type' in df.columns else set()\n countries = set(df['country'].dropna().unique()) if 'country' in df.columns else set()\n\n brands_with_latin = [brand for brand in brands if WORD_PATTERNS['latin'].search(str(brand))]\n models_with_latin = [word for model in models for word in str(model).split(' ') if WORD_PATTERNS['latin'].search(word)]\n \n brands_transl_dict = {self._transliterate_word(brand): brand for brand in brands_with_latin}\n models_transl_dict = {self._transliterate_word(model): model for model in models_with_latin}\n\n categories = set()\n for i in range(1, 5):\n col = f'category_name_{i}'\n if col in df.columns:\n categories.update(df[col].dropna().unique())\n \n print(f\"Созданы справочники: brands({len(brands):,}), models({len(models):,}), categories({len(categories):,})\")\n return df, brands, models, categories, types, countries, brands_transl_dict, models_transl_dict","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:31.178510Z","iopub.execute_input":"2025-10-25T02:42:31.178831Z","iopub.status.idle":"2025-10-25T02:42:31.206531Z","shell.execute_reply.started":"2025-10-25T02:42:31.178800Z","shell.execute_reply":"2025-10-25T02:42:31.205576Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### SpellChecker\n**Назначение:** Коррекция орфографии в запросах\n\n**Методы:**\n- `train()` - обучение спелл-чекера на терминах из данных\n- `setup_spell_checkers()` - обучение спелл-чекеров бренда, модели и общего\n- `correct_query()` - коррекция всего запроса\n- `_change_keyboard_layout()` - смена раскладки клавиатуры для слова\n- `_get_correction()` - нахождение оптимального исправления для слова с учетом транслитерации или смены раскладки\n- `_get_suggested_word()` - получение исправления от спелл-чекера\n- `_change_keyboard_layout()` - смена раскладки\n\n**Принципы работы:**\n- Несколько спеллчекеров SymSpell для разных сущностей: бренды, модели, общий (бренды + модели + категории + типы + страны)\n- Учет раскладок клавиатуры (ru/en)\n- Адаптивное расстояние поиска исправления в зависимости от длины слова len(word) // 3\n\n","metadata":{}},{"cell_type":"code","source":"class SpellChecker:\n \"\"\"Коррекция орфографии\"\"\"\n \n def __init__(self, preprocessor):\n self.preprocessor = preprocessor\n self.spell_checkers = {}\n self.dicts = {}\n \n def train(self, terms_list, to_split=True):\n \"\"\"Обучение спеллчекера\"\"\"\n sym_spell = SymSpell(config.SPELL_CHECKER_CONFIG['max_edit_distance'],\n config.SPELL_CHECKER_CONFIG['prefix_length'])\n \n all_terms = set()\n for term_list in terms_list:\n for entry in term_list:\n if to_split:\n all_terms.update(entry.split(' '))\n else:\n all_terms.add(entry)\n \n for term in all_terms:\n sym_spell.create_dictionary_entry(term, 1)\n \n print(f\"Спеллчекер обучен на {len(all_terms):,} терминах\")\n return sym_spell\n \n def setup_spell_checkers(self, brands, models, categories, types, countries, brands_transl_dict, models_transl_dict):\n \"\"\"Настройка трех спеллчекеров\"\"\"\n self.spell_checkers = {\n 'brand': {\n 'original': self.train([brands]),\n 'translated': self.train([brands_transl_dict.keys()])\n },\n 'model': {\n 'original': self.train([brands, models]),\n 'translated': self.train([models_transl_dict.keys()])\n },\n 'general': {\n 'original': self.train([brands, models, categories, types, countries]),\n }\n }\n \n self.dicts = {\n 'brand': brands_transl_dict,\n 'model': models_transl_dict\n }\n \n print(\"Все спеллчекеры обучены!\")\n \n def _change_keyboard_layout(self, word):\n \"\"\"Смена раскладки клавиатуры\"\"\"\n cyrillic_total = sum(1 for char in word if char in CYRILLIC_CHARS)\n latin_total = sum(1 for char in word if char in LATIN_CHARS)\n \n if cyrillic_total > latin_total:\n mapping = KEYBOARD_LAYOUTS['ru_to_en']\n elif latin_total > cyrillic_total:\n mapping = KEYBOARD_LAYOUTS['en_to_ru']\n else:\n return word\n \n return ''.join(mapping.get(char, char) for char in word)\n \n def _get_suggested_word(self, word, spell_checker):\n \"\"\"Получение предложений от спеллчекера\"\"\"\n try:\n suggestions = spell_checker.lookup(word, Verbosity.CLOSEST, len(word) // 3)\n if suggestions:\n return suggestions[0].term, suggestions[0].distance\n else:\n return None, None \n except Exception as e:\n return None, None\n \n def _get_correction(self, word, option):\n \"\"\"Получение исправления для слова\"\"\"\n changed_word = self._change_keyboard_layout(word)\n \n suggestions = [\n list(self._get_suggested_word(word, self.spell_checkers[option]['original'])),\n list(self._get_suggested_word(changed_word, self.spell_checkers[option]['original']))\n ]\n \n if option != 'general':\n for source_word in [word, changed_word]:\n suggested = list(self._get_suggested_word(source_word, self.spell_checkers[option]['translated']))\n if suggested[0] is not None:\n suggested[0] = self.dicts[option].get(suggested[0])\n suggestions.append(suggested)\n \n valid_suggestions = [s for s in suggestions if s[0] is not None and s[1] is not None]\n \n if not valid_suggestions:\n return None\n \n return min(valid_suggestions, key=lambda x: x[1])[0]\n\n def correct_query(self, query, option='general'):\n \"\"\"Коррекция запроса\"\"\"\n if not query or not isinstance(query, str):\n return query\n \n words = query.split()\n if not words:\n return query\n \n corrected_words = []\n for word in words:\n if word.isdigit():\n corrected_words.append(word)\n continue\n\n corrected_word = self._get_correction(word, option)\n corrected_words.append(corrected_word if corrected_word is not None else word)\n \n return ' '.join(corrected_words)","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:31.212118Z","iopub.execute_input":"2025-10-25T02:42:31.212563Z","iopub.status.idle":"2025-10-25T02:42:31.237171Z","shell.execute_reply.started":"2025-10-25T02:42:31.212540Z","shell.execute_reply":"2025-10-25T02:42:31.236376Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### QueryProcessor\n**Назначение:** Обработка поисковых запросов\n\n**Методы:**\n- `_find_brand` - извлечение бренда из запросов\n- `preprocess_queries` - предобработка запроса нормализацией, спеллчекерами (брендов, моделей, общим) и поиск бренда\n\n**Принципы работы:**\n- Предобработка запроса тремя спелл-чекерами (брендов, моделей, общим) с получением трех новых переменных\n- Поиск бренда:\n - Выбор точного совпадения\n - При его отсутствии - по частичному совпадению слов, выбирая бренд с наибольшей длиной самого длинного слова и максимальным процентом совпадения \n","metadata":{}},{"cell_type":"code","source":"class QueryProcessor:\n \"\"\"Обработка поисковых запросов\"\"\"\n \n def __init__(self, preprocessor, spell_checker):\n self.preprocessor = preprocessor\n self.spell_checker = spell_checker\n self.sorted_brands = None\n \n def _find_brand(self, query_text, min_match_ratio):\n \"\"\"Поиск бренда в запросе\"\"\"\n for entity in self.sorted_brands:\n if f\" {entity} \" in f\" {query_text} \":\n return entity\n \n best_match = None\n best_score = 0\n longest_word_length = 0\n\n query_words = query_text.split()\n for entity in self.sorted_brands:\n entity_words = entity.split()\n \n if len(entity_words) > 1:\n matched_words = sum(1 for ew in entity_words if ew in query_words)\n \n if matched_words > 0:\n match_ratio = matched_words / len(entity_words)\n current_longest = max(len(word) for word in entity_words)\n \n if (match_ratio >= min_match_ratio and \n (current_longest > longest_word_length or \n (current_longest == longest_word_length and match_ratio > best_score))):\n \n best_score = match_ratio\n longest_word_length = current_longest\n best_match = entity\n \n return best_match\n \n def preprocess_queries(self, df, brands):\n \"\"\"Предобработка поисковых запросов\"\"\"\n self.sorted_brands = sorted(brands, key=len, reverse=True)\n \n unique_queries = df['query'].dropna().unique()\n print(f\"Найдено {len(unique_queries):,} уникальных запросов из {len(df):,} всего\")\n\n query_mapping = {}\n \n for query in unique_queries:\n if not query or pd.isna(query):\n continue\n \n normalized = self.preprocessor.normalize_text(query)\n corrected_brand = self.spell_checker.correct_query(normalized, 'brand')\n query_mapping[query] = {\n 'brand': self._find_brand(corrected_brand, 0.5),\n 'model': self.spell_checker.correct_query(normalized, 'model'), \n 'general': self.spell_checker.correct_query(normalized)\n }\n \n df['query_model'] = df['query'].map(lambda x: query_mapping.get(x, {}).get('model', ''))\n df['query_general'] = df['query'].map(lambda x: query_mapping.get(x, {}).get('general', ''))\n df['found_brand'] = df['query'].map(lambda x: query_mapping.get(x, {}).get('brand', ''))\n \n print(\"Предобработка завершена!\")\n return df\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.238443Z","iopub.execute_input":"2025-10-25T02:42:31.238761Z","iopub.status.idle":"2025-10-25T02:42:31.267500Z","shell.execute_reply.started":"2025-10-25T02:42:31.238738Z","shell.execute_reply":"2025-10-25T02:42:31.266511Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### BM25Processor\n**Назначение:** Расчет текстовой релевантности по алгоритму BM25\n\n**Методы:**\n- `fit()` - обучение на тренировочных данных\n- `transform()` - расчет для новых данных\n- `fit_transform()` - обучение на тренировочных данных и расчет на всех новых из списка\n\n**Принципы работы:**\n- Расчет релевантности по столбцу, предобработанному спелл-чекером моделей, и моделью\n- Пакетная обработка для больших объемов","metadata":{}},{"cell_type":"code","source":"class BM25Processor:\n \"\"\"BM25 для поиска релевантности\"\"\"\n \n def __init__(self):\n self.k1 = config.BM25_CONFIG['k1']\n self.b = config.BM25_CONFIG['b']\n self.vocab_size = config.BM25_CONFIG['vocab_size']\n self.batch_size = config.BM25_CONFIG['batch_size']\n self.query_col = config.BM25_CONFIG['query_column']\n self.doc_col = config.BM25_CONFIG['document_column']\n \n self.vectorizer = None\n self.idf_diag = None\n self.avg_doc_length = None\n\n \n def fit(self, train_df):\n print(\"Обучение BM25...\")\n \n all_train_texts = pd.concat([train_df[self.query_col], train_df[self.doc_col]]).astype(str)\n \n self.vectorizer = CountVectorizer(\n max_features=self.vocab_size,\n analyzer='word',\n token_pattern=r'(?u)\\b\\w+\\b',\n lowercase=False\n )\n self.vectorizer.fit(all_train_texts)\n print(f\"Размер словаря: {len(self.vectorizer.vocabulary_):,}\")\n \n n_docs_train = len(train_df)\n df_counts = np.zeros(len(self.vectorizer.vocabulary_))\n doc_lengths_list = []\n\n for i in range(0, n_docs_train, self.batch_size):\n batch_docs = train_df[self.doc_col].iloc[i:i+self.batch_size].astype(str)\n batch_vectors = self.vectorizer.transform(batch_docs)\n df_counts += np.array((batch_vectors > 0).sum(axis=0)).flatten()\n doc_lengths_list.extend(np.array(batch_vectors.sum(axis=1)).flatten())\n\n idf = np.log((n_docs_train - df_counts + 0.5) / (df_counts + 0.5))\n self.avg_doc_length = np.mean(doc_lengths_list)\n self.idf_diag = sp.diags(idf)\n \n print(f\"Средняя длина документа: {self.avg_doc_length:.2f}\")\n\n \n def transform(self, df):\n unique_pairs = df[[self.query_col, self.doc_col]].drop_duplicates()\n total_pairs = len(unique_pairs)\n print(f\"Уникальных пар: {total_pairs:,}\")\n \n all_scores = []\n \n for i in range(0, total_pairs, self.batch_size):\n batch_pairs = unique_pairs.iloc[i:i+self.batch_size]\n\n query_vectors = self.vectorizer.transform(batch_pairs[self.query_col].astype(str))\n doc_vectors = self.vectorizer.transform(batch_pairs[self.doc_col].astype(str))\n\n doc_vectors_bm25 = doc_vectors.astype(np.float64)\n doc_lengths = np.array(doc_vectors.sum(axis=1)).flatten()\n tf = doc_vectors_bm25.data\n\n doc_indices = np.repeat(np.arange(len(batch_pairs)), \n np.diff(doc_vectors_bm25.indptr))\n doc_lengths_repeated = doc_lengths[doc_indices]\n\n bm25_scores = (tf * (self.k1 + 1)) / (tf + self.k1 * (1 - self.b + self.b * doc_lengths_repeated / self.avg_doc_length))\n doc_vectors_bm25.data = bm25_scores\n\n doc_vectors_bm25 = doc_vectors_bm25 * self.idf_diag\n scores = np.array(query_vectors.multiply(doc_vectors_bm25).sum(axis=1)).flatten()\n all_scores.extend(scores)\n \n if i % (self.batch_size) == 0 and i > 0:\n print(f\"Обработано {min(i + self.batch_size, total_pairs):,}/{total_pairs:,} пар\")\n \n unique_pairs = unique_pairs.copy()\n unique_pairs['bm25_query_model_score'] = all_scores\n\n result_df = df.merge(unique_pairs, on=[self.query_col, self.doc_col], how='left')\n return result_df\n\n \n def fit_transform(self, train_df, dfs):\n self.fit(train_df)\n train_df = self.transform(train_df)\n\n for i, df in enumerate(dfs):\n dfs[i] = self.transform(df)\n\n print(\"BM25 обработка завершена\")\n return train_df, dfs","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:31.268401Z","iopub.execute_input":"2025-10-25T02:42:31.268742Z","iopub.status.idle":"2025-10-25T02:42:31.291322Z","shell.execute_reply.started":"2025-10-25T02:42:31.268707Z","shell.execute_reply":"2025-10-25T02:42:31.290094Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### CosineSimilarityCalculator\n**Назначение:** Расчет семантической схожести через эмбеддинги\n\n**Модель:** `sentence-transformers/paraphrase-multilingual-mpnet-base-v2`\n\n**Методы:**\n- `_prepare_texts` - подготовка текстов для обработки\n- `_compute_embeddings` - вычисление эмбеддингов\n- `calculate` - расчет косинусной близости\n\n**Принципы работы:**\n- Сравнение столбцов:\n - запрос после обработки общим спелл-чекером\n - объединенные в одну строку тип, категории и страна товара\n- Расчет эмбеддингов только для уникальных текстов\n- Нормализация эмбеддингов\n- Пакетная обработка с прогресс-баром","metadata":{}},{"cell_type":"code","source":"class CosineSimilarityCalculator:\n \"\"\"Расчет косинусной схожести\"\"\"\n \n def __init__(self):\n self.model = SentenceTransformer(config.EMBEDDING_CONFIG['model_name'], config.EMBEDDING_CONFIG['device'])\n self.chunk_size = config.EMBEDDING_CONFIG['chunk_size']\n self.embedding_cache = {}\n self.normalize_embeddings = config.EMBEDDING_CONFIG['normalize_embeddings']\n self.convert_to_tensor = config.EMBEDDING_CONFIG['convert_to_tensor']\n self.show_progress_bar = config.EMBEDDING_CONFIG['show_progress_bar']\n self.text_configs = config.EMBEDDING_CONFIG['text_configs']\n\n \n def _prepare_texts(self, df):\n print(\"Формируем набор текстов для обработки...\")\n \n text_configs = self.text_configs\n texts = {}\n \n for config in text_configs:\n if config['source'] == 'single_column':\n texts[config['name']] = df[config['column']].fillna('').astype(str)\n elif config['source'] == 'combined_columns':\n combined = ''\n for col in config['columns']:\n combined += df[col].fillna('').astype(str) + ' '\n texts[config['name']] = combined.str.strip()\n \n texts1 = texts['query_text']\n texts2 = texts['category_text']\n \n all_unique_texts = set(texts1) | set(texts2)\n print(f\"Всего уникальных текстов для обработки: {len(all_unique_texts):,}\")\n \n return list(all_unique_texts), texts1, texts2\n\n \n def _compute_embeddings(self, texts):\n print(\"Вычисляем эмбеддинги...\")\n \n embeddings = self.model.encode(\n texts, \n convert_to_tensor= self.convert_to_tensor, \n normalize_embeddings=self.normalize_embeddings, \n show_progress_bar=self.show_progress_bar\n )\n \n self.embedding_cache = dict(zip(texts, embeddings))\n print(\"Эмбеддинги успешно вычислены\")\n\n \n def calculate(self, df):\n \"\"\"Расчет косинусной близости\"\"\"\n all_texts, texts1, texts2 = self._prepare_texts(df)\n self._compute_embeddings(all_texts)\n \n cosine_similarities = []\n \n for i in range(0, len(df), self.chunk_size):\n end_idx = min(i + self.chunk_size, len(df))\n chunk_texts1 = texts1.iloc[i:end_idx]\n chunk_texts2 = texts2.iloc[i:end_idx]\n\n emb1 = np.array([self.embedding_cache[text] for text in chunk_texts1])\n emb2 = np.array([self.embedding_cache[text] for text in chunk_texts2])\n\n chunk_cosine = np.einsum('ij,ij->i', emb1, emb2)\n cosine_similarities.extend(chunk_cosine)\n\n if i % (self.chunk_size * 5) == 0 and i > 0:\n print(f\"Обработано {min(i + self.chunk_size, len(df)):,}/{len(df):,} записей\")\n \n del self.embedding_cache\n import gc\n gc.collect()\n \n return df.assign(cosine_similarity=cosine_similarities)","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.292212Z","iopub.execute_input":"2025-10-25T02:42:31.292448Z","iopub.status.idle":"2025-10-25T02:42:31.312045Z","shell.execute_reply.started":"2025-10-25T02:42:31.292431Z","shell.execute_reply":"2025-10-25T02:42:31.311192Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Генерация признаков","metadata":{}},{"cell_type":"markdown","source":"### FeatureEngine\n**Назначение:** Генерация всех признаков для модели\n\n**Компоненты:**","metadata":{}},{"cell_type":"markdown","source":"#### QueryFeatures\n- `query_length` - длина запроса в символах\n- `query_global_frequency_log` - логарифм частоты запроса\n\nНе участвуют в обучении модели (временные):\n- `query_complexity` - произведение word_count * avg_word_length\n- `query_word_count` - количество слов в запросе\n- `query_avg_word_length` - средняя длина слова\n- `query_global_frequency` - глобальная частота запроса\n\n#### SimilarityRankFeatures\n- `brand_match` - бинарный признак совпадения найденного бренда с брендом товара\n- `similarity_rank` - ранг схожести в группе search_number: сортировка по совпадению бренда, bm25 и cosine_similarity","metadata":{}},{"cell_type":"code","source":"#==== ТЕКСТОВЫЕ ФИЧИ ====\nclass QueryFeatures:\n def process(self, df):\n df['query_length'] = df['query'].str.len().fillna(0).astype(np.int16)\n df['query_word_count'] = df['query'].str.split().str.len().fillna(0).astype(np.int8)\n df['query_avg_word_length'] = (df['query_length'] / (df['query_word_count'] + 1)).astype(np.float32)\n df['query_complexity'] = (df['query_word_count'] * df['query_avg_word_length']).astype(np.float32)\n \n query_freq = df['query'].value_counts().reset_index()\n query_freq.columns = ['query', 'query_global_frequency']\n df = df.merge(query_freq, on='query', how='left')\n \n df['query_global_frequency_log'] = np.log1p(df['query_global_frequency'].fillna(1)).astype(np.float32)\n \n return df\n \nclass SimilarityRankFeatures:\n def process(self, df):\n \n df['brand'] = df['brand'].astype(str)\n df['found_brand'] = df['found_brand'].astype(str)\n df['brand_match'] = (df['found_brand'] == df['brand']).astype(int)\n\n df = df.sort_values([\n 'search_number', 'brand_match', 'bm25_query_model_score', 'cosine_similarity'\n ], ascending=[True, False, False, False])\n \n df['similarity_rank'] = df.groupby('search_number').cumcount() + 1\n \n return df","metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true},"execution":{"iopub.status.busy":"2025-10-25T02:42:31.313015Z","iopub.execute_input":"2025-10-25T02:42:31.313331Z","iopub.status.idle":"2025-10-25T02:42:31.339754Z","shell.execute_reply.started":"2025-10-25T02:42:31.313304Z","shell.execute_reply":"2025-10-25T02:42:31.338527Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### TemporalProductMetrics\n- `historical_CTR` - исторический Click-Through Rate (клики/просмотры)\n- `historical_CVR` - историческая конверсия в покупки (покупки/просмотры)\n- `historical_Cart_Rate` - историческая доля добавлений в корзину (корзины/просмотры)\n- `historical_Conversion_Rate` - конверсия из кликов в покупки (покупки/клики)\n- `historical_Purchase_to_Cart_Rate` - конверсия из корзин в покупки (покупки/корзины)\n- `historical_data_confidence` - уверенность в исторических данных на основе логарифма просмотров\n\nНе участвуют в обучении модели (временные):\n- `cumulative_views` - кумулятивное количество просмотров товара\n- `cumulative_clicks` - кумулятивное количество кликов по товару\n- `cumulative_carts` - кумулятивное количество добавлений в корзину\n- `cumulative_purchases` - кумулятивное количество покупок товара\n- `cumulative_relevance_sum` - сумма значений релевантности по товару","metadata":{}},{"cell_type":"code","source":"#=== ПРОДУКТОВЫЕ ФИЧИ ===\nclass TemporalProductMetrics: \n def process(self, df, is_train, train_metrics=None):\n def calculate_cumulative_metrics(df, initial_metrics=None):\n df = df.sort_values(['actual_product_id', 'date'])\n df['session_counter'] = df.groupby('actual_product_id').cumcount()\n \n if initial_metrics:\n df['initial_views'] = df['actual_product_id'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_views', 0)\n )\n df['initial_clicks'] = df['actual_product_id'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_clicks', 0)\n )\n df['initial_carts'] = df['actual_product_id'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_carts', 0)\n )\n df['initial_purchases'] = df['actual_product_id'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_purchases', 0)\n )\n df['initial_relevance'] = df['actual_product_id'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_relevance_sum', 0)\n )\n else:\n df['initial_views'] = 0\n df['initial_clicks'] = 0\n df['initial_carts'] = 0\n df['initial_purchases'] = 0\n df['initial_relevance'] = 0\n \n df['cumulative_views'] = df.groupby('actual_product_id')['session_counter'].cumsum() + 1 + df['initial_views']\n \n df['is_click'] = (df['relevance'] >= 1).astype(np.int8)\n df['is_cart'] = (df['relevance'] >= 2).astype(np.int8)\n df['is_purchase'] = (df['relevance'] == 3).astype(np.int8)\n \n df['cumulative_clicks'] = df.groupby('actual_product_id')['is_click'].cumsum() + df['initial_clicks']\n df['cumulative_carts'] = df.groupby('actual_product_id')['is_cart'].cumsum() + df['initial_carts']\n df['cumulative_purchases'] = df.groupby('actual_product_id')['is_purchase'].cumsum() + df['initial_purchases']\n df['cumulative_relevance_sum'] = df.groupby('actual_product_id')['relevance'].cumsum() + df['initial_relevance']\n \n df.drop(['session_counter', 'initial_views', 'initial_clicks', 'initial_carts', \n 'initial_purchases', 'initial_relevance', 'is_click', 'is_cart', 'is_purchase'], \n axis=1, inplace=True)\n \n return df\n\n def calculate_final_metrics(df):\n required_columns = ['cumulative_views', 'cumulative_clicks', 'cumulative_carts', \n 'cumulative_purchases', 'cumulative_relevance_sum']\n \n for col in required_columns:\n if col not in df.columns:\n df[col] = 0\n \n mask = df['cumulative_views'] > 0\n df.loc[mask, 'historical_CTR'] = df.loc[mask, 'cumulative_clicks'] / df.loc[mask, 'cumulative_views']\n df.loc[~mask, 'historical_CTR'] = 0.0\n \n df.loc[mask, 'historical_CVR'] = df.loc[mask, 'cumulative_purchases'] / df.loc[mask, 'cumulative_views']\n df.loc[~mask, 'historical_CVR'] = 0.0\n \n df.loc[mask, 'historical_Cart_Rate'] = df.loc[mask, 'cumulative_carts'] / df.loc[mask, 'cumulative_views']\n df.loc[~mask, 'historical_Cart_Rate'] = 0.0\n\n mask_clicks = df['cumulative_clicks'] > 0\n df.loc[mask_clicks, 'historical_Conversion_Rate'] = df.loc[mask_clicks, 'cumulative_purchases'] / df.loc[mask_clicks, 'cumulative_clicks']\n df.loc[~mask_clicks, 'historical_Conversion_Rate'] = 0.0\n \n mask_carts = df['cumulative_carts'] > 0\n df.loc[mask_carts, 'historical_Purchase_to_Cart_Rate'] = df.loc[mask_carts, 'cumulative_purchases'] / df.loc[mask_carts, 'cumulative_carts']\n df.loc[~mask_carts, 'historical_Purchase_to_Cart_Rate'] = 0.0\n\n df['historical_data_confidence'] = (np.log1p(df['cumulative_views']) / 10.0).astype(np.float32)\n \n return df\n\n if is_train:\n df = calculate_cumulative_metrics(df)\n df = calculate_final_metrics(df)\n\n self.final_product_metrics = df[['actual_product_id', 'cumulative_views', 'cumulative_clicks', \n 'cumulative_carts', 'cumulative_purchases', 'cumulative_relevance_sum']].copy()\n else:\n latest_product_metrics = train_metrics.groupby('actual_product_id').agg({\n 'cumulative_views': 'last',\n 'cumulative_clicks': 'last', \n 'cumulative_carts': 'last',\n 'cumulative_purchases': 'last',\n 'cumulative_relevance_sum': 'last'\n }).fillna(0).reset_index()\n \n df = df.merge(\n latest_product_metrics, \n on='actual_product_id', \n how='left'\n )\n\n numeric_cols_to_fill = ['cumulative_views', 'cumulative_clicks', 'cumulative_carts', \n 'cumulative_purchases', 'cumulative_relevance_sum']\n for col in numeric_cols_to_fill:\n if col in df.columns:\n df[col] = df[col].fillna(0)\n \n if 'cumulative_views_x' in df.columns and 'cumulative_views_y' in df.columns:\n df['cumulative_views'] = df['cumulative_views_y']\n df = df.drop(['cumulative_views_x', 'cumulative_views_y'], axis=1)\n elif 'cumulative_views_x' in df.columns:\n df['cumulative_views'] = df['cumulative_views_x']\n df = df.drop('cumulative_views_x', axis=1)\n elif 'cumulative_views_y' in df.columns:\n df['cumulative_views'] = df['cumulative_views_y']\n df = df.drop('cumulative_views_y', axis=1)\n \n df = calculate_final_metrics(df)\n\n return df","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.341041Z","iopub.execute_input":"2025-10-25T02:42:31.341487Z","iopub.status.idle":"2025-10-25T02:42:31.371524Z","shell.execute_reply.started":"2025-10-25T02:42:31.341414Z","shell.execute_reply":"2025-10-25T02:42:31.370520Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### SessionContextFeatures\n- `product_age_days` - возраст товара в днях от даты создания до даты сессии\n- `day_of_week_sin/cos` - циклические признаки дня недели сессии\n- `month_sin/cos` - циклические признаки месяца сессии\n- `day_of_month_sin/cos` - циклические признаки дня месяца сессии\n- `product_create_month_sin/cos` - циклические признаки месяца создания товара\n- `position_normalized` - нормализованная позиция (позиция / общее кол-во товаров найдено)\n- `position_in_group` - порядковый номер позиции внутри поисковой выдачи\n- `group_size` - количество товаров в поисковой выдаче (search_number)\n- `browse_time_log` - логарифмированное время просмотра товара\n- `response_time_ms_log` - логарифмированное время ответа системы в миллисекундах\n- `selected_category_present` - флаг наличия выбранной пользователем категории","metadata":{}},{"cell_type":"code","source":"#=== СЕССИОННЫЕ / ВРЕМЕННЫЕ ФИЧИ ===\nclass SessionContextFeatures:\n def process(self, df):\n df['product_age_days'] = (df['date'] - df['date_of_create']).dt.days\n \n df['day_of_week_sin'] = np.sin(2 * np.pi * df['date'].dt.dayofweek / 7)\n df['day_of_week_cos'] = np.cos(2 * np.pi * df['date'].dt.dayofweek / 7)\n\n df['month_sin'] = np.sin(2 * np.pi * df['date'].dt.month / 12)\n df['month_cos'] = np.cos(2 * np.pi * df['date'].dt.month / 12)\n\n df['day_of_month_sin'] = np.sin(2 * np.pi * df['date'].dt.day / 31)\n df['day_of_month_cos'] = np.cos(2 * np.pi * df['date'].dt.day / 31)\n\n df['product_create_month_sin'] = np.sin(2 * np.pi * df['date_of_create'].dt.month / 12)\n df['product_create_month_cos'] = np.cos(2 * np.pi * df['date_of_create'].dt.month / 12)\n\n df['position_normalized'] = (df['product_position'] / df['response_products']).astype(np.float32) \n\n df['position_in_group'] = df.groupby('search_number')['product_position'].rank()\n df['group_size'] = df.groupby('search_number').transform('size')\n\n df['browse_time_log'] = np.log1p(df['browse_time']).astype(np.float32)\n df['response_time_ms_log'] = np.log1p(df['response_time'] / 1000).astype(np.float32)\n \n df['selected_category_present'] = (df['selected_category_id'] > 0).astype(np.int8)\n \n return df","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.372485Z","iopub.execute_input":"2025-10-25T02:42:31.372816Z","iopub.status.idle":"2025-10-25T02:42:31.391387Z","shell.execute_reply.started":"2025-10-25T02:42:31.372793Z","shell.execute_reply":"2025-10-25T02:42:31.390518Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### PriceFeatures\n- `price_byn` - цена товара в белорусских рублях\n- `discount_ratio` - коэффициент скидки (цена со скидкой / исходная цена)\n- `price_position_category1` - относительная цена товара в категории 1 уровня\n- `price_position_category4` - относительная цена товара в категории 4 уровня\n- `price_position_type` - относительная цена товара в рамках типа товара\n- `log_price` - логарифмированная цена товара\n- `rank_in_group_by_price` - нормализованный ранг товара по цене внутри поисковой выдачи\n- `rank_in_group_by_discount` - нормализованный ранг товара по размеру скидки внутри поисковой выдачи\n- `category_mean` - скользящее среднее значение цены по категории","metadata":{}},{"cell_type":"code","source":"#=== ЦЕНОВЫЕ ФИЧИ ===\nclass PriceFeatures:\n def process(self, df):\n df['price_byn'] = (df['price'] / 100).astype(np.float32)\n df['discount_ratio'] = np.where(\n df['price'] > 0,\n df['price_discount'] / df['price'],\n 0.0\n )\n\n category1_avg_price = df.groupby('category_name_1')['price_byn'].transform('mean')\n df['price_position_category1'] = (df['price_byn'] / category1_avg_price).astype(np.float32)\n \n category4_avg_price = df.groupby('category_name_4')['price_byn'].transform('mean')\n df['price_position_category4'] = (df['price_byn'] / category4_avg_price).astype(np.float32)\n \n type_avg_price = df.groupby('type')['price_byn'].transform('mean')\n df['price_position_type'] = (df['price_byn'] / type_avg_price).astype(np.float32)\n \n df['log_price'] = np.log1p(df['price_byn']).astype(np.float32)\n \n df['rank_in_group_by_price'] = df.groupby('search_number')['price_byn'].rank(\n method='dense', ascending=True\n ).astype(np.int16)\n\n df['rank_in_group_by_discount'] = df.groupby('search_number')['discount_ratio'].rank(\n method='dense', ascending=False\n ).fillna(0).astype(np.int16)\n\n return df\n\n\nclass CompetitivePriceFeatures:\n def process(self, df, is_train, train_category_stats=None):\n df = df.sort_values(['category_name_1', 'date']).reset_index(drop=True)\n \n def calculate_category_metrics(df, initial_stats=None):\n if initial_stats is not None:\n df = df.merge(initial_stats, on='category_name_1', how='left', suffixes=('', '_train'))\n df['initial_mean'] = df.get('mean', 0)\n df['initial_std'] = df.get('std', 0)\n df['initial_count'] = df.get('count', 0)\n else:\n df['initial_mean'] = 0\n df['initial_std'] = 0\n df['initial_count'] = 0\n \n df['cumulative_price_sum'] = df.groupby('category_name_1')['price_byn'].cumsum() - df['price_byn'] + df['initial_mean'] * df['initial_count']\n df['cumulative_price_count'] = df.groupby('category_name_1').cumcount() + df['initial_count']\n \n df['category_mean'] = np.where(\n df['cumulative_price_count'] > 0,\n df['cumulative_price_sum'] / df['cumulative_price_count'],\n df['initial_mean']\n ).astype(np.float32)\n \n df['price_diff_sq'] = (df['price_byn'] - df['category_mean'])**2\n df['cumulative_price_diff_sq'] = df.groupby('category_name_1')['price_diff_sq'].cumsum() - df['price_diff_sq'] + df['initial_std']**2 * df['initial_count']\n\n drop_cols = ['cumulative_price_sum', 'cumulative_price_count', 'price_diff_sq', \n 'cumulative_price_diff_sq', 'initial_mean', 'initial_std', 'initial_count']\n if 'mean' in df.columns:\n drop_cols.extend(['mean', 'std', 'count'])\n \n df.drop(drop_cols, axis=1, inplace=True, errors='ignore')\n \n return df\n\n if is_train:\n df = calculate_category_metrics(df)\n else:\n if train_category_stats is not None:\n latest_category_stats = train_category_stats.groupby('category_name_1').agg({\n 'price_byn': ['mean', 'std', 'count']\n }).fillna(0)\n latest_category_stats.columns = ['mean', 'std', 'count']\n latest_category_stats = latest_category_stats.reset_index()\n \n df = calculate_category_metrics(df, latest_category_stats)\n else:\n df = calculate_category_metrics(df)\n \n return df\n","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.392473Z","iopub.execute_input":"2025-10-25T02:42:31.392712Z","iopub.status.idle":"2025-10-25T02:42:31.421874Z","shell.execute_reply.started":"2025-10-25T02:42:31.392694Z","shell.execute_reply":"2025-10-25T02:42:31.420959Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### UserPreferences\n\n- `user_activity` - общая активность пользователя (количество сессий)\n- `user_activity_log` - логарифмированная активность пользователя\n- `avg_query_complexity` - средняя сложность поисковых запросов пользователя\n- `filters_usage_rate` - частота использования фильтров пользователем\n\nНе участвуют в обучении модели (временные):\n- `cumulative_complexity_sum` - накопленная сумма сложности запросов пользователя\n- `cumulative_sessions_count` - накопленное количество сессий пользователя\n- `cumulative_filters_sum` - накопленная сумма примененных фильтров","metadata":{}},{"cell_type":"code","source":"#=== ПОЛЬЗОВАТЕЛЬСКИЕ ФИЧИ ===\nclass UserBehaviorFeatures: \n def process(self, df, is_train, train_user_metrics=None):\n df = df.sort_values(['user_number', 'date']).reset_index(drop=True)\n \n def calculate_user_metrics(df, initial_metrics=None):\n df['session_counter'] = df.groupby('user_number').cumcount()\n df['user_activity'] = (df['session_counter'] - 1).clip(lower=0)\n df['user_activity_log'] = np.log1p(df['user_activity']).astype(np.float32)\n \n if initial_metrics:\n df['initial_complexity_sum'] = df['user_number'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_complexity_sum', 0)\n )\n df['initial_sessions_count'] = df['user_number'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_sessions_count', 0)\n )\n df['initial_filters_sum'] = df['user_number'].map(\n lambda x: initial_metrics.get(x, {}).get('cumulative_filters_sum', 0)\n )\n else:\n df['initial_complexity_sum'] = 0\n df['initial_sessions_count'] = 0\n df['initial_filters_sum'] = 0\n df['initial_device'] = 'new_user'\n \n df['cumulative_complexity_sum'] = df.groupby('user_number')['query_complexity'].cumsum() - df['query_complexity'] + df['initial_complexity_sum']\n df['cumulative_sessions_count'] = df.groupby('user_number').cumcount() + df['initial_sessions_count']\n df['cumulative_filters_sum'] = df.groupby('user_number')['filters_applied'].cumsum() - df['filters_applied'] + df['initial_filters_sum']\n \n df['avg_query_complexity'] = np.where(\n df['cumulative_sessions_count'] > 0,\n df['cumulative_complexity_sum'] / df['cumulative_sessions_count'],\n 0.0\n ).astype(np.float32)\n \n df['filters_usage_rate'] = np.where(\n df['cumulative_sessions_count'] > 0,\n df['cumulative_filters_sum'] / df['cumulative_sessions_count'],\n 0.0\n ).astype(np.float32)\n \n \n df.drop([\n 'session_counter', 'initial_complexity_sum', 'initial_sessions_count', \n 'initial_filters_sum', 'initial_device', 'query_complexity'\n ], axis=1, inplace=True, errors='ignore')\n \n return df\n\n if is_train:\n df = calculate_user_metrics(df)\n else:\n latest_user_metrics = train_user_metrics.groupby('user_number').agg({\n 'cumulative_complexity_sum': 'last',\n 'cumulative_sessions_count': 'last',\n 'cumulative_filters_sum': 'last',\n }).reset_index()\n \n numeric_cols = ['cumulative_complexity_sum', 'cumulative_sessions_count', 'cumulative_filters_sum']\n latest_user_metrics[numeric_cols] = latest_user_metrics[numeric_cols].fillna(0)\n \n initial_metrics_dict = latest_user_metrics.set_index('user_number').to_dict('index')\n \n df = calculate_user_metrics(df, initial_metrics_dict)\n \n return df","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.423370Z","iopub.execute_input":"2025-10-25T02:42:31.423683Z","iopub.status.idle":"2025-10-25T02:42:31.446093Z","shell.execute_reply.started":"2025-10-25T02:42:31.423653Z","shell.execute_reply":"2025-10-25T02:42:31.445367Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### FeatureInteractions\n- `product_age_days_x_historical_data_confidence` - взаимодействие возраста товара и уверенности в данных\n- `position_x_historical_ctr` - взаимодействие позиции и исторического CTR\n- `position_x_historical_cvr` - взаимодействие позиции и исторической конверсии\n- `position_x_historical_Cart_Rate` - взаимодействие позиции и доли добавления в корзину\n- `bm25_x_position` - взаимодействие BM25 релевантности и позиции\n- `bm25_x_historical_cvr` - взаимодействие BM25 релевантности и исторической конверсии","metadata":{}},{"cell_type":"code","source":"#=== ВЗАИМОДЕЙСТВИЯ ФИЧЕЙ ===\nclass FeatureInteractions:\n def process(self, df):\n\n df['product_age_days_x_historical_data_confidence'] = df['product_age_days'] * df['historical_data_confidence']\n df['position_x_historical_ctr'] = df['position_normalized'] * df['historical_CTR']\n df['position_x_historical_cvr'] = df['position_normalized'] * df['historical_Conversion_Rate']\n df['position_x_historical_Cart_Rate'] = df['position_normalized'] * df['historical_Cart_Rate']\n \n df['bm25_x_position'] = df['bm25_query_model_score'] * df['position_normalized']\n df['bm25_x_historical_cvr'] = df['bm25_query_model_score'] * df['historical_CVR']\n\n return df","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.447355Z","iopub.execute_input":"2025-10-25T02:42:31.447740Z","iopub.status.idle":"2025-10-25T02:42:31.471187Z","shell.execute_reply.started":"2025-10-25T02:42:31.447719Z","shell.execute_reply":"2025-10-25T02:42:31.470245Z"},"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### MemoryOptimizer\n- Оптимизация типов данных для экономии памяти\n- Категоризация строковых признаков","metadata":{}},{"cell_type":"code","source":"#=== ОПТИМИЗАЦИЯ ПАМЯТИ ===\nclass MemoryOptimizer:\n def process(self, df): \n for col in df.select_dtypes(include=['int64']).columns:\n df[col] = pd.to_numeric(df[col], downcast='integer')\n \n for col in df.select_dtypes(include=['float64']).columns:\n df[col] = pd.to_numeric(df[col], downcast='float')\n \n for col in df.select_dtypes(include=['object']).columns:\n if df[col].nunique() / len(df) < 0.5:\n df[col] = df[col].astype('category')\n \n return df","metadata":{"trusted":true,"jupyter":{"source_hidden":true},"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.472113Z","iopub.execute_input":"2025-10-25T02:42:31.472397Z","iopub.status.idle":"2025-10-25T02:42:31.490674Z","shell.execute_reply.started":"2025-10-25T02:42:31.472369Z","shell.execute_reply":"2025-10-25T02:42:31.489694Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FeatureEngine:\n def __init__(self):\n self.similarity_rank = SimilarityRankFeatures()\n self.temporal_metrics = TemporalProductMetrics()\n self.session_context = SessionContextFeatures()\n self.price = PriceFeatures()\n self.query_features = QueryFeatures()\n self.feature_interactions = FeatureInteractions()\n self.competitive_price = CompetitivePriceFeatures()\n self.user_behavior = UserBehaviorFeatures()\n self.memory_optimizer = MemoryOptimizer()\n\n def add_all_features(self, sessions, is_train, train_data=None):\n df = sessions.copy()\n\n df = self.temporal_metrics.process(df, is_train, train_data)\n df = self.query_features.process(df) \n df = self.user_behavior.process(df, is_train, train_data)\n df = self.price.process(df)\n df = self.competitive_price.process(df, is_train, train_data)\n df = self.session_context.process(df)\n df = self.similarity_rank.process(df)\n df = self.memory_optimizer.process(df)\n df = self.feature_interactions.process(df)\n \n return df","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:25.221581Z","iopub.execute_input":"2025-10-25T02:44:25.221907Z","iopub.status.idle":"2025-10-25T02:44:25.229959Z","shell.execute_reply.started":"2025-10-25T02:44:25.221885Z","shell.execute_reply":"2025-10-25T02:44:25.228759Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"#### FeatureNormalizer\n- Нормализация числовых признаков с помощью квантильного преобразования и RobustScaler\n- Преобразование категориальных переменных в коды","metadata":{}},{"cell_type":"code","source":"class FeatureNormalizer:\n def __init__(self):\n self.scalers = {}\n self.encoder = None\n \n def fit(self, df):\n self.scalers['historical'] = QuantileTransformer(output_distribution='normal', random_state=config.SEED)\n self.scalers['historical'].fit(df[['historical_CTR', 'historical_CVR', 'historical_Cart_Rate']])\n\n self.scalers['polynomial'] = QuantileTransformer(output_distribution='normal', random_state=config.SEED)\n self.scalers['polynomial'].fit(df[['bm25_x_historical_cvr']])\n\n self.scalers['ratio'] = QuantileTransformer(output_distribution='normal', random_state=config.SEED)\n self.scalers['ratio'].fit(df[['discount_ratio']])\n\n self.scalers['filters'] = RobustScaler()\n self.scalers['filters'].fit(df[['filters_applied']])\n \n def transform(self, df):\n df[['historical_CTR', 'historical_CVR', 'historical_Cart_Rate']] = \\\n self.scalers['historical'].transform(df[['historical_CTR', 'historical_CVR', 'historical_Cart_Rate']])\n \n df[['bm25_x_historical_cvr']] = self.scalers['polynomial'].transform(df[['bm25_x_historical_cvr']])\n \n df[['discount_ratio']] = self.scalers['ratio'].transform(df[['discount_ratio']])\n\n df[['filters_applied']] = self.scalers['filters'].transform(df[['filters_applied']])\n \n df['device_encoded'] = df['device'].astype('category').cat.codes\n\n df['type_encoded'] = df['type'].astype('category').cat.codes\n df['category_encoded'] = df['category_name_4'].astype('category').cat.codes\n \n return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.518295Z","iopub.execute_input":"2025-10-25T02:42:31.518645Z","iopub.status.idle":"2025-10-25T02:42:31.537843Z","shell.execute_reply.started":"2025-10-25T02:42:31.518622Z","shell.execute_reply":"2025-10-25T02:42:31.536830Z"},"_kg_hide-input":true,"_kg_hide-output":true,"jupyter":{"source_hidden":true}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Основной пайплайн","metadata":{}},{"cell_type":"code","source":"print_complete_config_summary(config)","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.538986Z","iopub.execute_input":"2025-10-25T02:42:31.539341Z","iopub.status.idle":"2025-10-25T02:42:31.561613Z","shell.execute_reply.started":"2025-10-25T02:42:31.539308Z","shell.execute_reply":"2025-10-25T02:42:31.560489Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train = train_df.copy()\ntest = test_df.copy()\nproducts = products_df.copy()\n\ndata_sampler = DataSampler()\n\ntrain = data_sampler.group_sampling(train, config.SAMPLING_CONFIG['max_group_size'])\ntrain = data_sampler.filter_short_groups(train)\n\ntrain, test = data_sampler.date_to_datetime([train, test])\n\nif config.IS_VALIDATION:\n train, validation = data_sampler.create_validation_split(train, 3)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.562607Z","iopub.execute_input":"2025-10-25T02:42:31.562898Z","iopub.status.idle":"2025-10-25T02:42:31.597929Z","shell.execute_reply.started":"2025-10-25T02:42:31.562877Z","shell.execute_reply":"2025-10-25T02:42:31.596989Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"preprocessor = TextPreprocessor()\nspell_checker = SpellChecker(preprocessor)\nquery_processor = QueryProcessor(preprocessor, spell_checker)\n\nproducts, brands, models, categories, types, countries, brands_transl_dict, models_transl_dict = preprocessor.normalize_products(products)\nproducts = preprocessor.process_column(products, 'model')\n\nspell_checker.setup_spell_checkers(\n brands, models, categories, types, countries, \n brands_transl_dict, models_transl_dict\n)\n\nproduct_processor = ProductFeaturesProcessor()\nproducts_prepared = product_processor.prepare_products(products)\n\nif config.IS_VALIDATION:\n train, validation, test = product_processor.process([train, validation, test])\nelse:\n train, test = product_processor.process([train, test])\n\nprint(\"Предобработка пользовательских запросов train\")\ntrain = query_processor.preprocess_queries(train, brands)\ntrain = preprocessor.process_column(train, 'query_model')\n\nprint(\"Предобработка пользовательских запросов test\")\ntest = query_processor.preprocess_queries(test, brands)\ntest = preprocessor.process_column(test, 'query_model')\n\nif config.IS_VALIDATION:\n print(\"Предобработка пользовательских запросов validation 1\")\n validation = query_processor.preprocess_queries(validation, brands)\n validation = preprocessor.process_column(validation, 'query_model')\n \nprint(\"Предобработка завершена!\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:31.599112Z","iopub.execute_input":"2025-10-25T02:42:31.599633Z","iopub.status.idle":"2025-10-25T02:42:40.854489Z","shell.execute_reply.started":"2025-10-25T02:42:31.599606Z","shell.execute_reply":"2025-10-25T02:42:40.853465Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"bm25_processor = BM25Processor()\n\nif config.IS_VALIDATION:\n train, dfs = bm25_processor.fit_transform(train_df=train, dfs=[validation, test])\n validation, test = dfs\nelse:\n train, dfs = bm25_processor.fit_transform(train_df=train, dfs=[test])\n test = dfs[0]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:40.855362Z","iopub.execute_input":"2025-10-25T02:42:40.855631Z","iopub.status.idle":"2025-10-25T02:42:40.913405Z","shell.execute_reply.started":"2025-10-25T02:42:40.855566Z","shell.execute_reply":"2025-10-25T02:42:40.912256Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cosine_calculator = CosineSimilarityCalculator()\ntrain = cosine_calculator.calculate(train)\ntest = cosine_calculator.calculate(test)\n\nif config.IS_VALIDATION:\n validation = cosine_calculator.calculate(validation)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:42:40.914624Z","iopub.execute_input":"2025-10-25T02:42:40.914950Z","iopub.status.idle":"2025-10-25T02:42:53.046375Z","shell.execute_reply.started":"2025-10-25T02:42:40.914919Z","shell.execute_reply":"2025-10-25T02:42:53.045512Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_engine = FeatureEngine()\n\nprint(\"Создание признаков для train\")\ntrain_features = feature_engine.add_all_features(train, is_train=True)\n\nprint(\"Создание признаков для test\")\ntest_features = feature_engine.add_all_features(test, is_train=False, train_data=train_features)\n\nif config.IS_VALIDATION:\n print(\"Создание признаков для validation\")\n validation_features = feature_engine.add_all_features(validation, is_train=False, train_data=train_features)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:30.841279Z","iopub.execute_input":"2025-10-25T02:44:30.841676Z","iopub.status.idle":"2025-10-25T02:44:31.077437Z","shell.execute_reply.started":"2025-10-25T02:44:30.841647Z","shell.execute_reply":"2025-10-25T02:44:31.076472Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_features.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:31.078935Z","iopub.execute_input":"2025-10-25T02:44:31.079228Z","iopub.status.idle":"2025-10-25T02:44:31.106873Z","shell.execute_reply.started":"2025-10-25T02:44:31.079205Z","shell.execute_reply":"2025-10-25T02:44:31.105547Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"normalizer = FeatureNormalizer()\nnormalizer.fit(train_features)\n\ntrain_features = normalizer.transform(train_features)\ntest_features = normalizer.transform(test_features)\n\nif config.IS_VALIDATION:\n validation_features = normalizer.transform(validation_features)\nprint(\"Нормализация завершена\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:31.108321Z","iopub.execute_input":"2025-10-25T02:44:31.108673Z","iopub.status.idle":"2025-10-25T02:44:31.160976Z","shell.execute_reply.started":"2025-10-25T02:44:31.108638Z","shell.execute_reply":"2025-10-25T02:44:31.160146Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_sorted = train_features.sort_values(['search_number'])\ntest_sorted = test_features.sort_values(['search_number'])\n\nX_train = train_sorted\nX_test = test_sorted.copy()\ny_train = train_sorted['relevance']\n\ntrain_groups = train_sorted.groupby('search_number').size().tolist()\n\nif config.IS_VALIDATION:\n validation_sorted = validation_features.sort_values(['search_number'])\n X_validation = validation_sorted.copy()\n y_validation = validation_sorted['relevance']\n validation_groups = validation_sorted.groupby('search_number').size().tolist()\n\nexisting_columns = [col for col in COLUMNS_TO_DROP if col in X_train.columns]\nX_train.drop(existing_columns, axis=1, inplace=True)\n\nexisting_columns = [col for col in COLUMNS_TO_DROP if col in X_test.columns]\nX_test.drop(existing_columns, axis=1, inplace=True)\n\nif config.IS_VALIDATION:\n existing_columns = [col for col in COLUMNS_TO_DROP if col in X_validation.columns]\n X_validation.drop(existing_columns, axis=1, inplace=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:31.162813Z","iopub.execute_input":"2025-10-25T02:44:31.163142Z","iopub.status.idle":"2025-10-25T02:44:31.186912Z","shell.execute_reply.started":"2025-10-25T02:44:31.163118Z","shell.execute_reply":"2025-10-25T02:44:31.185909Z"},"_kg_hide-input":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:31.187951Z","iopub.execute_input":"2025-10-25T02:44:31.188358Z","iopub.status.idle":"2025-10-25T02:44:31.203453Z","shell.execute_reply.started":"2025-10-25T02:44:31.188333Z","shell.execute_reply":"2025-10-25T02:44:31.202517Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Оптимизация гиперпараметров...","metadata":{}},{"cell_type":"markdown","source":"... была. Честное слово. Но в итоге подобранные на глаз гиперпараметры дали лучше результаты при сабмите, чем подобранные на датасете под валидационную выборку с Optuna. \n\nВалидационная выборка представляла собой вырезанные из трейна данные за последние два месяца, где оставлены только группы с 20 товарами (для приближения данных к тесту), они предобрабатывались отдельно от трейна. \n\nПредположения, почему не сработало:\n- на тесте и так смещенные по времени данные относительно трейна. Если из трейна вырезать последний месяц на валидацию, то валидационная выборка маленькая и получается неустойчивый подбор. Если вырезать два последних месяца на валидацию, то трейну может не хватить данных обучиться + получается еще больше смещение относительно теста. \n\n- гиперпараметрами затачиваем на выборку \"в прошлом\" относительно теста, поэтому лучше без этого. \n\n- недостаточно данных на валидацию, сложность в приближении ее к \"тестовым\" условиям (не только по месяцу, но и по глубине поисковой выдачи, на которой оценивалось, и распределениям)\n","metadata":{}},{"cell_type":"markdown","source":"# Обучение модели","metadata":{}},{"cell_type":"code","source":"params = config.LGBM_PARAMS\n\nlgb_ranker = lgb.LGBMRanker( \n objective=params['objective'],\n metric=params['metric'],\n boosting_type=params['boosting_type'], \n lambdarank_norm=params['lambdarank_norm'],\n\n lambdarank_truncation_level=params['lambdarank_truncation_level'],\n ndcg_eval_at=params['ndcg_eval_at'],\n\n num_leaves=params['num_leaves'],\n min_data_in_leaf=params['min_data_in_leaf'],\n min_child_samples=params['min_child_samples'],\n reg_alpha=params['reg_alpha'],\n reg_lambda=params['reg_lambda'],\n subsample=params['subsample'],\n colsample_bytree=params['colsample_bytree'],\n subsample_freq=params['subsample_freq'],\n \n learning_rate=params['learning_rate'],\n n_estimators=params['n_estimators'],\n n_jobs=params['n_jobs'],\n\n max_depth=params['max_depth'],\n max_bin=params['max_bin'],\n verbosity=params['verbosity'],\n force_row_wise=params['force_row_wise'],\n\n random_state=config.SEED\n)\n\nlgb_ranker.fit(\n X_train, y_train,\n group=train_groups,\n)\n\ntest_predictions = lgb_ranker.predict(X_test)\nsubmission = pd.DataFrame({\n 'session_product_id': test_sorted['session_product_id'],\n 'relevance': test_predictions\n})\nsubmission = submission.drop_duplicates(subset=['session_product_id'], keep='first')\nsubmission.to_csv(config.OUTPUT_CONFIG['submission_file'], index=False)\nprint(\"Сабмит готов\")","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:31.204560Z","iopub.execute_input":"2025-10-25T02:44:31.205481Z","iopub.status.idle":"2025-10-25T02:44:32.456084Z","shell.execute_reply.started":"2025-10-25T02:44:31.205450Z","shell.execute_reply":"2025-10-25T02:44:32.455291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_importance = pd.DataFrame({\n 'feature': X_train.columns,\n 'importance': lgb_ranker.feature_importances_\n}).sort_values('importance', ascending=False)\n\nfeature_importance.to_csv(config.OUTPUT_CONFIG['feature_importance_file'], index=False)\nprint(\"Feature importance сохранен в lgb_feature_importance.csv\")","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:32.456870Z","iopub.execute_input":"2025-10-25T02:44:32.457092Z","iopub.status.idle":"2025-10-25T02:44:32.465851Z","shell.execute_reply.started":"2025-10-25T02:44:32.457075Z","shell.execute_reply":"2025-10-25T02:44:32.464963Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def plot_feature_importance(feature_importance, top_n=20, save_path=None):\n \"\"\"\n Визуализация важности признаков\n \"\"\"\n fig, axes = plt.subplots(2, figsize=(16, 12))\n\n top_features = feature_importance.head(top_n)\n\n axes[0].barh(range(len(top_features)), top_features['importance'])\n axes[0].set_yticks(range(len(top_features)))\n axes[0].set_yticklabels(top_features['feature'])\n axes[0].set_title(f'Топ {top_n} самых важных признаков')\n axes[0].set_xlabel('Важность')\n axes[0].set_ylabel('Признаки')\n\n feature_importance['cumulative'] = feature_importance['importance'].cumsum() / feature_importance['importance'].sum()\n axes[1].plot(range(1, len(feature_importance) + 1), feature_importance['cumulative'])\n axes[1].set_xlabel('Количество признаков')\n axes[1].set_ylabel('Кумулятивная важность')\n axes[1].set_title('Кумулятивная важность признаков')\n axes[1].grid(True, alpha=0.3)\n \n plt.tight_layout()\n \n if save_path:\n plt.savefig(save_path, dpi=300, bbox_inches='tight')\n \n plt.show()\n\nplot_feature_importance(feature_importance, top_n=20)","metadata":{"trusted":true,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2025-10-25T02:44:32.466768Z","iopub.execute_input":"2025-10-25T02:44:32.467093Z","iopub.status.idle":"2025-10-25T02:44:33.090906Z","shell.execute_reply.started":"2025-10-25T02:44:32.467059Z","shell.execute_reply":"2025-10-25T02:44:33.089808Z"}},"outputs":[],"execution_count":null}]}