{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport random\nimport pandas as pd\nimport numpy as np\nfrom pathlib import Path\nimport pyarrow.parquet as pq\nfrom sklearn.model_selection import train_test_split, GroupShuffleSplit","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:06:01.034786Z","iopub.execute_input":"2026-10-17T13:06:01.03527Z","iopub.status.idle":"2026-10-17T13:06:03.584872Z","shell.execute_reply.started":"2026-10-17T13:06:01.035238Z","shell.execute_reply":"2026-10-17T13:06:03.58425Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SEED = 42\nTRAIN_DATA_PATH = '/kaggle/input/competitions/aeroclub-recsys-2025/train.parquet'\nTEST_DATA_PATH = '/kaggle/input/competitions/aeroclub-recsys-2025/test.parquet'\nSAMPLE_SUBMISSION_PATH = '/kaggle/input/competitions/aeroclub-recsys-2025/sample_submission.parquet'\n\nOUTPUT_SUBMISSION_PATH = '/kaggle/working/submission.csv'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:06:03.586366Z","iopub.execute_input":"2026-10-17T13:06:03.586856Z","iopub.status.idle":"2026-10-17T13:06:03.591096Z","shell.execute_reply.started":"2026-10-17T13:06:03.586829Z","shell.execute_reply":"2026-10-17T13:06:03.590337Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pyarrow.parquet as pq\n\npf = pq.ParquetFile(TEST_DATA_PATH)\n\nprint(\"Строк:\", pf.metadata.num_rows)\nprint(\"Столбцов:\", pf.metadata.num_columns)\nprint(\"Групп строк:\", pf.num_row_groups)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:06:03.592365Z","iopub.execute_input":"2026-10-17T13:06:03.592696Z","iopub.status.idle":"2026-10-17T13:06:03.65663Z","shell.execute_reply.started":"2026-10-17T13:06:03.592664Z","shell.execute_reply":"2026-10-17T13:06:03.655677Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"seed = 42\nrandom.seed(seed)\nnp.random.seed(seed)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:06:03.658395Z","iopub.execute_input":"2026-10-17T13:06:03.658727Z","iopub.status.idle":"2026-10-17T13:06:03.664482Z","shell.execute_reply.started":"2026-10-17T13:06:03.658703Z","shell.execute_reply":"2026-10-17T13:06:03.663686Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pf = pq.ParquetFile(TRAIN_DATA_PATH)\nfrac = 33\n\nparts = []\nfor i in range(pf.num_row_groups):\n    df = pf.read_row_group(i).to_pandas()\n    keep = pd.util.hash_pandas_object(df[\"ranker_id\"], index=False) % 100 < frac\n    df = df[keep]\n    for c in df.select_dtypes(\"float64\").columns:\n        df[c] = df[c].astype(\"float32\")\n    parts.append(df)\n\nraw_data_train = pd.concat(parts, ignore_index=True)\nprint(len(raw_data_train), raw_data_train[\"ranker_id\"].nunique())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:06:03.665589Z","iopub.execute_input":"2026-10-17T13:06:03.666275Z","iopub.status.idle":"2026-10-17T13:07:03.381736Z","shell.execute_reply.started":"2026-10-17T13:06:03.666246Z","shell.execute_reply":"2026-10-17T13:07:03.380996Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"gss = GroupShuffleSplit(n_splits=1, test_size=0.2, random_state=42)\ntrain_idx, val_idx = next(gss.split(raw_data_train, groups=raw_data_train[\"ranker_id\"]))\n\nraw_data_train, raw_data_val = (\n    raw_data_train.iloc[train_idx].reset_index(drop=True),\n    raw_data_train.iloc[val_idx].reset_index(drop=True),\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:03.383016Z","iopub.execute_input":"2026-10-17T13:07:03.383392Z","iopub.status.idle":"2026-10-17T13:07:12.008628Z","shell.execute_reply.started":"2026-10-17T13:07:03.383367Z","shell.execute_reply":"2026-10-17T13:07:12.007965Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_data_train.info()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:12.009955Z","iopub.execute_input":"2026-10-17T13:07:12.010321Z","iopub.status.idle":"2026-10-17T13:07:12.027467Z","shell.execute_reply.started":"2026-10-17T13:07:12.010288Z","shell.execute_reply":"2026-10-17T13:07:12.02665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_data_val.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:12.028357Z","iopub.execute_input":"2026-10-17T13:07:12.028663Z","iopub.status.idle":"2026-10-17T13:07:12.06819Z","shell.execute_reply.started":"2026-10-17T13:07:12.028631Z","shell.execute_reply":"2026-10-17T13:07:12.067387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"date_str_cols = [c for c in raw_data_train.columns if c.endswith(\"At\")] \ndatetime_col = [col for col in raw_data_train.columns\n                if raw_data_train[col].dtype == \"datetime64[ns]\" or col in date_str_cols]\n\ncategorical_col = [col for col in raw_data_train.columns\n                   if raw_data_train[col].dtype == \"object\" and col not in datetime_col]\n\nnum_cols = [c for c in raw_data_train.select_dtypes(include=\"number\").columns if c != \"selected\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:12.069211Z","iopub.execute_input":"2026-10-17T13:07:12.06958Z","iopub.status.idle":"2026-10-17T13:07:12.139497Z","shell.execute_reply.started":"2026-10-17T13:07:12.069557Z","shell.execute_reply":"2026-10-17T13:07:12.138162Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Преобразование дат","metadata":{}},{"cell_type":"code","source":"def add_date_features(df, col):\n    d = pd.to_datetime(df[col], errors=\"coerce\")   # для datetime64 ничего не меняет, строки разбирает\n    df[f\"{col}_year\"] = d.dt.year\n    df[f\"{col}_month\"] = d.dt.month\n    df[f\"{col}_day\"] = d.dt.day\n    df[f\"{col}_dow\"] = d.dt.dayofweek\n    df[f\"{col}_is_weekend\"] = (d.dt.dayofweek >= 5).astype(\"float32\").where(d.notna())\n    df[f\"{col}_doy\"] = d.dt.dayofyear\n    if (d.dt.hour != 0).any():\n        df[f\"{col}_hour\"] = d.dt.hour\n    df[f\"{col}_ts\"] = (d - pd.Timestamp(\"1970-01-01\")) // pd.Timedelta(\"1s\")\n    return df.drop(columns=col)\n\ndef converting_date_columns(df, cols):\n    transformed_df = df.copy()\n    for col in cols:\n        transformed_df = add_date_features(transformed_df, col)\n    return transformed_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:12.142587Z","iopub.execute_input":"2026-10-17T13:07:12.143278Z","iopub.status.idle":"2026-10-17T13:07:12.150473Z","shell.execute_reply.started":"2026-10-17T13:07:12.14325Z","shell.execute_reply":"2026-10-17T13:07:12.149683Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train = converting_date_columns(raw_data_train, datetime_col)\ndata_val = converting_date_columns(raw_data_val, datetime_col)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:12.151756Z","iopub.execute_input":"2026-10-17T13:07:12.152243Z","iopub.status.idle":"2026-10-17T13:07:24.392061Z","shell.execute_reply.started":"2026-10-17T13:07:12.152192Z","shell.execute_reply":"2026-10-17T13:07:24.391292Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Заполнение категориальных пропусков","metadata":{}},{"cell_type":"code","source":"def fit_cat_missing(df, cat_cols, threshold=0.5):\n    \"\"\"Считаем по train: какие категориальные столбцы удалить.\"\"\"\n    na_share = df[cat_cols].isna().mean()\n    cols_to_drop = na_share[na_share > threshold].index.tolist()\n    cols_to_fill = [c for c in cat_cols if c not in cols_to_drop]\n    return cols_to_drop, cols_to_fill\n\n\ndef transform_cat_missing(df, cols_to_drop, cols_to_fill, fill_value=\"missing\"):\n    \"\"\"Применяем одинаково к train / val / test.\"\"\"\n    out = df.drop(columns=cols_to_drop).copy()\n    for c in cols_to_fill:\n        out[c] = out[c].astype(\"object\").where(out[c].notna(), fill_value).astype(str)\n    return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:24.393098Z","iopub.execute_input":"2026-10-17T13:07:24.393427Z","iopub.status.idle":"2026-10-17T13:07:24.399485Z","shell.execute_reply.started":"2026-10-17T13:07:24.393404Z","shell.execute_reply":"2026-10-17T13:07:24.398657Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_to_drop, cols_to_fill = fit_cat_missing(raw_data_train, categorical_col)\ndata_train = transform_cat_missing(data_train, cols_to_drop, cols_to_fill)\ndata_val = transform_cat_missing(data_val, cols_to_drop, cols_to_fill)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:24.40034Z","iopub.execute_input":"2026-10-17T13:07:24.40073Z","iopub.status.idle":"2026-10-17T13:07:32.69379Z","shell.execute_reply.started":"2026-10-17T13:07:24.400701Z","shell.execute_reply":"2026-10-17T13:07:32.692869Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Заполнение числовых пропусков медианой","metadata":{}},{"cell_type":"code","source":"def fit_num_missing(df, num_cols, threshold=0.5):\n    \"\"\"Считаем по train: какие числовые столбцы удалить и какими медианами заполнять.\"\"\"\n    na_share = df[num_cols].isna().mean()\n    cols_to_drop = na_share[na_share > threshold].index.tolist()\n    cols_to_fill = [c for c in num_cols if c not in cols_to_drop]\n    medians = df[cols_to_fill].median()\n    return cols_to_drop, cols_to_fill, medians\n\n\ndef transform_num_missing(df, cols_to_drop, cols_to_fill, medians, add_indicator=False):\n    \"\"\"Применяем одинаково к train / val / test.\"\"\"\n    out = df.drop(columns=cols_to_drop).copy()\n    for c in cols_to_fill:\n        if add_indicator and out[c].isna().any():\n            out[f\"{c}_was_na\"] = out[c].isna().astype(\"int8\")\n        out[c] = out[c].fillna(medians[c])\n    return out","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:32.694967Z","iopub.execute_input":"2026-10-17T13:07:32.695426Z","iopub.status.idle":"2026-10-17T13:07:32.703083Z","shell.execute_reply.started":"2026-10-17T13:07:32.69539Z","shell.execute_reply":"2026-10-17T13:07:32.702263Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cols_to_drop, cols_to_fill, medians = fit_num_missing(raw_data_train, num_cols)\ndata_train = transform_num_missing(data_train, cols_to_drop, cols_to_fill, medians)\ndata_val = transform_num_missing(data_val, cols_to_drop, cols_to_fill, medians)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:32.704451Z","iopub.execute_input":"2026-10-17T13:07:32.704713Z","iopub.status.idle":"2026-10-17T13:07:35.26425Z","shell.execute_reply.started":"2026-10-17T13:07:32.704658Z","shell.execute_reply":"2026-10-17T13:07:35.263507Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"X_train = data_train.drop(columns=['Id', 'selected'], axis=1)\ny_train = data_train['selected']\n\nX_val = data_val.drop(columns=['Id', 'selected'], axis=1)\ny_val = data_val['selected']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:35.265224Z","iopub.execute_input":"2026-10-17T13:07:35.265629Z","iopub.status.idle":"2026-10-17T13:07:35.894016Z","shell.execute_reply.started":"2026-10-17T13:07:35.265605Z","shell.execute_reply":"2026-10-17T13:07:35.893189Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Baseline (Pointwise - CatBoost Classifier)","metadata":{}},{"cell_type":"code","source":"from catboost import CatBoostClassifier","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:35.894996Z","iopub.execute_input":"2026-10-17T13:07:35.895231Z","iopub.status.idle":"2026-10-17T13:07:36.51426Z","shell.execute_reply.started":"2026-10-17T13:07:35.89521Z","shell.execute_reply":"2026-10-17T13:07:36.513324Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cat_features = [col for col in X_train.columns if X_train[col].dtype == 'object']","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:36.515384Z","iopub.execute_input":"2026-10-17T13:07:36.515963Z","iopub.status.idle":"2026-10-17T13:07:36.522104Z","shell.execute_reply.started":"2026-10-17T13:07:36.515927Z","shell.execute_reply":"2026-10-17T13:07:36.521392Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cb_model = CatBoostClassifier(\n    iterations=1000,\n    learning_rate=0.05,\n    depth=8,\n    loss_function='Logloss',\n    cat_features=cat_features,\n    random_state=SEED,\n    task_type='GPU',\n    verbose=100\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:36.52311Z","iopub.execute_input":"2026-10-17T13:07:36.52353Z","iopub.status.idle":"2026-10-17T13:07:36.54098Z","shell.execute_reply.started":"2026-10-17T13:07:36.523494Z","shell.execute_reply":"2026-10-17T13:07:36.540165Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"cb_model.fit(X_train, y_train, eval_set=(X_val, y_val), early_stopping_rounds=200, use_best_model=True)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:07:36.541873Z","iopub.execute_input":"2026-10-17T13:07:36.542185Z","iopub.status.idle":"2026-10-17T13:08:48.23806Z","shell.execute_reply.started":"2026-10-17T13:07:36.542163Z","shell.execute_reply":"2026-10-17T13:08:48.237139Z"},"scrolled":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# HitRate@3","metadata":{}},{"cell_type":"code","source":"def hit_rate_at_3(y_true, y_score, groups):\n    df = pd.DataFrame({\n        \"y_true\": np.asarray(y_true),\n        \"y_score\": np.asarray(y_score),\n        \"ranker_id\": np.asarray(groups),\n    })\n\n    # Оставляем только группы с количеством рейсов > 10\n    group_size = df.groupby(\"ranker_id\")[\"ranker_id\"].transform(\"size\")\n    df = df[group_size > 10]\n\n    # Позиция выбранного рейса после сортировки по score\n    df[\"rank\"] = (\n        df.groupby(\"ranker_id\")[\"y_score\"]\n        .rank(method=\"first\", ascending=False)\n    )\n\n    # Есть ли правильный рейс в топ-3 каждой группы\n    hits = df[df[\"y_true\"] == 1].groupby(\"ranker_id\")[\"rank\"].min() <= 3\n\n    return hits.mean()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:08:48.239186Z","iopub.execute_input":"2026-10-17T13:08:48.239946Z","iopub.status.idle":"2026-10-17T13:08:48.246154Z","shell.execute_reply.started":"2026-10-17T13:08:48.239899Z","shell.execute_reply":"2026-10-17T13:08:48.245548Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"baseline_predict = cb_model.predict_proba(X_val)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:08:48.246884Z","iopub.execute_input":"2026-10-17T13:08:48.247142Z","iopub.status.idle":"2026-10-17T13:08:50.148839Z","shell.execute_reply.started":"2026-10-17T13:08:48.247118Z","shell.execute_reply":"2026-10-17T13:08:50.148159Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"hit_rate_at_3(y_val, baseline_predict[:, 1], X_val['ranker_id'])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:08:50.149865Z","iopub.execute_input":"2026-10-17T13:08:50.150529Z","iopub.status.idle":"2026-10-17T13:08:50.292034Z","shell.execute_reply.started":"2026-10-17T13:08:50.150494Z","shell.execute_reply":"2026-10-17T13:08:50.291063Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"cat_drop, cat_fill = fit_cat_missing(raw_data_train, categorical_col)\nnum_drop, num_fill, medians = fit_num_missing(raw_data_train, num_cols)\n\ndel raw_data_train, raw_data_val, data_train, data_val, X_train, y_train\ngc.collect()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:08:50.293375Z","iopub.execute_input":"2026-10-17T13:08:50.293696Z","iopub.status.idle":"2026-10-17T13:08:54.587028Z","shell.execute_reply.started":"2026-10-17T13:08:50.293671Z","shell.execute_reply":"2026-10-17T13:08:54.58632Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def predict_chunk(model, raw_chunk):\n    df = converting_date_columns(raw_chunk, datetime_col)\n    df = transform_cat_missing(df, cat_drop, cat_fill)\n    df = transform_num_missing(df, num_drop, num_fill, medians)\n\n    ids = df[\"Id\"].to_numpy()\n    ranker_ids = df[\"ranker_id\"].to_numpy()\n    X = df.drop(columns=[\"Id\"])[model.feature_names_]\n    return pd.DataFrame({\n        \"Id\": ids,\n        \"ranker_id\": ranker_ids,\n        \"y_score\": model.predict_proba(X)[:, 1].astype(\"float32\"),\n    })","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:08:54.58813Z","iopub.execute_input":"2026-10-17T13:08:54.588416Z","iopub.status.idle":"2026-10-17T13:08:54.594953Z","shell.execute_reply.started":"2026-10-17T13:08:54.588393Z","shell.execute_reply":"2026-10-17T13:08:54.594045Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pf_test = pq.ParquetFile(TEST_DATA_PATH)\nchunks = []\n\nfor batch in pf_test.iter_batches(batch_size=500_000):\n    raw_chunk = batch.to_pandas()\n    chunks.append(predict_chunk(cb_model, raw_chunk))\n    del raw_chunk\n    gc.collect()\n\nsub_df = pd.concat(chunks, ignore_index=True)\ndel chunks\ngc.collect()\n\n\nsub_df[\"selected\"] = (\n    sub_df.groupby(\"ranker_id\")[\"y_score\"]\n    .rank(method=\"first\", ascending=False)\n).astype(\"int\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:08:54.59628Z","iopub.execute_input":"2026-10-17T13:08:54.596614Z","iopub.status.idle":"2026-10-17T13:10:54.396646Z","shell.execute_reply.started":"2026-10-17T13:08:54.59659Z","shell.execute_reply":"2026-10-17T13:10:54.395986Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sub_df = sub_df.drop(columns=\"y_score\")\nsub_df.to_csv(OUTPUT_SUBMISSION_PATH, index=False)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-17T13:10:54.397687Z","iopub.execute_input":"2026-10-17T13:10:54.398099Z","iopub.status.idle":"2026-10-17T13:11:06.695783Z","shell.execute_reply.started":"2026-10-17T13:10:54.398075Z","shell.execute_reply":"2026-10-17T13:11:06.695163Z"}},"outputs":[],"execution_count":null}]}