{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.x"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Stage 3 v3: Full Statistics + Period Stats (300 features)\n## Features: v2 + [90,140] windows + offset versions + period stats","metadata":{}},{"cell_type":"code","source":"import os, subprocess, gc, warnings, time\nfrom datetime import date, timedelta\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nwarnings.filterwarnings('ignore')\n\nINPUT_DIR = '/kaggle/input/competitions/favorita-grocery-sales-forecasting'\nWORK_DIR  = '/kaggle/working'\n\nt0 = time.time()\ndef elapsed(): return f\"{time.time()-t0:.0f}s\"\n\nSEED = 42\nVERSION = 'v3'\nprint(f'VERSION: {VERSION}')\nprint('1st_place style: No model retention')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_7z(name):\n    for root, _, files in os.walk(INPUT_DIR):\n        for f in files:\n            if f == f'{name}.7z':\n                return os.path.join(root, f)\n    raise FileNotFoundError(f'{name}.7z')\n\ndef extract_7z(src, dst_dir=WORK_DIR):\n    os.makedirs(dst_dir, exist_ok=True)\n    for cmd in [['7z','e'], ['7za','e']]:\n        try:\n            r = subprocess.run(cmd + [src, f'-o{dst_dir}', '-y'],\n                               capture_output=True, timeout=600)\n            if r.returncode == 0: return\n        except: pass\n    import py7zr\n    with py7zr.SevenZipFile(src, 'r') as z:\n        z.extractall(path=dst_dir)\n\nfor name in ['items.csv', 'stores.csv', 'oil.csv', 'holidays_events.csv', 'test.csv']:\n    dst = os.path.join(WORK_DIR, name)\n    if not os.path.exists(dst):\n        extract_7z(find_7z(name))\n\nW = WORK_DIR\nitems  = pd.read_csv(f'{W}/items.csv',  index_col='item_nbr')\nstores = pd.read_csv(f'{W}/stores.csv', index_col='store_nbr')\noil    = pd.read_csv(f'{W}/oil.csv',    parse_dates=['date'], index_col='date')\noil    = oil.reindex(pd.date_range(oil.index.min(), '2017-08-31')).interpolate()['dcoilwtico']\n\nholidays = pd.read_csv(f'{W}/holidays_events.csv', parse_dates=['date'])\nnational_hols = set(holidays[(holidays['locale']=='National') & (~holidays['transferred'])]['date'])\ndel holidays\n\ndf_test = pd.read_csv(f'{W}/test.csv', parse_dates=['date'], dtype={'onpromotion': str})\ndf_test = df_test.set_index(['store_nbr', 'item_nbr', 'date'])\ndf_test['onpromotion'] = df_test['onpromotion'].map({'True': True, 'False': False}).fillna(False)\npromo_test = df_test['onpromotion'].unstack(level=-1).fillna(False)\n\nprint('Data loaded')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CUTOFF = '2017-05-01'\ntrain_csv = os.path.join(WORK_DIR, 'train.csv')\n\nif not os.path.exists(train_csv):\n    extract_7z(find_7z('train.csv'))\n\nchunks = []\nfor chunk in pd.read_csv(\n    train_csv, usecols=['date','store_nbr','item_nbr','unit_sales','onpromotion'],\n    parse_dates=['date'], dtype={'onpromotion': str},\n    converters={'unit_sales': lambda u: np.log1p(float(u)) if float(u) > 0 else 0},\n    chunksize=10_000_000, low_memory=False\n):\n    chunk = chunk[chunk['date'] >= CUTOFF]\n    if not len(chunk): continue\n    chunk['onpromotion'] = chunk['onpromotion'].map({'True': True, 'False': False}).fillna(False)\n    chunks.append(chunk)\n\ndf_train = pd.concat(chunks, ignore_index=True)\ndel chunks; gc.collect()\nos.remove(train_csv)\n\ndf_train = df_train.set_index(['store_nbr', 'item_nbr', 'date'])\npromo_train = df_train['onpromotion'].unstack(level=-1).fillna(False)\nsales_wide  = df_train['unit_sales'].unstack(level=-1).fillna(0)\ndel df_train; gc.collect()\n\nsales_wide  = sales_wide.reindex(promo_test.index).fillna(0)\npromo_train = promo_train.reindex(promo_test.index).fillna(False)\npromo_wide  = pd.concat([promo_train, promo_test], axis=1).fillna(False)\ndel promo_train, promo_test; gc.collect()\n\nprint(f'Train/test ready: {sales_wide.shape}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores['type_enc'] = stores['type'].map({'A':0,'B':1,'C':2,'D':3,'E':4})\nfam_map = {f:i for i,f in enumerate(sorted(items['family'].unique()))}\nitems['family_enc'] = items['family'].map(fam_map)\n\nstore_idx = sales_wide.index.get_level_values('store_nbr')\nitem_idx  = sales_wide.index.get_level_values('item_nbr')\n\nmeta = pd.DataFrame({\n    'type_enc'  : stores.loc[store_idx, 'type_enc'].values,\n    'cluster'   : stores.loc[store_idx, 'cluster'].values,\n    'family_enc': items.loc[item_idx,   'family_enc'].values,\n    'perishable': items.loc[item_idx,   'perishable'].values,\n    'store_nbr' : store_idx.values,\n    'item_nbr'  : item_idx.values,\n}, index=sales_wide.index)\n\ndf_item    = sales_wide.groupby(level='item_nbr').sum()\npromo_item = promo_wide.groupby(level='item_nbr').sum()\n\nclass_arr = items.loc[item_idx, 'class'].values\ntmp   = sales_wide.copy()\ntmp_p = promo_wide.copy()\ntmp.index   = pd.MultiIndex.from_arrays([class_arr, store_idx.values], names=['class','store_nbr'])\ntmp_p.index = pd.MultiIndex.from_arrays([class_arr, store_idx.values], names=['class','store_nbr'])\ndf_store_class    = tmp.groupby(level=['class','store_nbr']).sum()\npromo_store_class = tmp_p.groupby(level=['class','store_nbr']).sum()\ndel tmp, tmp_p; gc.collect()\n\nitem_pos         = pd.Series(range(len(df_item)), index=df_item.index)\nitem_reindex_pos = item_pos.reindex(item_idx).values.astype(int)\n\nsc_keys        = pd.MultiIndex.from_arrays([class_arr, store_idx.values])\nsc_pos         = pd.Series(range(len(df_store_class)), index=df_store_class.index)\nsc_reindex_pos = sc_pos.reindex(sc_keys).values.astype(int)\n\nprint('Aggregation prepared')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def get_timespan(df, dt, minus, periods):\n    cols = pd.date_range(dt - timedelta(days=minus), periods=periods)\n    return df.reindex(columns=cols, fill_value=0)\n\ndef prepare_dataset_v3(df, promo_df, t2017, is_train=True):\n    X = pd.DataFrame(index=df.index)\n    \n    X['promo_14'] = get_timespan(promo_df, t2017, 14, 14).sum(axis=1).values\n    for i in range(16):\n        col = pd.Timestamp(t2017 + timedelta(days=i))\n        X[f'promo_{i}'] = promo_df.get(col, pd.Series(False, index=df.index)).values.astype(np.uint8)\n    \n    X['month'] = t2017.month\n    X['dow_0'] = t2017.weekday()\n    X['is_hol'] = int(pd.Timestamp(t2017) in national_hols)\n    X['oil'] = float(oil.get(pd.Timestamp(t2017), oil.iloc[-1]))\n    \n    for w in [3, 7, 14, 30, 60, 90, 140]:\n        tmp = get_timespan(df, t2017, w, w)\n        X[f'mean_{w}'] = tmp.mean(axis=1).values\n        X[f'median_{w}'] = tmp.median(axis=1).values\n        X[f'max_{w}'] = tmp.max(axis=1).values\n        X[f'min_{w}'] = tmp.min(axis=1).values\n    \n    for w in [3, 7, 14, 30, 60, 90, 140]:\n        tmp = get_timespan(df, t2017 - timedelta(days=7), w, w)\n        X[f'mean_{w}_7d'] = tmp.mean(axis=1).values\n        X[f'median_{w}_7d'] = tmp.median(axis=1).values\n        X[f'max_{w}_7d'] = tmp.max(axis=1).values\n        X[f'min_{w}_7d'] = tmp.min(axis=1).values\n    \n    for w in [7, 14, 30, 60, 140]:\n        tmp_s = get_timespan(df, t2017, w, w)\n        X[f'has_sales_{w}'] = (tmp_s > 0).sum(axis=1).values\n        tmp_p = get_timespan(promo_df, t2017, w, w)\n        X[f'has_promo_{w}'] = tmp_p.sum(axis=1).values\n    \n    if is_train:\n        target_cols = pd.date_range(t2017, periods=16)\n        y = sales_wide.reindex(columns=target_cols, fill_value=0).values\n        return X, y\n    return X\n\ndef build_features_v3(t, is_train=True):\n    if is_train:\n        X1, y = prepare_dataset_v3(sales_wide, promo_wide, t, is_train=True)\n    else:\n        X1 = prepare_dataset_v3(sales_wide, promo_wide, t, is_train=False)\n    X1 = pd.concat([X1, meta], axis=1).reset_index(drop=True)\n    X2 = prepare_dataset_v3(df_item, promo_item, t, is_train=False)\n    X2 = X2.iloc[item_reindex_pos].reset_index(drop=True).add_prefix('item_')\n    X3 = prepare_dataset_v3(df_store_class, promo_store_class, t, is_train=False)\n    X3 = X3.iloc[sc_reindex_pos].reset_index(drop=True).add_prefix('sc_')\n    X_all = pd.concat([X1, X2, X3], axis=1).astype('float32')\n    if is_train:\n        return X_all, y\n    return X_all\n\nprint('Feature functions ready')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"TRAIN_DATES = [date(2017, 6, 21), date(2017, 6, 28), date(2017, 7, 5), date(2017, 7, 12)]\nVAL_DATE  = date(2017, 7, 26)\nTEST_DATE = date(2017, 8, 16)\n\nprint('Building features...')\nX_tr_list, y_tr_list = [], []\nfor t in TRAIN_DATES:\n    X_t, y_t = build_features_v3(t, is_train=True)\n    X_tr_list.append(X_t)\n    y_tr_list.append(y_t)\n    print(f'  {t}: {X_t.shape}')\n\nX_train = pd.concat(X_tr_list, ignore_index=True)\ny_train = np.concatenate(y_tr_list)\ndel X_tr_list, y_tr_list; gc.collect()\n\nX_val, y_val = build_features_v3(VAL_DATE, is_train=True)\nX_test = build_features_v3(TEST_DATE, is_train=False)\n\nw_train_base = (meta['perishable'].values * 0.25 + 1.0).astype('float32')\nw_train = np.tile(w_train_base, len(TRAIN_DATES))\nw_val   = (meta['perishable'].values * 0.25 + 1.0).astype('float32')\n\nprint(f'\\nX_train: {X_train.shape}')\nprint(f'X_val: {X_val.shape}')\nprint(f'X_test: {X_test.shape}')\nprint(f'\\n=== {VERSION.upper()} FEATURE COUNT ===' )\nprint(f'Total: {X_train.shape[1]} features ({X_train.shape[1]//3} per level × 3)')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"CAT_COLS = ['store_nbr', 'item_nbr', 'dow_0', 'type_enc', 'family_enc']\nparams = {\n    'num_leaves': 31, 'objective': 'regression_l2', 'max_depth': 8,\n    'min_data_in_leaf': 50, 'learning_rate': 0.05, 'feature_fraction': 0.75,\n    'bagging_fraction': 0.75, 'bagging_freq': 1, 'metric': 'l2',\n    'num_threads': 4, 'verbose': -1, 'seed': SEED,\n}\n\nprint('Training 16 models...')\nval_pred_list, test_pred_list = [], []\n\nfor day_idx in range(16):\n    if day_idx % 4 == 0:\n        print(f'Day {day_idx+1}/16 [{elapsed()}]')\n    \n    dtrain = lgb.Dataset(X_train, label=y_train[:, day_idx], weight=w_train[:len(y_train)], categorical_feature=CAT_COLS)\n    dval   = lgb.Dataset(X_val, label=y_val[:, day_idx], weight=w_val, categorical_feature=CAT_COLS, reference=dtrain)\n    \n    model = lgb.train(params, dtrain, num_boost_round=1000,\n                      valid_sets=[dtrain, dval], valid_names=['train', 'valid'],\n                      callbacks=[lgb.early_stopping(50), lgb.log_evaluation(100)],\n                      keep_training_booster=True)\n    \n    imp = model.feature_importance(importance_type='gain')\n    top_idx = np.argsort(imp)[-20:][::-1]\n    print(f'\\nDay {day_idx+1} TOP20 Features:')\n    for rank, idx in enumerate(top_idx, 1):\n        print(f'  {rank:2d}. {X_train.columns[idx]:25s} {imp[idx]:8.0f}')\n    \n    val_pred_list.append(model.predict(X_val, num_iteration=model.best_iteration))\n    test_pred_list.append(model.predict(X_test, num_iteration=model.best_iteration))\n    \n    del dtrain, dval, model; gc.collect()\n\nprint(f'\\nTraining complete [{elapsed()}]')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"val_pred = np.array(val_pred_list).transpose()\n\ndef nwrmsle(y_true_log, y_pred_log, weights):\n    diff = y_pred_log - y_true_log\n    return float(np.sqrt(np.sum(weights * diff**2) / np.sum(weights)))\n\nw_val_matrix = np.tile(w_val.reshape(-1, 1), 16)\nlocal_nwrmsle = nwrmsle(y_val, val_pred, w_val_matrix)\n\nprint(f'\\n=== {VERSION.upper()} RESULTS ===') \nprint(f'Val NWRMSLE: {local_nwrmsle:.5f}')\nprint(f'Time: {elapsed()}')","metadata":{},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_test = np.array(test_pred_list).transpose()\n\ndf_preds = pd.DataFrame(y_test, index=sales_wide.index,\n                        columns=pd.date_range('2017-08-16', periods=16)).stack().to_frame('unit_sales')\ndf_preds.index.set_names(['store_nbr', 'item_nbr', 'date'], inplace=True)\n\nsubmission = df_test[['id']].join(df_preds, how='left').fillna(0)\nsubmission['unit_sales'] = np.clip(np.expm1(submission['unit_sales']), 0, 1000)\n\nout_path = f'{WORK_DIR}/submission_{VERSION}.csv'\nsubmission.to_csv(out_path, index=False)\n\nprint(f'Submission saved: submission_{VERSION}.csv')\nprint(f'Max: {submission[\"unit_sales\"].max():.2f}')\nprint(f'Mean: {submission[\"unit_sales\"].mean():.4f}')","metadata":{},"outputs":[],"execution_count":null}]}