{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10.0"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Favorita -  LightGBM","metadata":{}},{"cell_type":"code","source":"import os, subprocess, gc, warnings, time\nfrom datetime import date, timedelta\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nwarnings.filterwarnings('ignore')\n\nINPUT_DIR = '/kaggle/input/competitions/favorita-grocery-sales-forecasting'\nWORK_DIR  = '/kaggle/working'\n\nt0 = time.time()\ndef elapsed(): return f\"{time.time()-t0:.0f}s\"\n\nSEED = 42\n\n# 7z 展開ヘルパー\ndef find_7z(name):\n    \"\"\"INPUT_DIR以下から {name}.7z を探す\"\"\"\n    for root, _, files in os.walk(INPUT_DIR):\n        for f in files:\n            if f == f'{name}.7z':\n                return os.path.join(root, f)\n    raise FileNotFoundError(f'{name}.7z が見つかりません: {INPUT_DIR}')\n\ndef extract_7z(src, dst_dir=WORK_DIR):\n    \"\"\"7z ファイルを dst_dir に展開 (7z CLI → py7zr の順で試行)\"\"\"\n    os.makedirs(dst_dir, exist_ok=True)\n    for cmd in [['7z','e'], ['7za','e']]:\n        try:\n            r = subprocess.run(cmd + [src, f'-o{dst_dir}', '-y'],\n                               capture_output=True, timeout=600)\n            if r.returncode == 0:\n                return\n        except (FileNotFoundError, subprocess.TimeoutExpired):\n            pass\n    import py7zr\n    with py7zr.SevenZipFile(src, 'r') as z:\n        z.extractall(path=dst_dir)\n\n# 小ファイルを一括展開\nSMALL_FILES = ['items.csv', 'stores.csv', 'oil.csv',\n               'holidays_events.csv', 'test.csv']\n\nprint('ファイルを展開中...')\nfor name in SMALL_FILES:\n    dst = os.path.join(WORK_DIR, name)\n    if not os.path.exists(dst):\n        extract_7z(find_7z(name))\n        print(f'  {name} 展開完了 ({os.path.getsize(dst)/1e6:.1f} MB)')\n    else:\n        print(f'  {name} 既存')\nprint(f'完了 [{elapsed()}]')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:08:54.144817Z","iopub.execute_input":"2026-06-01T03:08:54.151616Z","iopub.status.idle":"2026-06-01T03:08:54.183651Z","shell.execute_reply.started":"2026-06-01T03:08:54.151468Z","shell.execute_reply":"2026-06-01T03:08:54.182065Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 補助テーブル & test 読み込み\nW = WORK_DIR\n\nitems  = pd.read_csv(f'{W}/items.csv',  index_col='item_nbr')\nstores = pd.read_csv(f'{W}/stores.csv', index_col='store_nbr')\noil    = pd.read_csv(f'{W}/oil.csv',    parse_dates=['date'], index_col='date')\noil    = oil.reindex(pd.date_range(oil.index.min(), '2017-08-31'))\\\n            .interpolate()['dcoilwtico']\n\nholidays = pd.read_csv(f'{W}/holidays_events.csv', parse_dates=['date'])\nnational_hols = set(\n    holidays[(holidays['locale']=='National') & (~holidays['transferred'])]['date']\n)\ndel holidays\n\n# test を (store,item,date) indexで読み込む\n# onpromotion は str で読んで変換 (NaN が含まれる場合に bool 直接指定はエラーになる)\ndf_test = pd.read_csv(\n    f'{W}/test.csv', parse_dates=['date'],\n    dtype={'onpromotion': str}\n).set_index(['store_nbr', 'item_nbr', 'date'])\ndf_test['onpromotion'] = df_test['onpromotion'].map({'True': True, 'False': False}).fillna(False)\n\n# テストのonpromotionを横持ちにする: (store,item) × 16日分の onpromotion\npromo_test = df_test['onpromotion'].unstack(level=-1).fillna(False)\n\nprint(f'テストペア数: {len(promo_test)}  [{elapsed()}]')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:08:54.186721Z","iopub.execute_input":"2026-06-01T03:08:54.187405Z","iopub.status.idle":"2026-06-01T03:08:59.718764Z","shell.execute_reply.started":"2026-06-01T03:08:54.187367Z","shell.execute_reply":"2026-06-01T03:08:59.717784Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# train.csv 展開 → チャンク読み込み → 横持ちに変換\n\nCUTOFF = '2017-05-01'  # 一旦一部のデータで行う\ntrain_csv = os.path.join(WORK_DIR, 'train.csv')\n\nif not os.path.exists(train_csv):\n    print('train.csv.7z を展開中...')\n    extract_7z(find_7z('train.csv'))\n    print(f'展開完了: {os.path.getsize(train_csv)/1e9:.2f} GB  [{elapsed()}]')\n\nprint(f'チャンク読み込み ({CUTOFF} 以降)...')\nchunks = []\nfor chunk in pd.read_csv(\n    train_csv,\n    usecols=['date','store_nbr','item_nbr','unit_sales','onpromotion'],\n    parse_dates=['date'],\n    dtype={'onpromotion': str},   # str で読んで後から変換 (NaN対策)\n    converters={'unit_sales': lambda u: np.log1p(float(u)) if float(u) > 0 else 0},\n    chunksize=10_000_000, low_memory=False\n):\n    chunk = chunk[chunk['date'] >= CUTOFF]\n    if not len(chunk): continue\n    # NaN を False に変換してから bool にする\n    chunk['onpromotion'] = chunk['onpromotion'].map({'True': True, 'False': False}).fillna(False)\n    chunks.append(chunk)\n\ndf_train = pd.concat(chunks, ignore_index=True)\ndel chunks; gc.collect()\nos.remove(train_csv)\nprint(f'df_train: {df_train.shape}  [{elapsed()}]')\n\n# 横持ちに変換 (ceshine と同じ構造)\n#\n#  sales_wide の形:\n#    行 = (store_nbr, item_nbr)\n#    列 = date (Timestamp)\n#    値 = log1p(unit_sales) \n\ndf_train = df_train.set_index(['store_nbr', 'item_nbr', 'date'])\n\npromo_train = df_train['onpromotion'].unstack(level=-1).fillna(False)\nsales_wide  = df_train['unit_sales'].unstack(level=-1).fillna(0)\ndel df_train; gc.collect()\n\n# テストペアに揃え、訓練にないペアは 0 補完\nsales_wide  = sales_wide.reindex(promo_test.index).fillna(0)\npromo_train = promo_train.reindex(promo_test.index).fillna(False)\n\n# 訓練プロモ + テストプロモを結合\npromo_wide = pd.concat([promo_train, promo_test], axis=1).fillna(False)\ndel promo_train, promo_test; gc.collect()\n\nprint(f'sales_wide : {sales_wide.shape}  {sales_wide.memory_usage(deep=True).sum()/1e6:.0f} MB')\nprint(f'promo_wide : {promo_wide.shape}  [{elapsed()}]')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:08:59.720075Z","iopub.execute_input":"2026-06-01T03:08:59.720534Z","iopub.status.idle":"2026-06-01T03:13:58.379934Z","shell.execute_reply.started":"2026-06-01T03:08:59.720503Z","shell.execute_reply":"2026-06-01T03:13:58.378515Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# メタデータ作成\n#\n# sales_wide の行順 (store,item) に揃える\n\nstores['type_enc'] = stores['type'].map({'A':0,'B':1,'C':2,'D':3,'E':4})\nfam_map = {f:i for i,f in enumerate(sorted(items['family'].unique()))}\nitems['family_enc'] = items['family'].map(fam_map)\n\nstore_idx = sales_wide.index.get_level_values('store_nbr')\nitem_idx  = sales_wide.index.get_level_values('item_nbr')\n\nmeta = pd.DataFrame({\n    'type_enc'  : stores.loc[store_idx, 'type_enc'].values,\n    'cluster'   : stores.loc[store_idx, 'cluster'].values,\n    'family_enc': items.loc[item_idx,   'family_enc'].values,\n    'perishable': items.loc[item_idx,   'perishable'].values,\n    'store_nbr' : store_idx,\n    'item_nbr'  : item_idx,\n}, index=sales_wide.index)\n\nprint('メタデータ準備完了')\nprint(meta.head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:13:58.381461Z","iopub.execute_input":"2026-06-01T03:13:58.382201Z","iopub.status.idle":"2026-06-01T03:13:58.425876Z","shell.execute_reply.started":"2026-06-01T03:13:58.38217Z","shell.execute_reply":"2026-06-01T03:13:58.424569Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# get_timespan / prepare_dataset\n\ndef get_timespan(df, dt, minus, periods):\n    \"\"\"dt の minus 日前から periods 日分の列を返す\"\"\"\n    cols = pd.date_range(dt - timedelta(days=minus), periods=periods)\n    return df.reindex(columns=cols, fill_value=0)\n\ndef prepare_dataset(t2017, is_train=True):\n    \"\"\"\n    t2017    : 予測開始日 (datetime.date)\n    is_train : True なら y (n_pairs × 16) も返す\n    \"\"\"\n    # 統計量\n    X = pd.DataFrame({\n        'mean_3' : get_timespan(sales_wide, t2017,  3,  3).mean(axis=1).values,\n        'mean_7' : get_timespan(sales_wide, t2017,  7,  7).mean(axis=1).values,\n        'mean_14': get_timespan(sales_wide, t2017, 14, 14).mean(axis=1).values,\n        'promo_14': get_timespan(promo_wide, t2017, 14, 14).sum(axis=1).values,\n    }, index=sales_wide.index)\n\n    # 予測16日分のプロモフラグ (promo_0〜promo_15)\n    for i in range(16):\n        col = pd.Timestamp(t2017 + timedelta(days=i))\n        X[f'promo_{i}'] = promo_wide.get(col, pd.Series(False, index=sales_wide.index)).values.astype(np.uint8)\n\n    # カレンダー・原油\n    X['month']   = t2017.month\n    X['dow_0']   = t2017.weekday()\n    X['is_hol']  = int(pd.Timestamp(t2017) in national_hols)\n    X['oil']     = float(oil.get(pd.Timestamp(t2017), oil.iloc[-1]))\n\n    # メタデータ\n    # X と meta は同じ MultiIndex (sales_wide.index) を持つので reset_index 不要\n    X = pd.concat([X, meta], axis=1)\n\n    if is_train:\n        target_cols = pd.date_range(t2017, periods=16)\n        y = sales_wide.reindex(columns=target_cols, fill_value=0).values  # (n, 16日分)\n        return X, y\n    return X\n\n# 動作確認\nX_check, y_check = prepare_dataset(date(2017, 7, 26))\nprint(f'X shape: {X_check.shape}  y shape: {y_check.shape}')\nprint(X_check.head(2))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:13:58.428312Z","iopub.execute_input":"2026-06-01T03:13:58.428596Z","iopub.status.idle":"2026-06-01T03:13:58.690676Z","shell.execute_reply.started":"2026-06-01T03:13:58.428573Z","shell.execute_reply":"2026-06-01T03:13:58.689713Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 学習・バリデーション・テストデータ生成\n#\n# 4 窓 + val 窓を準備。\n# モデル化:\n\nTRAIN_DATES = [\n    date(2017, 6, 21),\n    date(2017, 6, 28),\n    date(2017, 7,  5),\n    date(2017, 7, 12),\n]\nVAL_DATE  = date(2017, 7, 26)\nTEST_DATE = date(2017, 8, 16)\n\ndef expand_for_single_model(X, y=None):\n    parts = [X.assign(day=i) for i in range(16)]  # day 列を追加して 16 回複製\n    X_out = pd.concat(parts, ignore_index=True)\n    if y is not None:\n        y_out = y.T.flatten()   # (16, n) → (n*16,)  day=0 の全ペア → day=1 の全ペア → ...\n        return X_out, y_out\n    return X_out\n\nprint('学習データ生成中...')\nX_tr_list, y_tr_list = [], []\nfor t in TRAIN_DATES:\n    X_t, y_t = prepare_dataset(t, is_train=True)\n    X_e, y_e = expand_for_single_model(X_t, y_t)\n    X_tr_list.append(X_e)\n    y_tr_list.append(y_e)\n    print(f'  {t}: X{X_e.shape}  [{elapsed()}]')\n\nX_train = pd.concat(X_tr_list, ignore_index=True)\ny_train = np.concatenate(y_tr_list)\ndel X_tr_list, y_tr_list; gc.collect()\n\nprint('バリデーションデータ生成中...')\nX_val_base, y_val_base = prepare_dataset(VAL_DATE, is_train=True)\nX_val, y_val = expand_for_single_model(X_val_base, y_val_base)\n\nprint('テストデータ生成中...')\nX_test_base = prepare_dataset(TEST_DATE, is_train=False)\n\n# 重み (perishable * 0.25 + 1)\nw_col   = np.tile(meta['perishable'].values * 0.25 + 1.0, 4 * 16)\nw_train = w_col[:len(y_train)]\nw_val   = np.tile(meta['perishable'].values * 0.25 + 1.0, 16)\n\nprint(f'\\nX_train: {X_train.shape}  y_train: {y_train.shape}')\nprint(f'X_val  : {X_val.shape}    X_test : {X_test_base.shape}  [{elapsed()}]')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:13:58.691885Z","iopub.execute_input":"2026-06-01T03:13:58.69222Z","iopub.status.idle":"2026-06-01T03:14:15.502334Z","shell.execute_reply.started":"2026-06-01T03:13:58.692183Z","shell.execute_reply":"2026-06-01T03:14:15.501397Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# LightGBM 学習\nCAT_COLS = ['store_nbr', 'item_nbr', 'dow_0', 'type_enc', 'family_enc']\n\nparams = {\n    'num_leaves'      : 2**5 - 1,\n    'objective'       : 'regression_l2',\n    'max_depth'       : 8,\n    'min_data_in_leaf': 50,\n    'learning_rate'   : 0.05,\n    'feature_fraction': 0.75,\n    'bagging_fraction': 0.75,\n    'bagging_freq'    : 1,\n    'metric'          : 'l2',\n    'num_threads'     : 4,\n    'verbose'         : -1,\n    'seed'            : SEED,\n}\n\ndtrain = lgb.Dataset(X_train, label=y_train, weight=w_train,\n                     categorical_feature=CAT_COLS)\ndval   = lgb.Dataset(X_val,   label=y_val,   weight=w_val,\n                     categorical_feature=CAT_COLS, reference=dtrain)\ndel X_train, y_train, w_train; gc.collect()\n\n# num_boost_round=100 で動作確認\nprint('LightGBM 学習開始 (100 rounds / 動作確認用)...')\nmodel = lgb.train(\n    params, dtrain,\n    num_boost_round=100,\n    valid_sets=[dtrain, dval],\n    valid_names=['train', 'valid'],\n    callbacks=[lgb.early_stopping(50), lgb.log_evaluation(20)],\n    keep_training_booster=True,\n)\nprint(f'最良 iteration: {model.best_iteration}  [{elapsed()}]')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:14:15.503754Z","iopub.execute_input":"2026-06-01T03:14:15.504245Z","iopub.status.idle":"2026-06-01T03:15:59.565355Z","shell.execute_reply.started":"2026-06-01T03:14:15.504209Z","shell.execute_reply":"2026-06-01T03:15:59.564663Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# バリデーションスコア\nval_pred = model.predict(X_val, num_iteration=model.best_iteration)\nmse = np.mean((val_pred - y_val) ** 2)\nprint(f'Val MSE (log scale): {mse:.5f}   RMSE: {mse**0.5:.5f}')\nprint('※ 収束前の参考値。')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:15:59.566361Z","iopub.execute_input":"2026-06-01T03:15:59.566654Z","iopub.status.idle":"2026-06-01T03:16:13.364761Z","shell.execute_reply.started":"2026-06-01T03:15:59.566612Z","shell.execute_reply":"2026-06-01T03:16:13.363833Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# 予測 & サブミッション作成\n\nprint('テスト予測中...')\ntest_pred = []\nfor i in range(16):\n    X_i = X_test_base.assign(day=i)\n    test_pred.append(model.predict(X_i, num_iteration=model.best_iteration))\n\ny_test = np.array(test_pred).transpose()   # (n_pairs, 16) \n\n# サブミッション\ndf_preds = pd.DataFrame(\n    y_test,\n    index=sales_wide.index,                     # MultiIndex(store_nbr, item_nbr)\n    columns=pd.date_range('2017-08-16', periods=16)\n).stack().to_frame('unit_sales')\ndf_preds.index.set_names(['store_nbr', 'item_nbr', 'date'], inplace=True)\n\nsubmission = df_test[['id']].join(df_preds, how='left').fillna(0)\nsubmission['unit_sales'] = np.clip(np.expm1(submission['unit_sales']), 0, 1000)\n\nout_path = f'{WORK_DIR}/submission_wide.csv'\nsubmission.to_csv(out_path, index=False)\n\nprint(f'submission_wide.csv 保存完了  行数: {len(submission)}')\nprint(submission['unit_sales'].describe())\nprint(f'\\n総経過時間: {elapsed()}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-06-01T03:16:13.365909Z","iopub.execute_input":"2026-06-01T03:16:13.366242Z","iopub.status.idle":"2026-06-01T03:16:44.571331Z","shell.execute_reply.started":"2026-06-01T03:16:13.366217Z","shell.execute_reply":"2026-06-01T03:16:44.570354Z"}},"outputs":[],"execution_count":null}]}