{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[],"dockerImageVersionId":28755,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from pathlib import Path\n\nINPUT = Path(\"/kaggle/input\")\n\nfor p in INPUT.rglob(\"*\"):\n    print(p)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.425791Z","iopub.execute_input":"2026-10-05T16:12:49.427002Z","iopub.status.idle":"2026-10-05T16:12:49.438117Z","shell.execute_reply.started":"2026-10-05T16:12:49.426953Z","shell.execute_reply":"2026-10-05T16:12:49.436854Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"PHASE 1","metadata":{}},{"cell_type":"code","source":"from pathlib import Path\nfrom collections import defaultdict\nimport subprocess\nimport shutil\nimport pandas as pd\nimport numpy as np\n\nINPUT = Path(\"/kaggle/input\")\nOUTPUT = Path(\"/kaggle/working/favorita_research\")\n\nOUTPUT.mkdir(parents=True, exist_ok=True)\n\nprint(\"7z:\", shutil.which(\"7z\"))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.440279Z","iopub.execute_input":"2026-10-05T16:12:49.441261Z","iopub.status.idle":"2026-10-05T16:12:49.483527Z","shell.execute_reply.started":"2026-10-05T16:12:49.441183Z","shell.execute_reply":"2026-10-05T16:12:49.482393Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def find_file(filename):\n    matches = list(INPUT.rglob(filename))\n\n    if len(matches) == 0:\n        raise FileNotFoundError(filename)\n\n    print(filename, \"->\", matches[0])\n    return matches[0]\n\n\nTRAIN = find_file(\"train.csv.7z\")\nITEMS = find_file(\"items.csv.7z\")\nTRANSACTIONS = find_file(\"transactions.csv.7z\")\nHOLIDAYS = find_file(\"holidays_events.csv.7z\")\nSTORES = find_file(\"stores.csv.7z\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.485124Z","iopub.execute_input":"2026-10-05T16:12:49.48556Z","iopub.status.idle":"2026-10-05T16:12:49.535622Z","shell.execute_reply.started":"2026-10-05T16:12:49.485529Z","shell.execute_reply":"2026-10-05T16:12:49.533614Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def read_small_7z_csv(path, **kwargs):\n    \"\"\"\n    Dùng cho các file nhỏ như items, stores, transactions, holidays.\n    Giải nén trực tiếp từ 7z vào pandas, không tạo CSV trung gian.\n    \"\"\"\n\n    process = subprocess.Popen(\n        [\"7z\", \"x\", \"-so\", str(path)],\n        stdout=subprocess.PIPE,\n        stderr=subprocess.DEVNULL\n    )\n\n    try:\n        df = pd.read_csv(process.stdout, **kwargs)\n    finally:\n        process.stdout.close()\n        return_code = process.wait()\n\n    if return_code != 0:\n        raise RuntimeError(f\"7z failed: {path}\")\n\n    return df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.539283Z","iopub.execute_input":"2026-10-05T16:12:49.539741Z","iopub.status.idle":"2026-10-05T16:12:49.550357Z","shell.execute_reply.started":"2026-10-05T16:12:49.539646Z","shell.execute_reply":"2026-10-05T16:12:49.548789Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def stream_7z_csv(path, usecols=None, chunksize=500_000):\n    \"\"\"\n    Stream CSV nằm trong .7z và trả từng chunk.\n    Không bao giờ load toàn bộ train.csv vào RAM hoặc disk.\n    \"\"\"\n\n    process = subprocess.Popen(\n        [\"7z\", \"x\", \"-so\", str(path)],\n        stdout=subprocess.PIPE,\n        stderr=subprocess.DEVNULL\n    )\n\n    try:\n        reader = pd.read_csv(\n            process.stdout,\n            usecols=usecols,\n            chunksize=chunksize,\n            low_memory=False\n        )\n\n        for chunk in reader:\n            yield chunk\n\n    finally:\n        if process.stdout:\n            process.stdout.close()\n\n        return_code = process.wait()\n\n        if return_code != 0:\n            raise RuntimeError(f\"7z failed: {path}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.552024Z","iopub.execute_input":"2026-10-05T16:12:49.552809Z","iopub.status.idle":"2026-10-05T16:12:49.597232Z","shell.execute_reply.started":"2026-10-05T16:12:49.552773Z","shell.execute_reply":"2026-10-05T16:12:49.595495Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"items = read_small_7z_csv(ITEMS)\n\nitems.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.59919Z","iopub.execute_input":"2026-10-05T16:12:49.600252Z","iopub.status.idle":"2026-10-05T16:12:49.657592Z","shell.execute_reply.started":"2026-10-05T16:12:49.600218Z","shell.execute_reply":"2026-10-05T16:12:49.656385Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"items.columns","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.659416Z","iopub.execute_input":"2026-10-05T16:12:49.659858Z","iopub.status.idle":"2026-10-05T16:12:49.666775Z","shell.execute_reply.started":"2026-10-05T16:12:49.659814Z","shell.execute_reply":"2026-10-05T16:12:49.665779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"perishable_items = set(\n    items.loc[\n        items[\"perishable\"] == 1,\n        \"item_nbr\"\n    ]\n)\n\nprint(\"Perishable items:\", len(perishable_items))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.668182Z","iopub.execute_input":"2026-10-05T16:12:49.668527Z","iopub.status.idle":"2026-10-05T16:12:49.701886Z","shell.execute_reply.started":"2026-10-05T16:12:49.6685Z","shell.execute_reply":"2026-10-05T16:12:49.700526Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"positive_days = defaultdict(int)\nmonths_present = defaultdict(set)\n\nusecols = [\n    \"date\",\n    \"store_nbr\",\n    \"item_nbr\",\n    \"unit_sales\"\n]\n\nchunk_number = 0\n\nfor chunk in stream_7z_csv(\n    TRAIN,\n    usecols=usecols,\n    chunksize=500_000\n):\n\n    chunk_number += 1\n\n    # date là ISO YYYY-MM-DD nên có thể filter string\n    EARTHQUAKE_START = \"2016-04-16\"\n    EARTHQUAKE_END   = \"2016-05-16\"\n    \n    mask = (\n        (chunk[\"date\"] >= \"2016-01-01\")\n        & (chunk[\"date\"] <= \"2016-12-31\")\n        & ~(\n            (chunk[\"date\"] >= EARTHQUAKE_START)\n            & (chunk[\"date\"] <= EARTHQUAKE_END)\n        )\n        & (chunk[\"item_nbr\"].isin(perishable_items))\n        & (chunk[\"unit_sales\"] > 0)\n    )\n\n    x = chunk.loc[\n        mask,\n        [\"date\", \"store_nbr\", \"item_nbr\"]\n    ].copy()\n\n    if x.empty:\n        continue\n\n    # số ngày có positive sales\n    counts = (\n        x.groupby([\"store_nbr\", \"item_nbr\"])\n        .size()\n    )\n\n    for key, value in counts.items():\n        positive_days[key] += int(value)\n\n    # số tháng xuất hiện\n    x[\"month\"] = x[\"date\"].str[:7]\n\n    unique_months = (\n        x[[\"store_nbr\", \"item_nbr\", \"month\"]]\n        .drop_duplicates()\n    )\n\n    for row in unique_months.itertuples(index=False):\n        key = (row.store_nbr, row.item_nbr)\n        months_present[key].add(row.month)\n\n    if chunk_number % 20 == 0:\n        print(\n            \"chunks:\",\n            chunk_number,\n            \"| candidate pairs:\",\n            len(positive_days)\n        )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:49.705536Z","iopub.execute_input":"2026-10-05T16:12:49.706472Z","iopub.status.idle":"2026-10-05T16:12:57.241137Z","shell.execute_reply.started":"2026-10-05T16:12:49.706439Z","shell.execute_reply":"2026-10-05T16:12:57.238027Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"records = []\n\nfor (store, item), n_days in positive_days.items():\n\n    records.append({\n        \"store_nbr\": store,\n        \"item_nbr\": item,\n        \"positive_days\": n_days,\n        \"months_present\": len(\n            months_present[(store, item)]\n        )\n    })\n\nstats = pd.DataFrame(records)\n\nstats.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.24195Z","iopub.status.idle":"2026-10-05T16:12:57.242252Z","shell.execute_reply.started":"2026-10-05T16:12:57.242116Z","shell.execute_reply":"2026-10-05T16:12:57.242131Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"eligible = stats[\n    (stats[\"positive_days\"] >= 250)\n    & (stats[\"months_present\"] == 12)\n].copy()\n\nprint(\"Eligible store-item pairs:\", len(eligible))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.243824Z","iopub.status.idle":"2026-10-05T16:12:57.244123Z","shell.execute_reply.started":"2026-10-05T16:12:57.243991Z","shell.execute_reply":"2026-10-05T16:12:57.244005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_ranking = (\n    eligible\n    .groupby(\"store_nbr\")\n    .size()\n    .reset_index(name=\"eligible_skus\")\n    .sort_values(\n        [\"eligible_skus\", \"store_nbr\"],\n        ascending=[False, True]\n    )\n)\n\nstore_ranking.head(10)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.246134Z","iopub.status.idle":"2026-10-05T16:12:57.246593Z","shell.execute_reply.started":"2026-10-05T16:12:57.246376Z","shell.execute_reply":"2026-10-05T16:12:57.246403Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"SELECTED_STORE = int(\n    store_ranking.iloc[0][\"store_nbr\"]\n)\n\nprint(\"Selected store:\", SELECTED_STORE)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.249825Z","iopub.status.idle":"2026-10-05T16:12:57.250187Z","shell.execute_reply.started":"2026-10-05T16:12:57.250028Z","shell.execute_reply":"2026-10-05T16:12:57.250044Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"store_candidates = (\n    eligible[\n        eligible[\"store_nbr\"] == SELECTED_STORE\n    ]\n    .copy()\n)\n\n# Tổng số SKU đủ điều kiện tại store được chọn\nN_ELIGIBLE = len(store_candidates)\n\n# Số positive_days cao nhất\nMAX_POSITIVE_DAYS = int(\n    store_candidates[\"positive_days\"].max()\n)\n\n# Bao nhiêu SKU cùng đạt mức cao nhất\nN_TIED_MAX = int(\n    (\n        store_candidates[\"positive_days\"]\n        == MAX_POSITIVE_DAYS\n    ).sum()\n)\n\nprint(\"Selected store:\", SELECTED_STORE)\nprint(\"Total eligible SKUs:\", N_ELIGIBLE)\nprint(\"Maximum positive_days:\", MAX_POSITIVE_DAYS)\nprint(\"SKUs tied at maximum:\", N_TIED_MAX)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.251842Z","iopub.status.idle":"2026-10-05T16:12:57.252238Z","shell.execute_reply.started":"2026-10-05T16:12:57.252072Z","shell.execute_reply":"2026-10-05T16:12:57.252089Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"selected_items_df = (\n    store_candidates[\n        store_candidates[\"positive_days\"]\n        == MAX_POSITIVE_DAYS\n    ]\n    .sort_values(\"item_nbr\")\n    .copy()\n)\n\nselected_items_df[\"rank\"] = range(\n    1,\n    len(selected_items_df) + 1\n)\n\nselected_items_df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.254254Z","iopub.status.idle":"2026-10-05T16:12:57.254849Z","shell.execute_reply.started":"2026-10-05T16:12:57.25454Z","shell.execute_reply":"2026-10-05T16:12:57.254567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"selected_items_df[\"total_eligible_skus\"] = N_ELIGIBLE\nselected_items_df[\"max_positive_days\"] = MAX_POSITIVE_DAYS\nselected_items_df[\"n_tied_at_max\"] = N_TIED_MAX\n\nselected_items_df[\"earthquake_exclusion_start\"] = \"2016-04-16\"\nselected_items_df[\"earthquake_exclusion_end\"] = \"2016-05-16\"\n\nselected_items_df[\"selection_rule\"] = (\n    \"retain all SKUs tied at maximum positive_days\"\n)\n\nselected_items_df[\"tie_break_rule\"] = \"none\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.256737Z","iopub.status.idle":"2026-10-05T16:12:57.257124Z","shell.execute_reply.started":"2026-10-05T16:12:57.256978Z","shell.execute_reply":"2026-10-05T16:12:57.256995Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"Selected store:\", SELECTED_STORE)\nprint(\"Max positive days:\", MAX_POSITIVE_DAYS)\nprint(\"Store candidates:\", len(store_candidates))\nprint(\"Selected items:\", len(selected_items_df))\n\ndisplay(selected_items_df.head())\ndisplay(items.head())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.259874Z","iopub.status.idle":"2026-10-05T16:12:57.260333Z","shell.execute_reply.started":"2026-10-05T16:12:57.260113Z","shell.execute_reply":"2026-10-05T16:12:57.260139Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tied_max_candidates = (\n    store_candidates[\n        store_candidates[\"positive_days\"] == MAX_POSITIVE_DAYS\n    ]\n    .merge(\n        items[\n            [\"item_nbr\", \"family\", \"class\", \"perishable\"]\n        ],\n        on=\"item_nbr\",\n        how=\"left\",\n        validate=\"one_to_one\"\n    )\n    .sort_values(\"item_nbr\")\n    .reset_index(drop=True)\n)\n\nprint(\"Number of tied-max candidates:\", len(tied_max_candidates))\n\ndisplay(tied_max_candidates.head(10))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.261912Z","iopub.status.idle":"2026-10-05T16:12:57.262357Z","shell.execute_reply.started":"2026-10-05T16:12:57.262173Z","shell.execute_reply":"2026-10-05T16:12:57.262198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"assert len(tied_max_candidates) == N_TIED_MAX\n\nassert tied_max_candidates[\"item_nbr\"].nunique() == len(\n    tied_max_candidates\n)\n\nassert tied_max_candidates[\"perishable\"].eq(1).all()\n\nassert tied_max_candidates[\"family\"].notna().all()\n\nprint(\"Audit checks passed.\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.264005Z","iopub.status.idle":"2026-10-05T16:12:57.264357Z","shell.execute_reply.started":"2026-10-05T16:12:57.264208Z","shell.execute_reply":"2026-10-05T16:12:57.264228Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"tied_path = OUTPUT / \"tied_max_candidates.csv\"\n\ntied_max_candidates.to_csv(\n    tied_path,\n    index=False\n)\n\nprint(\"Saved:\", tied_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.266726Z","iopub.status.idle":"2026-10-05T16:12:57.267226Z","shell.execute_reply.started":"2026-10-05T16:12:57.267041Z","shell.execute_reply":"2026-10-05T16:12:57.267058Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"selected_with_family = (\n    selected_items_df\n    .merge(\n        items[\n            [\"item_nbr\", \"family\", \"class\", \"perishable\"]\n        ],\n        on=\"item_nbr\",\n        how=\"left\",\n        validate=\"one_to_one\"\n    )\n)\n\nassert len(selected_with_family) == len(selected_items_df)\nassert selected_with_family[\"family\"].notna().all()\nassert selected_with_family[\"perishable\"].eq(1).all()\n\ndisplay(\n    selected_with_family[\n        [\"rank\", \"item_nbr\", \"family\", \"positive_days\"]\n    ].head(10)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.269319Z","iopub.status.idle":"2026-10-05T16:12:57.269874Z","shell.execute_reply.started":"2026-10-05T16:12:57.269586Z","shell.execute_reply":"2026-10-05T16:12:57.269615Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"pool_family = (\n    tied_max_candidates\n    .groupby(\"family\")\n    .size()\n    .rename(\"pool_n\")\n)\n\nsample_family = (\n    selected_with_family\n    .groupby(\"family\")\n    .size()\n    .rename(\"sample_n\")\n)\n\nfamily_audit = (\n    pd.concat(\n        [pool_family, sample_family],\n        axis=1\n    )\n    .fillna(0)\n    .reset_index()\n)\n\nfamily_audit[\"pool_n\"] = (\n    family_audit[\"pool_n\"]\n    .astype(int)\n)\n\nfamily_audit[\"sample_n\"] = (\n    family_audit[\"sample_n\"]\n    .astype(int)\n)\n\nfamily_audit[\"pool_pct\"] = (\n    family_audit[\"pool_n\"]\n    / len(tied_max_candidates)\n)\n\nfamily_audit[\"sample_pct\"] = (\n    family_audit[\"sample_n\"]\n    / len(selected_with_family)\n)\n\nfamily_audit[\"pct_point_diff\"] = (\n    family_audit[\"sample_pct\"]\n    - family_audit[\"pool_pct\"]\n)\n\nfamily_audit = (\n    family_audit\n    .sort_values(\n        \"sample_n\",\n        ascending=False\n    )\n    .reset_index(drop=True)\n)\n\ndisplay(family_audit)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.271993Z","iopub.status.idle":"2026-10-05T16:12:57.273183Z","shell.execute_reply.started":"2026-10-05T16:12:57.272925Z","shell.execute_reply":"2026-10-05T16:12:57.272957Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"family_audit_path = (\n    OUTPUT / \"selection_family_audit.csv\"\n)\n\nfamily_audit.to_csv(\n    family_audit_path,\n    index=False\n)\n\nprint(\"Saved:\", family_audit_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.274736Z","iopub.status.idle":"2026-10-05T16:12:57.275386Z","shell.execute_reply.started":"2026-10-05T16:12:57.275083Z","shell.execute_reply":"2026-10-05T16:12:57.275103Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\n    \"Pool total:\",\n    family_audit[\"pool_n\"].sum()\n)\n\nprint(\n    \"Sample total:\",\n    family_audit[\"sample_n\"].sum()\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.277773Z","iopub.status.idle":"2026-10-05T16:12:57.278159Z","shell.execute_reply.started":"2026-10-05T16:12:57.278011Z","shell.execute_reply":"2026-10-05T16:12:57.278029Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"family_audit_display = family_audit.copy()\n\nfamily_audit_display[\"pool_pct\"] = (\n    family_audit_display[\"pool_pct\"] * 100\n).round(1)\n\nfamily_audit_display[\"sample_pct\"] = (\n    family_audit_display[\"sample_pct\"] * 100\n).round(1)\n\nfamily_audit_display[\"pct_point_diff\"] = (\n    family_audit_display[\"pct_point_diff\"] * 100\n).round(1)\n\ndisplay(family_audit_display)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.28022Z","iopub.status.idle":"2026-10-05T16:12:57.280617Z","shell.execute_reply.started":"2026-10-05T16:12:57.280461Z","shell.execute_reply":"2026-10-05T16:12:57.280477Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"manifest_path = OUTPUT / \"sample_manifest.csv\"\n\nselected_items_df.to_csv(\n    manifest_path,\n    index=False\n)\n\ndisplay(selected_items_df.head())\n\nprint(\"Manifest saved to:\", manifest_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.282054Z","iopub.status.idle":"2026-10-05T16:12:57.282581Z","shell.execute_reply.started":"2026-10-05T16:12:57.282326Z","shell.execute_reply":"2026-10-05T16:12:57.282355Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"selection_summary = pd.DataFrame([{\n    \"selected_store\": SELECTED_STORE,\n    \"total_eligible_skus\": N_ELIGIBLE,\n    \"max_positive_days\": MAX_POSITIVE_DAYS,\n    \"n_tied_at_max\": N_TIED_MAX,\n    \"sample_size\": len(selected_items_df),\n    \"earthquake_exclusion_start\": \"2016-04-16\",\n    \"earthquake_exclusion_end\": \"2016-05-16\",\n    \"selection_rule\": \"retain all SKUs tied at maximum positive_days\",\n    \"tie_break_rule\": \"none\"\n}])\n\nselection_summary.to_csv(\n    OUTPUT / \"selection_summary.csv\",\n    index=False\n)\n\ndisplay(selection_summary)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.284137Z","iopub.status.idle":"2026-10-05T16:12:57.284474Z","shell.execute_reply.started":"2026-10-05T16:12:57.284318Z","shell.execute_reply":"2026-10-05T16:12:57.284334Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"PHASE 2","metadata":{}},{"cell_type":"code","source":"selected_items = set(\n    selected_items_df[\"item_nbr\"]\n)\n\nusecols = [\n    \"date\",\n    \"store_nbr\",\n    \"item_nbr\",\n    \"unit_sales\",\n    \"onpromotion\"\n]\n\nresearch_parts = []\n\nchunk_number = 0\n\nfor chunk in stream_7z_csv(\n    TRAIN,\n    usecols=usecols,\n    chunksize=500_000\n):\n\n    chunk_number += 1\n\n    mask = (\n        (chunk[\"date\"] >= \"2016-01-01\")\n        & (chunk[\"date\"] <= \"2017-08-15\")\n        & (chunk[\"store_nbr\"] == SELECTED_STORE)\n        & (chunk[\"item_nbr\"].isin(selected_items))\n    )\n\n    x = chunk.loc[mask].copy()\n\n    if not x.empty:\n        research_parts.append(x)\n\n    if chunk_number % 20 == 0:\n        print(\"chunks:\", chunk_number)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.286524Z","iopub.status.idle":"2026-10-05T16:12:57.286924Z","shell.execute_reply.started":"2026-10-05T16:12:57.286753Z","shell.execute_reply":"2026-10-05T16:12:57.286779Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sales = pd.concat(\n    research_parts,\n    ignore_index=True\n)\n\nsales[\"date\"] = pd.to_datetime(\n    sales[\"date\"]\n)\n\nprint(sales.shape)\n\nsales.head()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.288081Z","iopub.status.idle":"2026-10-05T16:12:57.288516Z","shell.execute_reply.started":"2026-10-05T16:12:57.288287Z","shell.execute_reply":"2026-10-05T16:12:57.288311Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"sales_path = OUTPUT / \"research_sales.parquet\"\n\nsales.to_parquet(\n    sales_path,\n    index=False,\n    compression=\"zstd\"\n)\n\nprint(sales_path)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.29047Z","iopub.status.idle":"2026-10-05T16:12:57.29097Z","shell.execute_reply.started":"2026-10-05T16:12:57.290724Z","shell.execute_reply":"2026-10-05T16:12:57.290749Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"research_items = items[\n    items[\"item_nbr\"].isin(selected_items)\n].copy()\n\nresearch_items.to_parquet(\n    OUTPUT / \"research_items.parquet\",\n    index=False,\n    compression=\"zstd\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.292935Z","iopub.status.idle":"2026-10-05T16:12:57.293238Z","shell.execute_reply.started":"2026-10-05T16:12:57.293102Z","shell.execute_reply":"2026-10-05T16:12:57.293116Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"transactions = read_small_7z_csv(\n    TRANSACTIONS\n)\n\ntransactions[\"date\"] = pd.to_datetime(\n    transactions[\"date\"]\n)\n\nresearch_transactions = transactions[\n    (transactions[\"store_nbr\"] == SELECTED_STORE)\n    & (transactions[\"date\"] >= \"2016-01-01\")\n    & (transactions[\"date\"] <= \"2017-08-15\")\n].copy()\n\nresearch_transactions.to_parquet(\n    OUTPUT / \"research_transactions.parquet\",\n    index=False,\n    compression=\"zstd\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.29507Z","iopub.status.idle":"2026-10-05T16:12:57.295527Z","shell.execute_reply.started":"2026-10-05T16:12:57.295292Z","shell.execute_reply":"2026-10-05T16:12:57.295317Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"holidays = read_small_7z_csv(\n    HOLIDAYS\n)\n\nholidays[\"date\"] = pd.to_datetime(\n    holidays[\"date\"]\n)\n\nresearch_holidays = holidays[\n    (holidays[\"date\"] >= \"2016-01-01\")\n    & (holidays[\"date\"] <= \"2017-08-15\")\n].copy()\n\nresearch_holidays.to_parquet(\n    OUTPUT / \"research_holidays.parquet\",\n    index=False,\n    compression=\"zstd\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.297275Z","iopub.status.idle":"2026-10-05T16:12:57.29762Z","shell.execute_reply.started":"2026-10-05T16:12:57.297476Z","shell.execute_reply":"2026-10-05T16:12:57.297492Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"stores = read_small_7z_csv(\n    STORES\n)\n\nresearch_store = stores[\n    stores[\"store_nbr\"] == SELECTED_STORE\n].copy()\n\nresearch_store.to_parquet(\n    OUTPUT / \"research_store.parquet\",\n    index=False\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.299336Z","iopub.status.idle":"2026-10-05T16:12:57.299779Z","shell.execute_reply.started":"2026-10-05T16:12:57.299525Z","shell.execute_reply":"2026-10-05T16:12:57.299549Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"for file in OUTPUT.iterdir():\n\n    size_mb = file.stat().st_size / (1024 ** 2)\n\n    print(\n        f\"{file.name:35s}\",\n        f\"{size_mb:.3f} MB\"\n    )","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.301094Z","iopub.status.idle":"2026-10-05T16:12:57.301455Z","shell.execute_reply.started":"2026-10-05T16:12:57.301318Z","shell.execute_reply":"2026-10-05T16:12:57.301334Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import shutil\n\narchive = shutil.make_archive(\n    \"/kaggle/working/favorita_research_extract\",\n    \"zip\",\n    OUTPUT\n)\n\nprint(archive)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-05T16:12:57.306079Z","iopub.status.idle":"2026-10-05T16:12:57.306585Z","shell.execute_reply.started":"2026-10-05T16:12:57.306347Z","shell.execute_reply":"2026-10-05T16:12:57.306374Z"}},"outputs":[],"execution_count":null}]}