{"nbformat": 4, "metadata": {"language_info": {"mimetype": "text/x-python", "codemirror_mode": {"name": "ipython", "version": 3}, "nbconvert_exporter": "python", "version": "3.6.3", "name": "python", "file_extension": ".py", "pygments_lexer": "ipython3"}, "kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}}, "cells": [{"metadata": {"_cell_guid": "de224c83-fa80-49cf-b66b-f2b6e2e8e67e", "_uuid": "86bfbcdbb5f6052cae55d8a2d21354c3bef738d2"}, "source": ["Original is [here](https://www.kaggle.com/ceshine/lgbm-starter)\n", "\n", "This version uses XGBoost instead of LightGBM.  But it runs too slowly, so it's set to do only the first 3 days.  Run it at home maybe with the full set?  Maybe somebody can figure out a way to speed it up.  It runs very quickly on my computer at home (allowing n_jobs to default, and it uses all 8 virtual cores; on Kaggle I tried setting n_jobs, but it doesn't make any clear difference).  Note that the CV score is misleading, since the first 3 days are the easiest to predict.\n", "\n", "I commented out the submission section, becuase obviously you don't want to submit something that only predicts the first 3 days."], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_cell_guid": "dc07f651-2a29-4cc5-80d4-fca67d0a7b66", "_uuid": "bb2e3efe23cfe1e68c0cd622be3de2a8e81a9a7a"}, "outputs": [], "source": ["from datetime import date, timedelta\n", "\n", "import pandas as pd\n", "import numpy as np\n", "from sklearn.metrics import mean_squared_error\n", "import lightgbm as lgb\n", "import xgboost as xgb"], "execution_count": null, "cell_type": "code"}, {"metadata": {"collapsed": true, "_cell_guid": "bd1da8c1-0a03-495e-a448-6957e6fdc60c", "_uuid": "116f514148006d967b091124f131f2a341a47a99"}, "outputs": [], "source": ["df_train = pd.read_csv(\n", "    '../input/train.csv', usecols=[1, 2, 3, 4, 5],\n", "    dtype={'onpromotion': bool},\n", "    converters={'unit_sales': lambda u: np.log1p(\n", "        float(u)) if float(u) > 0 else 0},\n", "    parse_dates=[\"date\"],\n", "    skiprows=range(1, 66458909)  # 2016-01-01\n", ")\n", "\n", "df_test = pd.read_csv(\n", "    \"../input/test.csv\", usecols=[0, 1, 2, 3, 4],\n", "    dtype={'onpromotion': bool},\n", "    parse_dates=[\"date\"]  # , date_parser=parser\n", ").set_index(\n", "    ['store_nbr', 'item_nbr', 'date']\n", ")\n", "\n", "items = pd.read_csv(\n", "    \"../input/items.csv\",\n", ").set_index(\"item_nbr\")"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "372f8dd0-cd63-4461-a33b-88ed5d2d636e", "_uuid": "0a94f0d6750d42dfd2cedf2728916e50de5f0f36"}, "outputs": [], "source": ["df_train.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "fbb6cbf5-064f-4d33-8272-706a06a6317e", "_uuid": "a1fcda19ec288d3073f2e2a589daad78359ccd9c"}, "outputs": [], "source": ["df_train.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "8c481ac6-4224-4dec-9d5a-0bfb0ecc001a", "_uuid": "9abd08f8b1a6f672a18d689a5de9b53dc5184774"}, "outputs": [], "source": ["df_test.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "bac113c2-99ad-49de-a765-278b6d029970", "_uuid": "684cb19c52b5fc64792fea9ba2d94358441891e0"}, "outputs": [], "source": ["df_test.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "bc45e968-a5cc-4cb6-ac9d-517db4385e79", "_uuid": "370de85d3bb784904891f9f8ba2122e55477add6"}, "outputs": [], "source": ["items.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "69ddef10-ab60-479b-87a7-7dbd790308cb", "_uuid": "9ec38b13a2b9fd2b78219dafc84db46e51788a4a"}, "outputs": [], "source": ["items.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"collapsed": true, "_cell_guid": "e9062e1a-3474-44a6-b112-6ff5ce3de213", "_uuid": "845ac1a19c89d62ef0f91c3e58449eaab4114411"}, "outputs": [], "source": ["df_2017 = df_train[df_train.date.isin(\n", "    pd.date_range(\"2017-05-31\", periods=7 * 11))].copy()\n", "del df_train"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "c6e3580e-75d2-4fa8-a00c-c573c9899a98", "_uuid": "e6bb3e215dbaf1a3cbcd620d2cef3e282e6afcea"}, "outputs": [], "source": ["df_2017.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "b19331a8-dc19-4a96-9de5-2cae305589cc", "_uuid": "4029ad191a587065a845a1ad5ae2c9c7422b1977"}, "outputs": [], "source": ["df_2017.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"collapsed": true, "_cell_guid": "7415cbb4-3414-409f-a8e2-88440070609a", "_uuid": "c89f9f39f8d38ceabf0c59d86309ee5727fd12c0"}, "outputs": [], "source": ["promo_2017_train = df_2017.set_index(\n", "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack(\n", "        level=-1).fillna(False)\n", "promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)\n", "promo_2017_test = df_test[[\"onpromotion\"]].unstack(level=-1).fillna(False)\n", "promo_2017_test.columns = promo_2017_test.columns.get_level_values(1)\n", "promo_2017_test = promo_2017_test.reindex(promo_2017_train.index).fillna(False)\n", "promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)\n", "del promo_2017_test, promo_2017_train"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "118c51da-6d7f-4602-8827-2747f530f111", "_uuid": "797be50414a87d47ddb9e26bee8d18c0e2625051"}, "outputs": [], "source": ["promo_2017.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "31634596-a259-4a7f-9a30-40d8b6668c55", "_uuid": "7acaae20b6aed587b777e3cd2871446c64bc6c7d"}, "outputs": [], "source": ["promo_2017.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "a209bbe3-baf6-4054-bd4f-64a9948dc313", "_uuid": "0ba9afbfb285eca6d9b053af039a562e779e541c"}, "outputs": [], "source": ["promo_2017.columns"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "9045210a-d845-464d-b5a9-1747723a80f2", "_uuid": "ef66234ecab16ca5ff898339947cb99f0ad837d4"}, "source": ["<code>promo_2017</code> is just a big data frame to tell which items were on promotion in which stores on which dates.  Rows represent store-item combinations.  Columns represent dates."], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "7e925530-deda-421e-a36b-3d37b6983e02", "_uuid": "b83c0008eb123ce63301f07a101671218f76f034"}, "outputs": [], "source": ["df_2017 = df_2017.set_index(\n", "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"unit_sales\"]].unstack(\n", "        level=-1).fillna(0)\n", "df_2017.columns = df_2017.columns.get_level_values(1)\n", "df_2017.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "1e2227e5-82be-40f6-a82d-eef3d28b4b7a", "_uuid": "0a8f210599ac1810d89a064cb167bdd191abadce"}, "outputs": [], "source": ["df_2017.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "ba14d6f6-4745-49ad-b709-f368429356d8", "_uuid": "48f50cda6cf64e87853f5eea9b53fbe75621f529"}, "source": ["So the rows in <code>df_2017</code> now correspond to store-item combinations, the columns represent dates (isomorphic to <code>promo_2017</code>), and the data cells contain the target variable."], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "a8f9dcea-dbc2-4779-979a-2b013641d6aa", "_uuid": "64404288687a286daf39d56dcd0f7cdd66b4b81f"}, "outputs": [], "source": ["items = items.reindex(df_2017.index.get_level_values(1))\n", "items.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "8e0aed2d-3d6b-412e-a10e-4dfc64288ccc", "_uuid": "b5ddc9e60fc985773cf07a73358ef64bed7c7c7d"}, "outputs": [], "source": ["items.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "0b6252f8-fec9-41c7-9255-840402fb9472", "_uuid": "7bb16e36b844a4efe02c6627b64197fc7b145924"}, "source": ["<code>items</code> now lines up with <code>df_2017</code>, which means everything is repeated for each store."], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_cell_guid": "c6c4c7a2-f651-4366-8f52-834662acbf06", "_uuid": "a35aee6cbf59ed66e292005f7bab6b276fcb9380"}, "outputs": [], "source": ["# Return that portion of the data frame that corresponds to the time period\n", "#   beginning \"minus\" days before \"dt\" and extending for \"periods\" days\n", "def get_timespan(df, dt, minus, periods):\n", "    return df[\n", "        pd.date_range(dt - timedelta(days=minus), periods=periods)\n", "    ]"], "execution_count": null, "cell_type": "code"}, {"metadata": {"collapsed": true, "_cell_guid": "03404e89-4ea1-43b0-8921-764acca6f43f", "_uuid": "90e7d24b08aaef02318a96618eaeaba500502265"}, "outputs": [], "source": ["def prepare_dataset(t2017, is_train=True):\n", "    X = pd.DataFrame({  # Mean target for different retrospective timespans & total # promotions\n", "        \"mean_3_2017\": get_timespan(df_2017, t2017, 3, 3).mean(axis=1).values,\n", "        \"mean_7_2017\": get_timespan(df_2017, t2017, 7, 7).mean(axis=1).values,\n", "        \"mean_14_2017\": get_timespan(df_2017, t2017, 14, 14).mean(axis=1).values,\n", "        \"promo_14_2017\": get_timespan(promo_2017, t2017, 14, 14).sum(axis=1).values\n", "    })\n", "    for i in range(16):  # Promotions on future days\n", "        X[\"promo_{}\".format(i)] = promo_2017[\n", "            t2017 + timedelta(days=i)].values.astype(np.uint8)\n", "    if is_train:\n", "        y = df_2017[  # Target values for future days\n", "            pd.date_range(t2017, periods=16)\n", "        ].values\n", "        return X, y\n", "    return X"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "979b14fe-7939-4bd4-b78e-cfbf970d5a96", "_uuid": "ecf20bc6f9fbd9468796dfeffca4d735af42295f"}, "outputs": [], "source": ["print(\"Preparing dataset...\")\n", "t2017 = date(2017, 6, 21)\n", "X_l, y_l = [], []\n", "for i in range(4):\n", "    delta = timedelta(days=7 * i)\n", "    X_tmp, y_tmp = prepare_dataset(\n", "        t2017 + delta\n", "    )\n", "    X_l.append(X_tmp)\n", "    y_l.append(y_tmp)\n", "X_train = pd.concat(X_l, axis=0)\n", "y_train = np.concatenate(y_l, axis=0)\n", "del X_l, y_l\n", "X_val, y_val = prepare_dataset(date(2017, 7, 26))\n", "X_test = prepare_dataset(date(2017, 8, 16), is_train=False)"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "66dc9fb9-c099-4773-9494-ae297abf96a9", "_uuid": "084296935967437ff10d8e2b79a163785d769651"}, "outputs": [], "source": ["X_train.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "de00dbc0-998c-443c-8072-f7842a5452f6", "_uuid": "3bf504e21817b885414bf6fb22de846b797a6d19"}, "outputs": [], "source": ["X_train.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "6d2003a0-25ef-44e1-adeb-df8effa2679b", "_uuid": "b44e7b2e99c4fa72d7a623004149568e63db7eed"}, "outputs": [], "source": ["y_train.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "7b07b18c-4be3-4d85-868d-c5d82d857c53", "_uuid": "e0be4207f2311e56896bd61b85f52bf00ddc56e9"}, "outputs": [], "source": ["y_train"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "ed18d6f2-fecd-4b81-96d5-9205e4db4c08", "_uuid": "61d49d719ea3354fcd2331cad04abe2a2c9e9313"}, "outputs": [], "source": ["X_val.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "b9a1803f-06e5-4e81-a66e-f749cc55945b", "_uuid": "909e1d9068f19b2039c5b8145a6c699f35795e11"}, "outputs": [], "source": ["X_val.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "b8da5c67-3bb0-42df-816c-fc255f092476", "_uuid": "ed42c12ab06cb1e3a2adbd3b8327619a5d2885b0"}, "outputs": [], "source": ["y_val.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "b9f27400-78f0-4e8f-955f-3fd85f69abd4", "_uuid": "cd034c5f19b155874f27a8395c728d5f5ba06e1d"}, "outputs": [], "source": ["y_val"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "fe4a131c-3b5b-4809-9cc2-65e5a35fc6f3", "_uuid": "a7c55e53451866fd97f654daf12a8da90e523289"}, "outputs": [], "source": ["X_test.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "90f48e57-69bb-4c91-b3e7-5ef2cf14fe8c", "_uuid": "e97f034d64eaf9447b525dd44abb07d8fc0ec183"}, "outputs": [], "source": ["X_test.head()"], "execution_count": null, "cell_type": "code"}, {"metadata": {"collapsed": true, "_cell_guid": "4083f2d0-3f33-4090-8c6d-20d3dd179388", "_uuid": "716ef2568ad0b368c1fde3379307c560ed05c8ab"}, "outputs": [], "source": ["xgb_params = {\n", "    'eta': 0.05,\n", "    'max_depth': 6,\n", "    'subsample': 0.80,\n", "    'objective': 'reg:linear',\n", "    'eval_metric': 'rmse',\n", "    'silent': 1,\n", "    'n_jobs': 5\n", "}"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "09dbd81e-e73b-42f7-80b1-f8cd6d5d2722", "_uuid": "68ab624f9cde3ccc55a2304269dc1e722ebbd26c"}, "source": ["## THE BLOCK BELOW CUTS THIS DOWN TO THE FIRST 3 DAYS\n", "## CHANGE IT IF YOU WANT TO RUN THE WHOLE THING"], "cell_type": "markdown"}, {"metadata": {"_cell_guid": "46d13f8c-2342-4d59-a448-73b317809c9c", "_uuid": "e87eb3fcd741632ad7498fc750a8a7bf147ce72b"}, "outputs": [], "source": ["## THE BLOCK BELOW CUTS THIS DOWN TO THE FIRST 3 DAYS\n", "n_preds = 16\n", "n_preds = 3  # COMMENT OUT THIS LINE TO RUN FOR ALL DAYS\n", "y_val = y_val[:, :n_preds]\n", "y_val.shape"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "6021f28d-d244-4810-bf77-eebf02a04af8", "_uuid": "7dd76fbdd5f4bf267ce96492053eb69da535cf1f"}, "outputs": [], "source": ["MAX_ROUNDS = 1000\n", "val_pred = []\n", "test_pred = []\n", "cate_vars = []\n", "for i in range(n_preds):\n", "    print(\"=\" * 50)\n", "    print(\"Step %d\" % (i+1))\n", "    print(\"=\" * 50)\n", "    y_train_ = y_train[:, i]\n", "    y_val_ = y_val[:, i]\n", "    dtrain = xgb.DMatrix( X_train,  y_train_,\n", "                          weight=pd.concat([items[\"perishable\"]] * 4) * 0.25 + 1  )\n", "    dval = xgb.DMatrix( X_val,  y_val_, \n", "                        weight=items[\"perishable\"] * 0.25 + 1  )\n", "    dtest = xgb.DMatrix( X_test )\n", "    evals = [(dtrain,'train'),(dval,'eval')]\n", "    bst = xgb.train(\n", "        xgb_params, dtrain, num_boost_round=MAX_ROUNDS,\n", "        evals=evals, early_stopping_rounds=50, verbose_eval=50\n", "    )\n", "\n", "    val_pred.append(bst.predict(dval, ntree_limit=bst.best_ntree_limit))\n", "    test_pred.append(bst.predict(dtest, ntree_limit=bst.best_ntree_limit))"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "fd9df814-c21e-4469-832d-4def0cbb6733", "_uuid": "e96db5cb36a756288f247d21f5e963cf240d70dc"}, "outputs": [], "source": ["weights=pd.concat([items[\"perishable\"]]) * 0.25 + 1\n", "print(\"Validation mse:\", mean_squared_error(\n", "    y_val, np.array(val_pred).transpose(), sample_weight=weights))"], "execution_count": null, "cell_type": "code"}, {"metadata": {"_cell_guid": "eb108660-1e03-4513-903c-f079a819e80b", "_uuid": "63c9803349bcf25c9524beaad3396eef8342894a"}, "source": ["## UNCOMMENT THE BLOCKS BELOW TO CREATE A SUBMISSION FILE"], "cell_type": "markdown"}, {"metadata": {"collapsed": true, "_cell_guid": "c57fd297-ea20-4c79-91ac-7d1866742fc6", "_uuid": "641065b01c87a7af08f34da824be574d342904a3"}, "outputs": [], "source": ["#print(\"Making submission...\")\n", "#y_test = np.array(test_pred).transpose()\n", "#df_preds = pd.DataFrame(\n", "#    y_test, index=df_2017.index,\n", "#    columns=pd.date_range(\"2017-08-16\", periods=16)\n", "#).stack().to_frame(\"unit_sales\")\n", "#df_preds.index.set_names([\"store_nbr\", \"item_nbr\", \"date\"], inplace=True)\n", "\n", "#submission = df_test[[\"id\"]].join(df_preds, how=\"left\").fillna(0)\n", "#submission[\"unit_sales\"] = np.clip(np.expm1(submission[\"unit_sales\"]), 0, 1000)"], "execution_count": null, "cell_type": "code"}, {"metadata": {"collapsed": true, "_cell_guid": "7bbb916c-23ee-47e9-89b2-9cfb22628e6b", "_uuid": "0a927af9d32f08e52173493e0a0684eee87267a4"}, "outputs": [], "source": ["#submission.to_csv('lgb.csv', float_format='%.4f', index=None)"], "execution_count": null, "cell_type": "code"}, {"metadata": {"collapsed": true, "_cell_guid": "c3e2e707-b9d2-4b11-99b0-07b6e92fe072", "_uuid": "8260176b1a442bc0759766b09efb6d314c1318c9"}, "outputs": [], "source": [], "execution_count": null, "cell_type": "code"}], "nbformat_minor": 1}