{"nbformat": 4, "nbformat_minor": 1, "metadata": {"kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}, "language_info": {"nbconvert_exporter": "python", "file_extension": ".py", "pygments_lexer": "ipython3", "name": "python", "mimetype": "text/x-python", "version": "3.6.3", "codemirror_mode": {"name": "ipython", "version": 3}}}, "cells": [{"source": ["Original is [here](https://www.kaggle.com/ceshine/lgbm-starter)\n", "\n", "This is a sloppy revision of my earlier notebook (which was substantively identical to Ceshine's script) to include validation data in the training set.  That's normally what you would do before making a final submission (at least when you use holdout validation, as in this case).  Obviously it would be better to improve the model first, since it was billed as a \"starter\" that was \"watered down.\"  I was just curious to see how it would do.  Turns out it does perform a ittle bit better (same 3-decimal-place public score, but it sorts higher).  It also happens to reduce the score of my ensemble, which either means that it's producing more redundant information or that my ensemble weights are overfit, most likely the latter."], "metadata": {"_cell_guid": "de224c83-fa80-49cf-b66b-f2b6e2e8e67e", "_uuid": "86bfbcdbb5f6052cae55d8a2d21354c3bef738d2"}, "cell_type": "markdown"}, {"source": ["from datetime import date, timedelta\n", "\n", "import pandas as pd\n", "import numpy as np\n", "from sklearn.metrics import mean_squared_error\n", "import lightgbm as lgb"], "execution_count": null, "metadata": {"_cell_guid": "dc07f651-2a29-4cc5-80d4-fca67d0a7b66", "_uuid": "bb2e3efe23cfe1e68c0cd622be3de2a8e81a9a7a", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_train = pd.read_csv(\n", "    '../input/train.csv', usecols=[1, 2, 3, 4, 5],\n", "    dtype={'onpromotion': bool},\n", "    converters={'unit_sales': lambda u: np.log1p(\n", "        float(u)) if float(u) > 0 else 0},\n", "    parse_dates=[\"date\"],\n", "    skiprows=range(1, 66458909)  # 2016-01-01\n", ")\n", "\n", "df_test = pd.read_csv(\n", "    \"../input/test.csv\", usecols=[0, 1, 2, 3, 4],\n", "    dtype={'onpromotion': bool},\n", "    parse_dates=[\"date\"]  # , date_parser=parser\n", ").set_index(\n", "    ['store_nbr', 'item_nbr', 'date']\n", ")\n", "\n", "items = pd.read_csv(\n", "    \"../input/items.csv\",\n", ").set_index(\"item_nbr\")"], "execution_count": null, "metadata": {"_cell_guid": "bd1da8c1-0a03-495e-a448-6957e6fdc60c", "_uuid": "116f514148006d967b091124f131f2a341a47a99", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_train.shape"], "execution_count": null, "metadata": {"_cell_guid": "372f8dd0-cd63-4461-a33b-88ed5d2d636e", "_uuid": "0a94f0d6750d42dfd2cedf2728916e50de5f0f36", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_train.head()"], "execution_count": null, "metadata": {"_cell_guid": "fbb6cbf5-064f-4d33-8272-706a06a6317e", "_uuid": "a1fcda19ec288d3073f2e2a589daad78359ccd9c", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_test.shape"], "execution_count": null, "metadata": {"_cell_guid": "8c481ac6-4224-4dec-9d5a-0bfb0ecc001a", "_uuid": "9abd08f8b1a6f672a18d689a5de9b53dc5184774", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_test.head()"], "execution_count": null, "metadata": {"_cell_guid": "bac113c2-99ad-49de-a765-278b6d029970", "_uuid": "684cb19c52b5fc64792fea9ba2d94358441891e0", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["items.shape"], "execution_count": null, "metadata": {"_cell_guid": "bc45e968-a5cc-4cb6-ac9d-517db4385e79", "_uuid": "370de85d3bb784904891f9f8ba2122e55477add6", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["items.head()"], "execution_count": null, "metadata": {"_cell_guid": "69ddef10-ab60-479b-87a7-7dbd790308cb", "_uuid": "9ec38b13a2b9fd2b78219dafc84db46e51788a4a", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_2017 = df_train[df_train.date.isin(\n", "    pd.date_range(\"2017-05-31\", periods=7 * 11))].copy()\n", "del df_train"], "execution_count": null, "metadata": {"_cell_guid": "e9062e1a-3474-44a6-b112-6ff5ce3de213", "_uuid": "845ac1a19c89d62ef0f91c3e58449eaab4114411", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_2017.shape"], "execution_count": null, "metadata": {"_cell_guid": "c6e3580e-75d2-4fa8-a00c-c573c9899a98", "_uuid": "e6bb3e215dbaf1a3cbcd620d2cef3e282e6afcea", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_2017.head()"], "execution_count": null, "metadata": {"_cell_guid": "b19331a8-dc19-4a96-9de5-2cae305589cc", "_uuid": "4029ad191a587065a845a1ad5ae2c9c7422b1977", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["promo_2017_train = df_2017.set_index(\n", "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack(\n", "        level=-1).fillna(False)\n", "promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)\n", "promo_2017_test = df_test[[\"onpromotion\"]].unstack(level=-1).fillna(False)\n", "promo_2017_test.columns = promo_2017_test.columns.get_level_values(1)\n", "promo_2017_test = promo_2017_test.reindex(promo_2017_train.index).fillna(False)\n", "promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)\n", "del promo_2017_test, promo_2017_train"], "execution_count": null, "metadata": {"_cell_guid": "7415cbb4-3414-409f-a8e2-88440070609a", "_uuid": "c89f9f39f8d38ceabf0c59d86309ee5727fd12c0", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["promo_2017.shape"], "execution_count": null, "metadata": {"_cell_guid": "118c51da-6d7f-4602-8827-2747f530f111", "_uuid": "797be50414a87d47ddb9e26bee8d18c0e2625051", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["promo_2017.head()"], "execution_count": null, "metadata": {"_cell_guid": "31634596-a259-4a7f-9a30-40d8b6668c55", "_uuid": "7acaae20b6aed587b777e3cd2871446c64bc6c7d", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["promo_2017.columns"], "execution_count": null, "metadata": {"_cell_guid": "a209bbe3-baf6-4054-bd4f-64a9948dc313", "_uuid": "0ba9afbfb285eca6d9b053af039a562e779e541c", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["<code>promo_2017</code> is just a big data frame to tell which items were on promotion in which stores on which dates.  Rows represent store-item combinations.  Columns represent dates."], "metadata": {"_cell_guid": "9045210a-d845-464d-b5a9-1747723a80f2", "_uuid": "ef66234ecab16ca5ff898339947cb99f0ad837d4"}, "cell_type": "markdown"}, {"source": ["df_2017 = df_2017.set_index(\n", "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"unit_sales\"]].unstack(\n", "        level=-1).fillna(0)\n", "df_2017.columns = df_2017.columns.get_level_values(1)\n", "df_2017.shape"], "execution_count": null, "metadata": {"_cell_guid": "7e925530-deda-421e-a36b-3d37b6983e02", "_uuid": "b83c0008eb123ce63301f07a101671218f76f034", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["df_2017.head()"], "execution_count": null, "metadata": {"_cell_guid": "1e2227e5-82be-40f6-a82d-eef3d28b4b7a", "_uuid": "0a8f210599ac1810d89a064cb167bdd191abadce", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["So the rows in <code>df_2017</code> now correspond to store-item combinations, the columns represent dates (isomorphic to <code>promo_2017</code>), and the data cells contain the target variable."], "metadata": {"_cell_guid": "ba14d6f6-4745-49ad-b709-f368429356d8", "_uuid": "48f50cda6cf64e87853f5eea9b53fbe75621f529"}, "cell_type": "markdown"}, {"source": ["items = items.reindex(df_2017.index.get_level_values(1))\n", "items.head()"], "execution_count": null, "metadata": {"_cell_guid": "a8f9dcea-dbc2-4779-979a-2b013641d6aa", "_uuid": "64404288687a286daf39d56dcd0f7cdd66b4b81f", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["items.shape"], "execution_count": null, "metadata": {"_cell_guid": "8e0aed2d-3d6b-412e-a10e-4dfc64288ccc", "_uuid": "b5ddc9e60fc985773cf07a73358ef64bed7c7c7d", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["<code>items</code> now lines up with <code>df_2017</code>, which means everything is repeated for each store."], "metadata": {"_cell_guid": "0b6252f8-fec9-41c7-9255-840402fb9472", "_uuid": "7bb16e36b844a4efe02c6627b64197fc7b145924"}, "cell_type": "markdown"}, {"source": ["# Return that portion of the data frame that corresponds to the time period\n", "#   beginning \"minus\" days before \"dt\" and extending for \"periods\" days\n", "def get_timespan(df, dt, minus, periods):\n", "    return df[\n", "        pd.date_range(dt - timedelta(days=minus), periods=periods)\n", "    ]"], "execution_count": null, "metadata": {"_cell_guid": "c6c4c7a2-f651-4366-8f52-834662acbf06", "_uuid": "a35aee6cbf59ed66e292005f7bab6b276fcb9380", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["def prepare_dataset(t2017, is_train=True):\n", "    X = pd.DataFrame({  # Mean target for different retrospective timespans & total # promotions\n", "        \"mean_3_2017\": get_timespan(df_2017, t2017, 3, 3).mean(axis=1).values,\n", "        \"mean_7_2017\": get_timespan(df_2017, t2017, 7, 7).mean(axis=1).values,\n", "        \"mean_14_2017\": get_timespan(df_2017, t2017, 14, 14).mean(axis=1).values,\n", "        \"promo_14_2017\": get_timespan(promo_2017, t2017, 14, 14).sum(axis=1).values\n", "    })\n", "    for i in range(16):  # Promotions on future days\n", "        X[\"promo_{}\".format(i)] = promo_2017[\n", "            t2017 + timedelta(days=i)].values.astype(np.uint8)\n", "    if is_train:\n", "        y = df_2017[  # Target values for future days\n", "            pd.date_range(t2017, periods=16)\n", "        ].values\n", "        return X, y\n", "    return X"], "execution_count": null, "metadata": {"_cell_guid": "03404e89-4ea1-43b0-8921-764acca6f43f", "_uuid": "90e7d24b08aaef02318a96618eaeaba500502265", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["print(\"Preparing dataset...\")\n", "t2017 = date(2017, 6, 21)\n", "X_l, y_l = [], []\n", "for i in range(6):\n", "    delta = timedelta(days=7 * i)\n", "    X_tmp, y_tmp = prepare_dataset(\n", "        t2017 + delta\n", "    )\n", "    X_l.append(X_tmp)\n", "    y_l.append(y_tmp)\n", "X_train = pd.concat(X_l, axis=0)\n", "y_train = np.concatenate(y_l, axis=0)\n", "del X_l, y_l\n", "# X_val, y_val = prepare_dataset(date(2017, 7, 26))\n", "X_test = prepare_dataset(date(2017, 8, 16), is_train=False)"], "execution_count": null, "metadata": {"_cell_guid": "979b14fe-7939-4bd4-b78e-cfbf970d5a96", "_uuid": "ecf20bc6f9fbd9468796dfeffca4d735af42295f", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["X_train.shape"], "execution_count": null, "metadata": {"_cell_guid": "a0e674f9-3446-40ac-a4a7-b900bcfff2b6", "_uuid": "0f99e7d94bd56e5728158f7e0f8157ace297d0eb", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["X_train.head()"], "execution_count": null, "metadata": {"_cell_guid": "8ff1b57c-18c8-4c25-9f7d-b817f02b528f", "_uuid": "f5a728076c9f676cab7ce223c411e4cf63782db5", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["y_train.shape"], "execution_count": null, "metadata": {"_cell_guid": "198b0cad-0366-495f-9e5a-9c905506d85e", "_uuid": "c8bf9e1b75a615c18706934e73cd9411bfc05480", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["y_train"], "execution_count": null, "metadata": {"_cell_guid": "caf003cd-6883-452e-acb5-40101198f60c", "_uuid": "a7ecd99574873d8c69b08a6064978509a13bae54", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["#X_val.shape"], "execution_count": null, "metadata": {"_cell_guid": "8d53e014-de37-42cc-a316-80d30d3f221f", "_uuid": "7794e126a4982631b1bff64a75d7c895bc0c7900", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["#X_val.head()"], "execution_count": null, "metadata": {"_cell_guid": "52d52f92-4fe8-430b-a761-df9456f73195", "_uuid": "7e41809e28fe0afdb8eaabb3321ad208621e0e93", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["#y_val.shape"], "execution_count": null, "metadata": {"_cell_guid": "419da11f-6837-443c-a698-be16d1d83a67", "_uuid": "046369b7726eab26a87714c0723a55cd50ea4c9e", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["#y_val"], "execution_count": null, "metadata": {"_cell_guid": "1cbd7056-2644-498b-818b-5c9f4e8eddcc", "_uuid": "e1f3c9008e51df37a7af7eeae82414b339a44327", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["X_test.shape"], "execution_count": null, "metadata": {"_cell_guid": "7e98bf21-f1ba-4cf2-b37c-430e788d5786", "_uuid": "d478aa3f1afcd364f2d142bb40a73a5c50261c33", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["X_test.head()"], "execution_count": null, "metadata": {"_cell_guid": "8e5e484f-5285-41b4-ab72-081dec06a790", "_uuid": "84eaa0314c1b810c38d01fdc682e2e23fb9321d8", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["print(\"Training and predicting models...\")\n", "params = {\n", "    'num_leaves': 2**5 - 1,\n", "    'objective': 'regression_l2',\n", "    'max_depth': 8,\n", "    'min_data_in_leaf': 50,\n", "    'learning_rate': 0.05,\n", "    'feature_fraction': 0.75,\n", "    'bagging_fraction': 0.75,\n", "    'bagging_freq': 1,\n", "    'metric': 'l2',\n", "    'num_threads': 4\n", "}"], "execution_count": null, "metadata": {"_cell_guid": "193811d6-8182-469f-b05d-2464cb4d44db", "_uuid": "3523c5904c9e8e1ef53068e71a0e759c19a0fbcb", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["MAX_ROUNDS = 1000\n", "val_pred = []\n", "test_pred = []\n", "cate_vars = []\n", "for i in range(16):\n", "    print(\"=\" * 50)\n", "    print(\"Step %d\" % (i+1))\n", "    print(\"=\" * 50)\n", "    dtrain = lgb.Dataset(\n", "        X_train, label=y_train[:, i],\n", "        categorical_feature=cate_vars,\n", "        weight=pd.concat([items[\"perishable\"]] * 6) * 0.25 + 1\n", "    )\n", "#    dval = lgb.Dataset(\n", "#        X_val, label=y_val[:, i], reference=dtrain,\n", "#        weight=items[\"perishable\"] * 0.25 + 1,\n", "#        categorical_feature=cate_vars)\n", "    bst = lgb.train(\n", "        params, dtrain, num_boost_round=700,\n", "        valid_sets=[dtrain], verbose_eval=50\n", "    )\n", "    print(\"\\n\".join((\"%s: %.2f\" % x) for x in sorted(\n", "        zip(X_train.columns, bst.feature_importance(\"gain\")),\n", "        key=lambda x: x[1], reverse=True\n", "    )))\n", "#    val_pred.append(bst.predict(\n", "#        X_val, num_iteration=bst.best_iteration or MAX_ROUNDS))\n", "    test_pred.append(bst.predict(\n", "        X_test, num_iteration=700))"], "execution_count": null, "metadata": {"_cell_guid": "6021f28d-d244-4810-bf77-eebf02a04af8", "_uuid": "7dd76fbdd5f4bf267ce96492053eb69da535cf1f", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["#print(\"Validation mse:\", mean_squared_error(\n", "#    y_val, np.array(val_pred).transpose()))"], "execution_count": null, "metadata": {"_cell_guid": "fd9df814-c21e-4469-832d-4def0cbb6733", "_uuid": "e96db5cb36a756288f247d21f5e963cf240d70dc", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["print(\"Making submission...\")\n", "y_test = np.array(test_pred).transpose()\n", "df_preds = pd.DataFrame(\n", "    y_test, index=df_2017.index,\n", "    columns=pd.date_range(\"2017-08-16\", periods=16)\n", ").stack().to_frame(\"unit_sales\")\n", "df_preds.index.set_names([\"store_nbr\", \"item_nbr\", \"date\"], inplace=True)\n", "\n", "submission = df_test[[\"id\"]].join(df_preds, how=\"left\").fillna(0)\n", "submission[\"unit_sales\"] = np.clip(np.expm1(submission[\"unit_sales\"]), 0, 1000)"], "execution_count": null, "metadata": {"_cell_guid": "c57fd297-ea20-4c79-91ac-7d1866742fc6", "_uuid": "641065b01c87a7af08f34da824be574d342904a3", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": ["submission.to_csv('lgb6w.csv', float_format='%.4f', index=None)"], "execution_count": null, "metadata": {"_cell_guid": "7bbb916c-23ee-47e9-89b2-9cfb22628e6b", "_uuid": "0a927af9d32f08e52173493e0a0684eee87267a4", "collapsed": true}, "outputs": [], "cell_type": "code"}, {"source": [], "execution_count": null, "metadata": {"_cell_guid": "c3e2e707-b9d2-4b11-99b0-07b6e92fe072", "_uuid": "8260176b1a442bc0759766b09efb6d314c1318c9", "collapsed": true}, "outputs": [], "cell_type": "code"}]}