{"metadata": {"kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}, "language_info": {"nbconvert_exporter": "python", "version": "3.6.4", "pygments_lexer": "ipython3", "mimetype": "text/x-python", "name": "python", "file_extension": ".py", "codemirror_mode": {"name": "ipython", "version": 3}}}, "nbformat_minor": 1, "nbformat": 4, "cells": [{"metadata": {"collapsed": true, "_cell_guid": "b5bc3dd2-d432-41ad-8731-56b1c21a4691", "_uuid": "564f3bd6ed5387101df4cabadc867ca4b9e2c513"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["VAL_DATE = '2016-08-01'\n", "START_DATE = '2014-01-01'\n", "MAX_ROUNDS = 2000"]}, {"metadata": {"_cell_guid": "8109a560-e57d-4828-8f2c-f0c0b0e59099", "_uuid": "57d8b20dd99705bae1acd7423ac69f93bf93932f"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["from datetime import date, timedelta\n", "import pandas as pd\n", "import numpy as np\n", "from sklearn.metrics import mean_squared_error\n", "import lightgbm as lgb\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))"]}, {"metadata": {"collapsed": true, "_cell_guid": "8a31e6e6-4cbe-46a3-af9c-c3520ec7aeaa", "_uuid": "9aaee071ed84db12364828ae9c0665a2ff6309bb"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["indir = '../input/second-week-sales/'\n", "indir2 = '../input/favorita-grocery-sales-forecasting/'"]}, {"metadata": {"_cell_guid": "85f89b9c-c64d-4ae8-8983-2924dcd83bec", "_uuid": "0db4c7fc524868a1e5c2c6794611323f75ad4462"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["print(check_output([\"ls\", indir[:-1]]).decode(\"utf8\"))\n", "print(check_output([\"ls\", indir2[:-1]]).decode(\"utf8\"))"]}, {"metadata": {"_cell_guid": "64e2138c-2c44-4f4f-8eae-12b2c1e57b03", "_uuid": "def11e07bea5f275b5cf8b4b72aea8f974473c24"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["train_in = pd.read_csv(\n", "    indir+'train_2nd_week.csv',\n", "    dtype={'onpromotion': bool},\n", "    converters={'unit_sales': lambda u: np.log1p(\n", "        float(u)) if float(u) > 0 else 0},\n", "    parse_dates=[\"date\"],\n", ")\n", "print(train_in.shape)\n", "train_in.head()"]}, {"metadata": {"_cell_guid": "9ace61be-d9f4-43c4-af70-3ec3d90af918", "_uuid": "031d0010a9e85650f456d7ad0ab1210b00f6d2ee"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["test_in = pd.read_csv(\n", "    indir2+\"test.csv\",\n", "    dtype={'onpromotion': bool},\n", "    parse_dates=[\"date\"]  # , date_parser=parser\n", ")\n", "print(test_in.shape)\n", "test_in.head()"]}, {"metadata": {"_cell_guid": "4666502c-3e5f-467d-80f0-17f9202ca1ff", "_uuid": "6ff67836a2596113f80d2b7b9e3e07240fcf189b"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["new_items = pd.read_csv(indir+'new_items.csv')\n", "print(new_items.shape)\n", "nnew = new_items.shape[0]\n", "new_items.head()"]}, {"metadata": {"_cell_guid": "5b77abb6-90ae-4113-84fb-01c4f4d81b26", "_uuid": "fbe6d1d2ebacc8f798d1e42d04f2eecc4e7758d6"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["test_new = test_in.merge(new_items, on='item_nbr', how='inner')\n", "print(test_new.shape)\n", "test_new.head()"]}, {"metadata": {"_cell_guid": "39be5ba1-1c8d-4402-a74f-30b12473d87f", "_uuid": "765a3e63a6cb3df1c39b74dacbac1ea970418e8c"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["all_2nd_week_items = pd.read_csv( indir+'items_2nd_week.csv', parse_dates=['date'] )\n", "all_2nd_week_items.head()"]}, {"metadata": {"_cell_guid": "657fc292-56cb-415d-a3ae-d390a73f047f", "_uuid": "e19b22ac6d05530499b83c952b2e5d47d9999470"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["stores = pd.read_csv(indir2+\"stores.csv\")\n", "stores.head()"]}, {"metadata": {"_cell_guid": "9eae6ad7-9a31-4120-a9fc-3cc7c4dc0636", "_uuid": "a2b99d5421e6ddb0b670a856038e1d6c9354dec3"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["store_nbrs = stores[['store_nbr']].copy()\n", "store_nbrs['dummy'] = 1\n", "all_2nd_week_items['dummy'] = 1\n", "all_combos = all_2nd_week_items.merge(store_nbrs, on='dummy', how='outer').drop('dummy',axis=1)\n", "print( all_2nd_week_items.shape, store_nbrs.shape, all_combos.shape )\n", "all_combos.head()"]}, {"metadata": {"_cell_guid": "06e93d3a-ae51-474c-b55d-d40372a02a1a", "_uuid": "2bd26e720fd517575b88b7ad1e954184a7f6fef5"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["print(train_in.shape)\n", "train_in.head()"]}, {"metadata": {"_cell_guid": "a397f6d4-906a-4a82-acc1-50f68cf1c450", "_uuid": "4a29a91b3de35cf1000a1e99934c5e71ad1caa5c"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["train = all_combos.merge(train_in, on=['date','item_nbr','store_nbr'], how='outer')\n", "train['unit_sales'] = train.unit_sales.fillna(0)\n", "print(train.shape)\n", "train.head()"]}, {"metadata": {"_cell_guid": "e0a5016e-cb64-42a9-b496-83fddc0e2f96", "_uuid": "8c8a2c798d27ee2b84dd1df42493ac247a3d7e10"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["items = pd.read_csv(indir2+\"items.csv\")\n", "items.head()"]}, {"metadata": {"_cell_guid": "48c33ef2-b739-4bfc-a1cb-5217c518a8be", "_uuid": "eccceb01fcb0e0cd382d68cb844fdef0556b2153"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["df = train.merge(items, on='item_nbr')\n", "test = test_new.merge(items, on='item_nbr')\n", "df.head()"]}, {"metadata": {"_cell_guid": "0b204d72-1f97-4909-82a3-6fd370cd2be4", "_uuid": "77968e9ce1641f6e0855148d1e089d10c77a27de"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["test.head()"]}, {"metadata": {"_cell_guid": "283fc314-780f-4282-832e-476926e0248b", "_uuid": "590f2f79c38ffffd7fb63b5e0b259f2b1f709644"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["store_code = df[df.date<VAL_DATE].groupby(\"store_nbr\").unit_sales.sum().rename('store')\n", "df = df.join(store_code, on='store_nbr', how='left')\n", "test = test.join(store_code, on='store_nbr', how='left')\n", "df.head()"]}, {"metadata": {"_cell_guid": "e3d4c5f8-ba05-4fbe-9c69-55bed50841a9", "_uuid": "b7d80a5197e0ab91a4d4822fdcca36a3304b5d55"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["test.head()"]}, {"metadata": {"_cell_guid": "847fb0e3-b272-4246-a295-3f97a324b4d6", "_uuid": "a6d5011f1ecfc56582c6f8aec1205948ecb85e95"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["family_code = df[df.date<VAL_DATE].groupby(\"family\").unit_sales.sum().rename('fam')\n", "df = df.join(family_code, on='family', how='left')\n", "test = test.join(family_code, on='family', how='left')\n", "df.head()"]}, {"metadata": {"_cell_guid": "b6301336-4e47-46d0-8673-a03c5723b571", "_uuid": "e79b4dc6282a73a7c0de762dc11750eb2d62e53d"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["test.head()"]}, {"metadata": {"_cell_guid": "c58edbe2-b823-41f7-9ff1-20e25804bbf9", "_uuid": "5d15174408484f6a9a0da5d8b78c165f28d9d56d"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["class_code = df[df.date<VAL_DATE].groupby(\"class\").unit_sales.sum().rename('cla')\n", "df = df.join(class_code, on='class', how='left')\n", "test = test.join(class_code, on='class', how='left')\n", "df.head()"]}, {"metadata": {"_cell_guid": "47301c01-206d-4c0f-8b6b-56d4ce030991", "_uuid": "c682697593398256ccaa52d48fb1b271591954c5"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["df['dow'] = df['date'].dt.dayofweek\n", "test['dow'] = test['date'].dt.dayofweek\n", "dow_code = df[df.date<VAL_DATE].groupby(\"dow\").unit_sales.sum().rename('day')\n", "df = df.join(dow_code, on='dow', how='left')\n", "test = test.join(dow_code, on='dow', how='left')\n", "df['promo'] = df.onpromotion.astype(float)\n", "test['promo'] = test.onpromotion.astype(float)\n", "df.head()"]}, {"metadata": {"_cell_guid": "d087b0b4-5317-40b4-8d98-eab66de694b9", "_uuid": "e25dba1e869b792dd6313b8bb8865524dbe796a5"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["df_train = df.drop(['date','store_nbr','item_nbr','family','class','dow','onpromotion'],axis=1)\n", "df_test = test.drop(['id','date','store_nbr','item_nbr','family','class','dow','onpromotion'],axis=1)\n", "df_train['store_class'] = df_train['store']*df_train['cla']\n", "df_test['store_class'] = df_test['store']*df_test['cla']\n", "df_train['store_fam'] = df_train['store']*df_train['fam']\n", "df_test['store_fam'] = df_test['store']*df_test['fam']\n", "df_train['store_day'] = df_train['store']*df_train['day']\n", "df_test['store_day'] = df_test['store']*df_test['day']\n", "df_train['store_prom'] = df_train['store']*(df_train['promo'].fillna(0.5))\n", "df_test['store_prom'] = df_test['store']*(df_test['promo'].fillna(0.5))\n", "df_train['store_per'] = df_train['store']*df_train['perishable']\n", "df_test['store_per'] = df_test['store']*df_test['perishable']\n", "df_train['class_day'] = df_train['cla']*df_train['day']\n", "df_test['class_day'] = df_test['cla']*df_test['day']\n", "df_train['class_prom'] = df_train['cla']*(df_train['promo'].fillna(0.5))\n", "df_test['class_prom'] = df_test['cla']*(df_test['promo'].fillna(0.5))\n", "df_train['class_per'] = df_train['cla']*df_train['perishable']\n", "df_test['class_per'] = df_test['cla']*df_test['perishable']\n", "df_train['day_prom'] = df_train['day']*(df_train['promo'].fillna(0.5))\n", "df_test['day_prom'] = df_test['day']*(df_test['promo'].fillna(0.5))\n", "df_train['day_per'] = df_train['day']*df_train['perishable']\n", "df_test['day_per'] = df_test['day']*df_test['perishable']\n", "df_train['per_prom'] = df_train['perishable']*(df_train['promo'].fillna(0.5))\n", "df_test['per_prom'] = df_test['perishable']*(df_test['promo'].fillna(0.5))\n", "df_train['fam_day'] = df_train['fam']*df_train['day']\n", "df_test['fam_day'] = df_test['fam']*df_test['day']\n", "df_train['fam_prom'] = df_train['fam']*(df_train['promo'].fillna(0.5))\n", "df_test['fam_prom'] = df_test['fam']*(df_test['promo'].fillna(0.5))\n", "df_train['fam_per'] = df_train['fam']*df_train['perishable']\n", "df_test['fam_per'] = df_test['fam']*df_test['perishable']\n", "df_train.head()"]}, {"metadata": {"_cell_guid": "e2073fb0-c28c-4116-b21e-d8e27d57318e", "_uuid": "96c1196c9b9376b4bbc4dc2b2b1cc1237afe1f8c"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["df_test.head()"]}, {"metadata": {"collapsed": true, "_cell_guid": "0c3dbb59-a342-42b3-aa76-83ad6e4e3d8f", "_uuid": "235071afaa19963f19d3c9b3be944382086775f5"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["df_val = df_train[df.date>=VAL_DATE]\n", "df_train = df_train[(df.date<VAL_DATE) & (df.date>START_DATE)]"]}, {"metadata": {"_cell_guid": "5bef657e-828b-4e1e-b5d9-ac65892b7b75", "_uuid": "e15f7b701488bf335129ab9568ab39ecaf5936c8"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["X_train = df_train.drop(['unit_sales'],axis=1).fillna(0.5)\n", "y_train = df_train.unit_sales.values\n", "X_val = df_val.drop(['unit_sales'],axis=1).fillna(0.5)\n", "y_val = df_val.unit_sales.values\n", "X_test = df_test.fillna(0.5)\n", "X_train.head()"]}, {"metadata": {"collapsed": true, "_cell_guid": "48c11a0e-e6d1-4c1c-a84d-f28c410a4d29", "_uuid": "0ba922c74b5d7f8f707a7c5a888fa7884e9436bb"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["dtrain = lgb.Dataset( X_train, label=y_train, weight=X_train.perishable * 0.25 + 1 )\n", "dval = lgb.Dataset( X_val, label=y_val, weight=X_val.perishable * 0.25 + 1 )"]}, {"metadata": {"collapsed": true, "_cell_guid": "74e6789c-0e13-4262-b377-50b9d7632b04", "_uuid": "27d02a777d2b01347f8bb79827c079a9413bcfae"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["params = {\n", "    'num_leaves': 100,\n", "    'objective': 'regression',\n", "    'min_data_in_leaf': 100,\n", "    'learning_rate': 0.03,\n", "    'feature_fraction': 0.7,\n", "    'bagging_fraction': 0.9,\n", "    'bagging_freq': 3,\n", "    'metric': 'l2',\n", "    'lambda_l2': .3,\n", "    'lambda_l1': .02,\n", "    'max_depth': 9,\n", "    'min_split_gain': 1e-3,\n", "    'num_threads': 4\n", "}"]}, {"metadata": {"_cell_guid": "24ef444a-8a75-4b44-87d8-b1339b02df3d", "_uuid": "a1e2812638bc406938a2ca6afbad1d6a4f9f9615"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["bst = lgb.train( params, dtrain, num_boost_round=MAX_ROUNDS,\n", "        valid_sets=[dtrain, dval], early_stopping_rounds=80, verbose_eval=50)"]}, {"metadata": {"collapsed": true, "_cell_guid": "e9e0efa3-67ed-47dc-b05d-0ec0b0b47efb", "_uuid": "6daaeb2da1c7df90148a137fa4e6c1529fff6503"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["val_pred = bst.predict(X_val, num_iteration=bst.best_iteration or MAX_ROUNDS)\n", "test_pred = bst.predict(X_test, num_iteration=bst.best_iteration or MAX_ROUNDS)\n", "mse = mean_squared_error( y_val, np.array(val_pred), sample_weight=X_val.perishable * 0.25 + 1 )\n", "np.sqrt(mse)"]}, {"metadata": {"collapsed": true, "_cell_guid": "bfbda22f-8e10-409b-a6d6-e83e4326f3b0", "_uuid": "18c44c67461718383226e2658ffae2d5d976f249"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["pd.Series(y_val).describe()"]}, {"metadata": {"collapsed": true, "_cell_guid": "0d42410a-d189-4172-a94d-be9f6aa6982b", "_uuid": "fcba56506da01588323f5b603bd71a9f4c37b3f8"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["np.expm1(y_val.mean())"]}, {"metadata": {"collapsed": true, "_cell_guid": "4b3bd04c-7d15-45ba-a4c2-4023b6e1c38d", "_uuid": "3ec3321cfc423826e35f77a7a67bcfe750f95fd5"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["pd.Series(val_pred).describe()"]}, {"metadata": {"collapsed": true, "_cell_guid": "560abe10-ce48-4747-93d8-f23a0bc1a490", "_uuid": "16af3dca4877d8ec2096413fd7da1d296ebd1e27"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["pd.Series(y_train).describe()"]}, {"metadata": {"collapsed": true, "_cell_guid": "bdc5648e-358d-4a8e-aa10-2b0a0830277c", "_uuid": "1b4ba2490b3f3d0155d1d50ed704190b93041365"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["pd.Series(test_pred).describe()"]}, {"metadata": {"collapsed": true, "_cell_guid": "4e541af9-2351-44f8-8c44-e19525be15b2", "_uuid": "3301866eefc7c75f236b47120d5572a917a54738"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["np.expm1(test_pred.mean())"]}, {"metadata": {"collapsed": true, "_cell_guid": "6abdafe1-c62d-4cf0-ab80-39a575de10f3", "_uuid": "dcab6205328ab90a8cddb310989764be63d65dc9"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["scaledown = { # to account for gradual introudction of new products and ramp-up in first week\n", "    '2017-08-16': .0001,\n", "    '2017-08-17': .0002,\n", "    '2017-08-18': .0006,\n", "    '2017-08-19': .0013,\n", "    '2017-08-20': .0020,\n", "    '2017-08-21': .0033,\n", "    '2017-08-22': .0048,\n", "    '2017-08-23': .0063,\n", "    '2017-08-24': .0078,\n", "    '2017-08-25': .0093,\n", "    '2017-08-26': .0106,\n", "    '2017-08-27': .0120,\n", "    '2017-08-28': .0128,\n", "    '2017-08-29': .0136,\n", "    '2017-08-30': .0143,\n", "    '2017-08-31': .0151\n", "}"]}, {"metadata": {"collapsed": true, "_cell_guid": "b4a3927a-a8a8-4191-ab13-0958376fc771", "_uuid": "0e6720c2a6123ef4ca059721df918b40203b1dea"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["sub = test[['id','date']].copy()\n", "dates = list( sub.date.dt.strftime('%Y-%m-%d').values )\n", "scales = [scaledown[d] for d in dates]\n", "preds = np.expm1(test_pred * np.array(scales))\n", "sub['unit_sales'] = preds\n", "sub.drop(['id'],axis=1).set_index('date').apply(np.log1p).reset_index().groupby('date').mean().apply(np.expm1)"]}, {"metadata": {"collapsed": true, "_cell_guid": "cfb3a57e-9fe9-4ce5-89f0-9769513463ef", "_uuid": "01c035eb679aad4bcff9db0575739ca52935ddfe"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["sub.drop(['date'],axis=1,inplace=True)\n", "print(sub.shape)\n", "sub.head()"]}, {"metadata": {"collapsed": true, "_cell_guid": "0afda661-bf67-44d1-af14-e3ede77ef077", "_uuid": "03ca1a6363c826e60cba1464e9509c4f869423e0"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["sub.tail()"]}, {"metadata": {"collapsed": true, "_cell_guid": "886e91b2-5336-418b-b581-a32cd636380f", "_uuid": "c9eaf5e9537b50c6db5a56e138225d092ba7745e"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": ["sub.to_csv('new_items_sub.csv', float_format='%.6f', index=None)"]}, {"metadata": {"collapsed": true, "_cell_guid": "3defc5da-3202-4386-a1ed-c32707b039a3", "_uuid": "c6b746c899cbcafda1a45c7a6a5d6b5fe5e4a1bc"}, "outputs": [], "execution_count": null, "cell_type": "code", "source": []}]}