{"nbformat": 4, "nbformat_minor": 1, "cells": [{"outputs": [], "cell_type": "code", "execution_count": null, "source": ["\"\"\"\n", "This is based on Lingzhi's upgraded version \n", "  (https://www.kaggle.com/vrtjso/lgbm-one-step-ahead)\n", "of Ceshine's LGBM starter script\n", "  (https://www.kaggle.com/ceshine/lgbm-starter),\n", "\n", "but it is just the data preparation steps,\n", "so we don't have to repeat them every time.\n", "\"\"\"\n", "\n", "from datetime import date, timedelta\n", "\n", "import pandas as pd\n", "import numpy as np\n", "from sklearn.metrics import mean_squared_error\n", "import lightgbm as lgb"], "metadata": {"_cell_guid": "85bd1ecb-8be3-47f4-add8-17cc39c743c0", "collapsed": true, "_uuid": "c02f38e857702c3b23c6c5409c4a47fa43d77646"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["df_train = pd.read_csv(\n", "    '../input/train.csv', usecols=[1, 2, 3, 4, 5],\n", "    dtype={'onpromotion': bool},\n", "    converters={'unit_sales': lambda u: np.log1p(\n", "        float(u)) if float(u) > 0 else 0},\n", "    parse_dates=[\"date\"],\n", "    skiprows=range(1, 66458909)  # 2016-01-01\n", ")\n", "\n", "df_test = pd.read_csv(\n", "    \"../input/test.csv\", usecols=[0, 1, 2, 3, 4],\n", "    dtype={'onpromotion': bool},\n", "    parse_dates=[\"date\"]  # , date_parser=parser\n", ").set_index(\n", "    ['store_nbr', 'item_nbr', 'date']\n", ")\n", "\n", "items = pd.read_csv(\n", "    \"../input/items.csv\",\n", ").set_index(\"item_nbr\")"], "metadata": {"_cell_guid": "a4e286ba-a4e5-46be-a934-e27fd34bf7a2", "collapsed": true, "_uuid": "48b1bbf3a7076f45a8d1c8b5fc61cd9d8149f1d2"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["df_2017 = df_train.loc[df_train.date>=pd.datetime(2017,1,1)]\n", "del df_train"], "metadata": {"_cell_guid": "3ea3adff-3728-4356-97eb-7f0ca61b8615", "collapsed": true, "_uuid": "8ed7b631959d35089ad8f276da70afc097039874"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["promo_2017_train = df_2017.set_index(\n", "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack(\n", "        level=-1).fillna(False)\n", "promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)\n", "promo_2017_test = df_test[[\"onpromotion\"]].unstack(level=-1).fillna(False)\n", "promo_2017_test.columns = promo_2017_test.columns.get_level_values(1)\n", "promo_2017_train = promo_2017_train.reindex(promo_2017_test.index).fillna(False)\n", "promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)\n", "del promo_2017_test, promo_2017_train"], "metadata": {"_cell_guid": "117fe451-831b-4e2f-88d6-4afa28030965", "collapsed": true, "_uuid": "03915e0077769a8a4f7c7c239666e82a1f09bfed"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["df_2017 = df_2017.set_index(\n", "    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"unit_sales\"]].unstack(\n", "        level=-1).fillna(0)\n", "df_2017.columns = df_2017.columns.get_level_values(1)\n", "\n", "items = items.reindex(df_2017.index.get_level_values(1))"], "metadata": {"_cell_guid": "c01bf063-31f7-4218-b911-88b2746ddf9c", "collapsed": true, "_uuid": "b288d8eaa962dcd8c14c172638009ee359fc5e50"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["print(df_2017.shape)\n", "df_2017.head()"], "metadata": {"_cell_guid": "c3a7001f-b2c6-4396-be42-248a289dc5b6", "_uuid": "5e5f97f468bd4e20f31c8bc2440d09aa361e6a71"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["test_all = df_test[[]].reset_index()\n", "store_by_item = test_all[test_all.date==pd.to_datetime('2017-08-16')].drop(['date'],axis=1)\n", "del test_all\n", "print(store_by_item.shape)\n", "store_by_item.head()"], "metadata": {}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["df_2017 = store_by_item.join(df_2017,on=['store_nbr','item_nbr']).set_index(df_2017.index.names).fillna(0)\n", "print(df_2017.shape)\n", "df_2017.head()"], "metadata": {"_cell_guid": "c18e78dd-b6e8-4eda-921e-2ad09f6721e6", "_uuid": "90bd6c01d3e796cd63d462add99bcf2e66380e6e"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["def get_timespan(df, dt, minus, periods, freq='D'):\n", "    return df[pd.date_range(dt - timedelta(days=minus), periods=periods, freq=freq)]\n", "\n", "def prepare_dataset(t2017, is_train=True):\n", "    X = pd.DataFrame({\n", "        \"day_1_2017\": get_timespan(df_2017, t2017, 1, 1).values.ravel(),\n", "        \"mean_3_2017\": get_timespan(df_2017, t2017, 3, 3).mean(axis=1).values,\n", "        \"mean_7_2017\": get_timespan(df_2017, t2017, 7, 7).mean(axis=1).values,\n", "        \"mean_14_2017\": get_timespan(df_2017, t2017, 14, 14).mean(axis=1).values,\n", "        \"mean_30_2017\": get_timespan(df_2017, t2017, 30, 30).mean(axis=1).values,\n", "        \"mean_60_2017\": get_timespan(df_2017, t2017, 60, 60).mean(axis=1).values,\n", "        \"mean_140_2017\": get_timespan(df_2017, t2017, 140, 140).mean(axis=1).values,\n", "        \"promo_14_2017\": get_timespan(promo_2017, t2017, 14, 14).sum(axis=1).values,\n", "        \"promo_60_2017\": get_timespan(promo_2017, t2017, 60, 60).sum(axis=1).values,\n", "        \"promo_140_2017\": get_timespan(promo_2017, t2017, 140, 140).sum(axis=1).values\n", "    })\n", "    for i in range(7):\n", "        X['mean_4_dow{}_2017'.format(i)] = get_timespan(df_2017, t2017, 28-i, 4, freq='7D').mean(axis=1).values\n", "        X['mean_20_dow{}_2017'.format(i)] = get_timespan(df_2017, t2017, 140-i, 20, freq='7D').mean(axis=1).values\n", "    for i in range(16):\n", "        X[\"promo_{}\".format(i)] = promo_2017[\n", "            t2017 + timedelta(days=i)].values.astype(np.uint8)\n", "    if is_train:\n", "        y = df_2017[\n", "            pd.date_range(t2017, periods=16)\n", "        ].values\n", "        return X, y\n", "    return X"], "metadata": {"_cell_guid": "aadab5c3-cb4b-43f7-aeb2-84940fb47140", "collapsed": true, "_uuid": "6763ecf28f42618486da84539abd280a131d344d"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["t2017 = date(2017, 5, 31)\n", "X_l, y_l = [], []\n", "for i in range(6):\n", "    delta = timedelta(days=7 * i)\n", "    X_tmp, y_tmp = prepare_dataset(\n", "        t2017 + delta\n", "    )\n", "    X_l.append(X_tmp)\n", "    y_l.append(y_tmp)\n", "X_train = pd.concat(X_l, axis=0)\n", "y_train = np.concatenate(y_l, axis=0)\n", "del X_l, y_l\n", "X_val, y_val = prepare_dataset(date(2017, 7, 26))\n", "X_test = prepare_dataset(date(2017, 8, 16), is_train=False)"], "metadata": {"_cell_guid": "fa95c0a8-40e4-4950-8493-c8e8a81a3e70", "_uuid": "6f744c0a2fdf7941076024e136fe92daf0466c65"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": ["X_train.to_csv('X_train.csv', float_format='%.6f', index=None)\n", "pd.DataFrame(y_train).to_csv('y_train.csv', float_format='%.6f', index=None)\n", "X_val.to_csv('X_val.csv', float_format='%.6f', index=None)\n", "pd.DataFrame(y_val).to_csv('y_val.csv', float_format='%.6f', index=None)\n", "X_test.to_csv('X_test.csv', float_format='%.6f', index=None)\n", "pd.DataFrame(index=df_2017.index).to_csv('stores_items.csv')\n", "df_test[['id']].to_csv('test_ids.csv')"], "metadata": {"_cell_guid": "f19fcab5-f292-4e79-a6e2-ac7ffc48062c", "collapsed": true, "_uuid": "7be8e5f6c7836e1ca78f3695ce5eeb57e62b996a"}}, {"outputs": [], "cell_type": "code", "execution_count": null, "source": [], "metadata": {"_cell_guid": "5ce9401e-c01b-4ea5-96d6-982a88116d39", "collapsed": true, "_uuid": "3ba4e2df1e48b824975cf8c5a6f638a79eaddb4d"}}], "metadata": {"kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}, "language_info": {"mimetype": "text/x-python", "codemirror_mode": {"version": 3, "name": "ipython"}, "file_extension": ".py", "nbconvert_exporter": "python", "name": "python", "pygments_lexer": "ipython3", "version": "3.6.3"}}}