{"cells":[{"metadata":{"_uuid":"95bd9ac72b02ae8cdb2227073360c868a83e2159"},"cell_type":"markdown","source":"# Import"},{"metadata":{"trusted":true},"cell_type":"code","source":"# インタネットをOnになる\n!pip install py7zr","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nfrom datetime import date, timedelta\nimport gc\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\nimport lightgbm as lgb\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(pd.__version__)\nprint(np.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import py7zr\n\nfor f in ['test.csv.7z', 'items.csv.7z', 'stores.csv.7z', 'train.csv.7z']:\n    zf = py7zr.SevenZipFile(os.path.join('..', 'input', f), mode='r')\n    zf.extractall('../working')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('../working/'))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# Interesting points\n\n* We have to predict the sale of an Item in a Store in 16 days windows. Last day in train data is 8/15, using data to this cutoff date, we have to predict sale of Milk in Store A in each of next 16 dates (8/16, 8/17, ..., 8/30, 8/31). One can do 1-day-rolling-training (using data up to 8/29 as train to predict 8/30, using data up to 8/30 as train to predict 8/31), but we don't have real label in 16 dates window.\n* As with many Tabular data, there are many many features and feature engineering is the hardest part. And this is a TimeSeries feature which makes problem even harder.\n* Training data is really big (~125 million rows in almost 4 years). So the question is: do we really need to use all data? If not, how much?\n* I'm not familiar with TimeSeries data, so this is a huge opportunity to learn TimeSeries feature engineering.\n\n# Winning Solution\n\nThis is the simplified solution of [1st place solution](https://www.kaggle.com/shixw125/1st-place-lgb-model-public-0-506-private-0-511). Lets dive in.\n\nValidation strategy:\n\n> train data：20170531 - 20170719 or 20170614 - 20170719, different models are trained with different data set. validition: 20170726 - 20170810\n\nIdea:\n\nBuild 16 models, each using same training data but predict for different day in future. For example: model 1 trained on X then predicts for t+1. Model 2 trained on X then predicts for t+2, etc...\n\nFeature Engineering: The most cool part in winning solution. And this notebook tries to break down the feature engineering part."},{"metadata":{"trusted":true,"_uuid":"05d275401b99a1454c2cbedeff098e5970117d2b"},"cell_type":"code","source":"df_train = pd.read_csv(\n    '../working/train.csv', usecols=[1, 2, 3, 4, 5],\n    dtype={'onpromotion': bool},\n    converters={'unit_sales': lambda u: np.log1p(\n        float(u)) if float(u) > 0 else 0},\n    parse_dates=[\"date\"],\n    skiprows=range(1, 66458909)  # from 2016-01-01\n)\n\ndf_test = pd.read_csv(\n    \"../working/test.csv\", usecols=[0, 1, 2, 3, 4],\n    dtype={'onpromotion': bool},\n    parse_dates=[\"date\"]\n).set_index(\n    ['store_nbr', 'item_nbr', 'date']\n)\n\nitems = pd.read_csv(\n    \"../working/items.csv\",\n).set_index(\"item_nbr\")\n\nstores = pd.read_csv(\n    \"../working/stores.csv\",\n).set_index(\"store_nbr\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6b5a2a40de3ab798efc6927e63dce81805c4089"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c6c8cd96442b591402c25b43b3c72961ddb6d27"},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8809bdf82f08061bd063c1096dff2f8d3d76b8f"},"cell_type":"code","source":"items.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa83452147117bb88cac9074b511c8dfce3f818f"},"cell_type":"code","source":"stores.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ff31e359efe8a802c0465e432ae66ce025f96c4"},"cell_type":"code","source":"le = LabelEncoder()\nitems['family'] = le.fit_transform(items['family'].values)\nstores['city'] = le.fit_transform(stores['city'].values)\nstores['state'] = le.fit_transform(stores['state'].values)\nstores['type'] = le.fit_transform(stores['type'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"35c2619438a4e7ceb2fc1790ab8640e0bd9a7073"},"cell_type":"code","source":"items.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c382645f1ea9cb6c4b585380d2a8fc2caf32691a"},"cell_type":"code","source":"stores.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6a317faeb70a700bda6febd2dca6a0da1a1e0d8"},"cell_type":"code","source":"df_2017 = df_train.loc[df_train.date>=pd.datetime(2017,1,1)]\ndel df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c204cece87a7d2ede71b7f24fbb9dd6bee8bcec"},"cell_type":"code","source":"df_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57d4293f21eb878429c800dce4a78b89f5d51306"},"cell_type":"code","source":"df_2017.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"506eb57355922e9045170999f0121ad6c9ffa532"},"cell_type":"code","source":"promo_2017_train = df_2017.set_index([\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack(level=-1).fillna(False)\npromo_2017_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4427d2f36e64eedfb22fbe60243501f3fafdccc7"},"cell_type":"code","source":"promo_2017_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8172c9659a530b4e3ac2841ba1a20566646fb691"},"cell_type":"code","source":"promo_2017_train.columns.get_level_values(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e1a9cb943ecfd6d03a2f6ce9175feb4228cef31"},"cell_type":"code","source":"promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)\npromo_2017_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"937ba2568f4f5b8f62117e9fce967f6dab7cc907"},"cell_type":"code","source":"promo_2017_test = df_test[[\"onpromotion\"]].unstack(level=-1).fillna(False)\npromo_2017_test.columns = promo_2017_test.columns.get_level_values(1)\npromo_2017_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fe7e6bb3fe42ccb5fa57a0ff49b07d9ad824816"},"cell_type":"code","source":"promo_2017_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45ec39c63946496d4d863edc41f682184f55a015"},"cell_type":"code","source":"# align test (store, item) index as same as train index.\n# some (store, item) will be missing.\npromo_2017_test = promo_2017_test.reindex(promo_2017_train.index).fillna(False)\npromo_2017_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e22be5b64ba77f2a0ae950691b0ce1a023e6c965"},"cell_type":"code","source":"promo_2017_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e893a5c743c19f43d1ead24170757c7eab99d8cb"},"cell_type":"code","source":"promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)\ndel promo_2017_test, promo_2017_train\npromo_2017.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba34bcc0a03be1260b77c70f3feb14973562d30a"},"cell_type":"code","source":"promo_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8b491322930e95c799ed9d86e4f4cd82eab39f1"},"cell_type":"code","source":"df_2017 = df_2017.set_index([\"store_nbr\", \"item_nbr\", \"date\"])[[\"unit_sales\"]].unstack(level=-1).fillna(0)\ndf_2017.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abf4010002278f3604115a679f6592425e00e7a1"},"cell_type":"code","source":"df_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b459bb5d87861fd6910b6fd2c9db6e0a85261ace"},"cell_type":"code","source":"df_2017.columns = df_2017.columns.get_level_values(1)\ndf_2017.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91f7b0047890e5b6096a5fe9dafb036d6bbe2d1d"},"cell_type":"code","source":"df_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"590d95e398f461a4f393cd8b87beacf548a684be"},"cell_type":"code","source":"items.shape, stores.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"items.shape, stores.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"959dc9cfe255673634f14314e6e0a27dcb93ff33"},"cell_type":"code","source":"# align items\nitems = items.reindex(df_2017.index.get_level_values(1))\nstores = stores.reindex(df_2017.index.get_level_values(0))\nitems.shape, stores.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edbbbb3b97ce8742d258e0b960bff1e6bdaf4dba"},"cell_type":"code","source":"items.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a699c8075a209d6300ed317c4f0f8dacfe08c02b"},"cell_type":"code","source":"stores.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6d0653e7c4e2c13111e109a7fb5fcddb5c8ddad"},"cell_type":"code","source":"df_2017_item = df_2017.groupby('item_nbr')[df_2017.columns].sum()\ndf_2017_item.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d7cae252d49fb97fed08592bd8160555fce2928"},"cell_type":"code","source":"df_2017_item.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"387f788704942b36216e45cd29d5c0b1173c858b"},"cell_type":"code","source":"promo_2017_item = promo_2017.groupby('item_nbr')[promo_2017.columns].sum()\npromo_2017_item.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a083a989e0d51b831e08dd7466c38f9abcc921a"},"cell_type":"code","source":"promo_2017_item.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"884101de50c237e73ee436ca8efdc7448681cd8a"},"cell_type":"code","source":"df_2017_store_class = df_2017.reset_index()\ndf_2017_store_class.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6ee83c1c921efd3ade46b1eb8b10f33b900d952"},"cell_type":"code","source":"df_2017_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae9ddf7dd0738e1f4240994897ba969c38628638"},"cell_type":"code","source":"# df and items now have same row index.\ndf_2017_store_class['class'] = items['class'].values\ndf_2017_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4189a973c21893d638490db34e4f5659f8b2e4a6"},"cell_type":"code","source":"df_2017_store_class_index = df_2017_store_class[['class', 'store_nbr']]\ndf_2017_store_class_index.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96014b432e98fcad86102ed2feb2ad1526b57ea7"},"cell_type":"code","source":"df_2017_store_class_index.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e6da80e1d1879f6bda90b5def2bc5f78beea19a"},"cell_type":"code","source":"df_2017_store_class = df_2017_store_class.groupby(['class', 'store_nbr'])[df_2017.columns].sum()\ndf_2017_store_class.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e326e3d82b261f4f4e869ad47a07f84f37ac4189"},"cell_type":"code","source":"df_2017_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"863277b19e6c1784495d77cdb41e16ff0d8c810b"},"cell_type":"code","source":"df_2017_promo_store_class = promo_2017.reset_index()\ndf_2017_promo_store_class.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e1289b49b2b97365569fe2f74b6b02c888008e70"},"cell_type":"code","source":"df_2017_promo_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bbce1604740cb2b428612afc9b6c2d93a345c9c"},"cell_type":"code","source":"df_2017_promo_store_class['class'] = items['class'].values\ndf_2017_promo_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de47be895a8d241964aa85b706f5b0bbac76ae0c"},"cell_type":"code","source":"df_2017_promo_store_class_index = df_2017_promo_store_class[['class', 'store_nbr']]\ndf_2017_promo_store_class = df_2017_promo_store_class.groupby(['class', 'store_nbr'])[promo_2017.columns].sum()\ndf_2017_promo_store_class.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0c66cec0e72719f870b195dab164e6cfed5d056"},"cell_type":"code","source":"df_2017_promo_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ae4b7c3600b0d3280f1c8bc18f8b8ce2d89ef89"},"cell_type":"code","source":"def get_timespan(df, dt, minus, periods, freq='D'):\n    \"\"\" Back minus days, get n==periods dates, each period is freq (D) away.\n        if dt=6/16, minus=5, period=3, fred=D\n        then return 6/11, 6/12, 6/13.\n    \"\"\"\n    return df[pd.date_range(dt - timedelta(days=minus), periods=periods, freq=freq)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50432d97cfcc09669deab2c45a0882600ace8cb4"},"cell_type":"code","source":"t2017 = date(2017, 6, 14)\npd.date_range(t2017, periods=16)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8471f6f3dbc704ec1df0650146cd3c8ac650f6cc"},"cell_type":"code","source":"dt = t2017 = date(2017, 6, 14)\nminus = 5\nperiods = 3\nfreq = \"D\"\npd.date_range(dt - timedelta(days=minus), periods=periods, freq=freq)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1edaeae0187d695199907098660cd79fe0a72e32"},"cell_type":"code","source":"print(\"Preparing dataset...\")\nt2017 = date(2017, 6, 14)\nnum_days = 6\nX_l, y_l = [], []\nfor i in range(num_days):\n    delta = timedelta(days=7 * i)\n    print(t2017, delta, t2017 + delta)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9dc3fe1b617347d20690fca276ea6eb11af57017"},"cell_type":"code","source":"def prepare_dataset(df, promo_df, t2017, is_train=True, name_prefix=None):\n    \"\"\"\n    args:\n    ----\n        df: sale data\n        promo_df: promo data\n        t2017: pivot date\n    \"\"\"\n    \n    # Promotion counts.\n    # How many promotions in last 14 days.\n    # How many promotions in next 3 days. \n    X = {\n        \"promo_14_2017\": get_timespan(promo_df, t2017, 14, 14).sum(axis=1).values,\n        \"promo_3_2017_aft\": get_timespan(promo_df, t2017 + timedelta(days=16), 15, 3).sum(axis=1).values, \n        \"promo_7_2017_aft\": get_timespan(promo_df, t2017 + timedelta(days=16), 15, 7).sum(axis=1).values,\n        \"promo_14_2017_aft\": get_timespan(promo_df, t2017 + timedelta(days=16), 15, 14).sum(axis=1).values,\n    }\n\n    # Sale on promotion and Non-promotion days.\n    for i in [3, 7, 14]:\n        # get sale in last i days.\n        # get promo flag in last i days. if date has promo, value is 1. else, value is 0.\n        tmp1 = get_timespan(df, t2017, i, i)\n        tmp2 = (get_timespan(promo_df, t2017, i, i) > 0) * 1\n\n        # average sale on promo dates in last i=3, 7, ... days. if last 3 days, 2 has promo, then average sale of these 2 promo dates.\n        X['has_promo_mean_%s' % i] = (tmp1 * tmp2.replace(0, np.nan)).mean(axis=1).values\n        # most recent dates has more influences on average sale.\n        # if i == 3, np.power(0.9, np.arange(i)[::-1]) == [0.81, 0.9 , 1.  ]\n        X['has_promo_mean_%s_decay' % i] = (tmp1 * tmp2.replace(0, np.nan) * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n        # average sale on NON-promo dates in last i=3, 7, ... days. if last 3 days, 2 has NO-promo, then average sale of these 2 dates.\n        X['no_promo_mean_%s' % i] = (tmp1 * (1 - tmp2).replace(0, np.nan)).mean(axis=1).values\n        # most recent dates has more influences on average sale.\n        # if i == 3, np.power(0.9, np.arange(i)[::-1]) == [0.81, 0.9 , 1.  ]\n        X['no_promo_mean_%s_decay' % i] = (tmp1 * (1 - tmp2).replace(0, np.nan) * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n\n        \n    # Stats on sale values.\n    for i in [3, 7, 14]:\n        # Get sale in last i days.\n        tmp = get_timespan(df, t2017, i, i)\n        # descritive stats on sale.\n        X['mean_%s' % i] = tmp.mean(axis=1).values\n        X['median_%s' % i] = tmp.median(axis=1).values\n        X['min_%s' % i] = tmp.min(axis=1).values\n        X['max_%s' % i] = tmp.max(axis=1).values\n        X['std_%s' % i] = tmp.std(axis=1).values\n        # weighted mean, most recent contribute more on average.\n        X['mean_%s_decay' % i] = (tmp * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n        # get diff of day_T vs. day_T-1 > get mean.\n        X['diff_%s_mean' % i] = tmp.diff(axis=1).mean(axis=1).values\n\n\n    # using same stats on sale, but now shift back 1 more week.\n    for i in [3, 7, 14]:\n        tmp = get_timespan(df, t2017 + timedelta(days=-7), i, i)\n        X['diff_%s_mean_2' % i] = tmp.diff(axis=1).mean(axis=1).values\n        X['mean_%s_decay_2' % i] = (tmp * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n        X['mean_%s_2' % i] = tmp.mean(axis=1).values\n        X['median_%s_2' % i] = tmp.median(axis=1).values\n        X['min_%s_2' % i] = tmp.min(axis=1).values\n        X['max_%s_2' % i] = tmp.max(axis=1).values\n        X['std_%s_2' % i] = tmp.std(axis=1).values\n\n\n    for i in [7, 14]:\n        # sale in last 3 days.\n        tmp = get_timespan(df, t2017, i, i)\n        # how many days has sales in last 3 day\n        X['has_sales_days_in_last_%s' % i] = (tmp > 0).sum(axis=1).values\n        # distance to last day has sale. if i == 3, sales = [0, 4, 3], then distance = 1\n        # if sales = [0, 4, 0], then distance = 2, etc...\n        X['last_has_sales_day_in_last_%s' % i] = i - ((tmp > 0) * np.arange(i)).max(axis=1).values\n        # distance to first day has sale.\n        X['first_has_sales_day_in_last_%s' % i] = ((tmp > 0) * np.arange(i, 0, -1)).max(axis=1).values\n\n        # promo in last 3 days\n        tmp = get_timespan(promo_df, t2017, i, i)\n        # how many days has promo\n        X['has_promo_days_in_last_%s' % i] = (tmp > 0).sum(axis=1).values\n        # distance to last day has promo\n        X['last_has_promo_day_in_last_%s' % i] = i - ((tmp > 0) * np.arange(i)).max(axis=1).values\n        # distance to first day has promo\n        X['first_has_promo_day_in_last_%s' % i] = ((tmp > 0) * np.arange(i, 0, -1)).max(axis=1).values\n\n\n    # promo in next 16 days\n    tmp = get_timespan(promo_df, t2017 + timedelta(days=16), 15, 15)\n    # how many promo in next 16 days\n    X['has_promo_days_in_after_15_days'] = (tmp > 0).sum(axis=1).values\n    # distance to last day has promo\n    X['last_has_promo_day_in_after_15_days'] = i - ((tmp > 0) * np.arange(15)).max(axis=1).values\n    # distance to first day has promo\n    X['first_has_promo_day_in_after_15_days'] = ((tmp > 0) * np.arange(15, 0, -1)).max(axis=1).values\n\n    # get sale in day t-1, t-2, t-3, ...\n    for i in range(1, 7):\n        X['day_%s_2017' % i] = get_timespan(df, t2017, i, 1).values.ravel()\n\n    # get promo in date ... t-3, t-2, t-1, t, t1, t2, t3...\n    for i in range(-7, 7):\n        X[\"promo_{}\".format(i)] = promo_df[t2017 + timedelta(days=i)].values.astype(np.uint8)\n\n    # for each day of week, for example wednesday\n    # get previous 4/20 wednesdays sales, then take average\n    for i in range(7):\n        X['mean_4_dow{}_2017'.format(i)] = get_timespan(df, t2017, 28-i, 4, freq='7D').mean(axis=1).values\n        X['mean_20_dow{}_2017'.format(i)] = get_timespan(df, t2017, 140-i, 20, freq='7D').mean(axis=1).values\n\n    X = pd.DataFrame(X)\n\n    if is_train:\n        y = df[\n            pd.date_range(t2017, periods=16)\n        ].values\n        return X, y\n    if name_prefix is not None:\n        X.columns = ['%s_%s' % (name_prefix, c) for c in X.columns]\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"deacb303d90ee799e388e038ca159394c7c01bbc"},"cell_type":"code","source":"tmp1 = get_timespan(df_2017, t2017, 3, 3).head()\ntmp1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d87ed1af4ea722b51881ca366748868bba6ac7f"},"cell_type":"code","source":"tmp2 = (get_timespan(promo_2017, t2017, 3, 3) > 0) * 1\ntmp2.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e95c199c7253d82c488fe1bc426d920a54523ef"},"cell_type":"code","source":"(tmp1 * tmp2.replace(0, np.nan)).mean(axis=1).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8095f5ad74e75c71be3b9947d1551228401c3f1"},"cell_type":"code","source":"np.power(0.9, np.arange(3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ce9724b31e9128266ed56fe353d10820f0f9bda"},"cell_type":"code","source":"np.power(0.9, np.arange(3)[::-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b648db773996e7b6a3df39bb99f65b3882cb981b"},"cell_type":"code","source":"tmp = get_timespan(df_2017, t2017, 3, 3)\ntmp.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"812e7b6cc6dc1289d1e865aef27f579058274bdf"},"cell_type":"code","source":"tmp.head().diff(axis=1)#.mean(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa4b60b90baf5cc1e8ad916a8ddeb3dab20f203d"},"cell_type":"code","source":"tmp.head().diff(axis=1).mean(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ecc94783aa97b60d43751b16182e4280555f7a21"},"cell_type":"code","source":"tmp = get_timespan(df_2017, t2017, 3, 3)\ntmp.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"001f0406f9d77b619d4c39d4a0a8c8e8830c23f3"},"cell_type":"code","source":"(tmp > 0).sum(axis=1).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3728f3ea7f1ed4a77a8f6522d72c3496dcb70be5"},"cell_type":"code","source":"((tmp > 0) * np.arange(3)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0c5c55d7239aa396c908d6de03f7ca12601fa6d"},"cell_type":"code","source":"((tmp > 0) * np.arange(3)).head().max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffe63c279d0f88229bae0c854c4912fa81590c72"},"cell_type":"code","source":"3 - ((tmp > 0) * np.arange(3)).head().max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"237767335a3c265c8feceb3935c151908ab1e934"},"cell_type":"code","source":"np.arange(3, 0, -1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3a5f33dacbf301c7b1a57605e8a1e76ceafabc8"},"cell_type":"code","source":"((tmp > 0) * np.arange(3, 0, -1)).head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3b0a10fb64be60dd09f28ca649af3520331441d"},"cell_type":"code","source":"((tmp > 0) * np.arange(3, 0, -1)).head().max(axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"797d796cdc9046e2c6734ced012fce36c787438d"},"cell_type":"code","source":"t2017","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f62b2fd1d150bd7ac8e76ad00b5aaa26a81bd9d2"},"cell_type":"code","source":"get_timespan(df_2017, t2017, 28-0, 4, freq='7D').head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36ed3d0925491aadeb5eb47305fe56bc5403d3e3"},"cell_type":"code","source":"t2017 = date(2017, 6, 14)\nminus = 28\nperiods = 4\npd.date_range(t2017 - timedelta(days=minus), periods=periods, freq=\"7D\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"promo_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb42cd67c587acbad5a558538d37ce5917b2240d"},"cell_type":"code","source":"print(\"Preparing dataset...\")\nt2017 = date(2017, 6, 14)\nnum_days = 5\nX_l, y_l = [], []\nfor i in range(num_days):\n    delta = timedelta(days=7 * i)\n    print(t2017, delta, t2017 + delta)\n    print(\"process sales...\")\n    X_tmp, y_tmp = prepare_dataset(df_2017, promo_2017, t2017 + delta)\n    print(X_tmp.shape, y_tmp.shape)\n\n    print(\"process items...\")\n    X_tmp2 = prepare_dataset(df_2017_item, promo_2017_item, t2017 + delta, is_train=False, name_prefix='item')\n    print(X_tmp2.shape)\n    X_tmp2.index = df_2017_item.index\n    print(X_tmp2.shape)\n    X_tmp2 = X_tmp2.reindex(df_2017.index.get_level_values(1)).reset_index(drop=True)\n    print(X_tmp2.shape)\n\n    print(\"process store...\")\n    X_tmp3 = prepare_dataset(df_2017_store_class, df_2017_promo_store_class, t2017 + delta, is_train=False, name_prefix='store_class')\n    print(X_tmp3.shape)\n    X_tmp3.index = df_2017_store_class.index\n    print(X_tmp3.shape)\n    X_tmp3 = X_tmp3.reindex(df_2017_store_class_index).reset_index(drop=True)\n    print(X_tmp3.shape)\n\n    print(\"append sale, item, store, item-context, store-context data.\")\n    X_tmp = pd.concat([X_tmp, X_tmp2, X_tmp3, items.reset_index(), stores.reset_index()], axis=1)\n    X_tmp['date'] = t2017 + delta\n    print(X_tmp.shape)\n          \n    X_l.append(X_tmp)\n    y_l.append(y_tmp)\n\n    del X_tmp2\n    gc.collect()\n    print(\"-\" * 30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dad99c51cfd8296fdd27758a25a7ca9e4f2dd71"},"cell_type":"code","source":"X_train = pd.concat(X_l, axis=0)\ny_train = np.concatenate(y_l, axis=0)\ndel X_l, y_l\nX_train.shape, y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"941dbb3c717cc362a6b7d39816c4a72b2652fff7"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5e027a90ba475c494caa647e396b03429b0ad2b"},"cell_type":"code","source":"y_train[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.remove('../working/stores.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nwith open(os.path.join('..', 'working', 'X_train.pkl'), 'wb') as f:\n    pickle.dump(X_train, f)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nwith open(os.path.join('..', 'working', 'y_train.pkl'), 'wb') as f:\n    pickle.dump(y_train, f)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1dd4e200b3a7dab26ee2d83e5036d6dde3729e6a"},"cell_type":"code","source":"X_val, y_val = prepare_dataset(df_2017, promo_2017, date(2017, 7, 26))\n\nX_val2 = prepare_dataset(df_2017_item, promo_2017_item, date(2017, 7, 26), is_train=False, name_prefix='item')\nX_val2.index = df_2017_item.index\nX_val2 = X_val2.reindex(df_2017.index.get_level_values(1)).reset_index(drop=True)\n\nX_val3 = prepare_dataset(df_2017_store_class, df_2017_promo_store_class, date(2017, 7, 26), is_train=False, name_prefix='store_class')\nX_val3.index = df_2017_store_class.index\nX_val3 = X_val3.reindex(df_2017_store_class_index).reset_index(drop=True)\n\nX_val = pd.concat([X_val, X_val2, X_val3, items.reset_index(), stores.reset_index()], axis=1)\nX_val['date'] = date(2017, 7, 26)\nX_val.shape, y_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(os.path.join('..', 'working', 'X_val.pkl'), 'wb') as f:\n    pickle.dump(X_val, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(os.path.join('..', 'working', 'y_val.pkl'), 'wb') as f:\n    pickle.dump(y_val, f)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26375e87c5c9da8ef2620d3d459d1c3fa3348eef"},"cell_type":"code","source":"X_test = prepare_dataset(df_2017, promo_2017, date(2017, 8, 16), is_train=False)\n\nX_test2 = prepare_dataset(df_2017_item, promo_2017_item, date(2017, 8, 16), is_train=False, name_prefix='item')\nX_test2.index = df_2017_item.index\nX_test2 = X_test2.reindex(df_2017.index.get_level_values(1)).reset_index(drop=True)\n\nX_test3 = prepare_dataset(df_2017_store_class, df_2017_promo_store_class, date(2017, 8, 16), is_train=False, name_prefix='store_class')\nX_test3.index = df_2017_store_class.index\nX_test3 = X_test3.reindex(df_2017_store_class_index).reset_index(drop=True)\n\nX_test = pd.concat([X_test, X_test2, X_test3, items.reset_index(), stores.reset_index()], axis=1)\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f298327c070a3e6b49ea2ab7c0b6fa6536aa8879"},"cell_type":"code","source":"del X_test2, X_val2, df_2017_item, promo_2017_item, df_2017_store_class, df_2017_promo_store_class, df_2017_store_class_index\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"117c67cd565aa449b3f68084c10a8da6f590aa70"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}