{"cells":[{"metadata":{"trusted":true,"_uuid":"7b8cb0c361f28278afed36b6b5be2cf9d7aa2f62"},"cell_type":"code","source":"\"\"\"\nThis is an upgraded version of Ceshine's LGBM starter script, simply adding more\naverage features and weekly average features on it.\n\"\"\"\nfrom datetime import date, timedelta\nimport gc # garbage collector\nimport pandas as pd\nimport numpy as np\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.preprocessing import LabelEncoder\n\nimport lightgbm as lgb\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43b44c255dccaa11f6a87188dfeef8bce5917ae9"},"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    #start_mem = df.memory_usage().sum() / 1024**2\n    #print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            elif str(col_type)[:3] == 'float':\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n\n    #end_mem = df.memory_usage().sum() / 1024**2\n    #print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    #print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10115db8f13b112df3fb93c5b084eb736cd2c63c"},"cell_type":"code","source":"import sys\ndef sizeof_fmt(num, suffix='B'):\n    ''' By Fred Cirera, after https://stackoverflow.com/a/1094933/1870254'''\n    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n        if abs(num) < 1024.0:\n            return \"%3.1f%s%s\" % (num, unit, suffix)\n        num /= 1024.0\n    return \"%.1f%s%s\" % (num, 'Yi', suffix)\n\nfor name, size in sorted(((name, sys.getsizeof(value)) for name,value in locals().items()), key= lambda x: -x[1])[:10]:\n    print(\"{:>30}: {:>8}\".format(name,sizeof_fmt(size)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc09741555cd38d98d5c4aedf7a3bd1d6cda04c1"},"cell_type":"code","source":"#del _48\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17d7abcd95987cde97d6c6fc9e79eea55112e9bb"},"cell_type":"markdown","source":"# Loading of CSVs"},{"metadata":{"_uuid":"a203bc7ea2b9909f596fc2d0ce34e7eafbfc3ce6"},"cell_type":"markdown","source":"## items"},{"metadata":{"_uuid":"8327a13f1df32550cff381b49c4fdee0dfee967f"},"cell_type":"markdown","source":"item_nbr is set as the index of the dataframe"},{"metadata":{"trusted":true,"_uuid":"8c328a28a74a0f9a131f241a44242c6de2732076"},"cell_type":"code","source":"items = pd.read_csv(\n    \"../input/items.csv\",\n).set_index(\"item_nbr\")\n\n#items = reduce_mem_usage(items)\n\nitems.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b43008a8239431216cfab2e65043bb6980aa52bc"},"cell_type":"markdown","source":"## stores"},{"metadata":{"_uuid":"e3fc5fd91aa2f48aec495c8481db524b533d3b44"},"cell_type":"markdown","source":"store_nbr is set as the index of the dataframe"},{"metadata":{"trusted":true,"_uuid":"6f7a9d28e32bd0e24dc82da39d423588bf1a81ab"},"cell_type":"code","source":"stores = pd.read_csv(\n    \"../input/stores.csv\",\n).set_index(\"store_nbr\")\n\n#stores = reduce_mem_usage(stores)\n\nstores.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5f483364c3a856849b033603a881d48ed0d89d57"},"cell_type":"markdown","source":"## test"},{"metadata":{"_uuid":"6ae41d59ce14a9ef2f67cd01682c3d4f63b7cbce"},"cell_type":"markdown","source":"We make sure that that date feature is of type datetime64 and we set three indices."},{"metadata":{"trusted":true,"_uuid":"117a601676f126b13cff361ac43da3a09367b1d4"},"cell_type":"code","source":"df_test = pd.read_csv(\n    \"../input/test.csv\", usecols=[0, 1, 2, 3, 4],\n    dtype= {'onpromotion': bool},\n    parse_dates=[\"date\"]  # , date_parser=parser\n).set_index(\n    ['store_nbr', 'item_nbr', 'date']\n)\n\n#df_test = reduce_mem_usage(df_test)\n\ndf_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd7cb01d483fc9673e5c288d3588ac4ba1ed8d79"},"cell_type":"code","source":"df_test.dtypes","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e2899f254b54a52c653e40a44b9b2bdc51b260af"},"cell_type":"markdown","source":"## train"},{"metadata":{"_uuid":"e4cfe38484f5ea890023dc8c9cb968089064999f"},"cell_type":"markdown","source":"We convert unit_sales to its logarithmic value. \n\nWe also make sure that the feature date is of type datetime and \"onpromotion\" of type bool.\n\nWe take rows after 2016."},{"metadata":{"trusted":true,"_uuid":"6a70fb65e6b1771a8d58c758e8609fcfab3d19fd"},"cell_type":"code","source":"df_train = pd.read_csv(\n    '../input/train.csv', usecols=[1, 2, 3, 4, 5],\n    dtype={'onpromotion': bool},\n    converters={'unit_sales': lambda u: np.log1p(\n        float(u)) if float(u) > 0 else 0},\n    parse_dates=[\"date\"],\n    skiprows=range(1, 66458909)  # 2016-01-01\n)\n#df_train = reduce_mem_usage(df_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6596804e6bd6bae158caa0c8d98e130729449101"},"cell_type":"code","source":"#df_train.head()\ndf_train.dtypes\n#df_train['item_nbr'].max()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3eccf700b9db5223809be3d25c9c2d1d2795c11"},"cell_type":"markdown","source":"# Encoding the categorial features"},{"metadata":{"trusted":true,"_uuid":"a06d057ac2499f7d3c8d9f9e7227eb87dcee6859"},"cell_type":"code","source":"le = LabelEncoder()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90af9e26ec6480d075ffdf39f7f8fdc557a6c00d"},"cell_type":"markdown","source":"## items"},{"metadata":{"trusted":true,"_uuid":"1cd96c93eb340ac28745e393d6c728052e970c1a"},"cell_type":"code","source":"items['family'] = le.fit_transform(items['family'].values)\n#items = reduce_mem_usage(items)\nitems.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"531f394452f36474414f9bdc93d6ca7be107fc12"},"cell_type":"markdown","source":"## stores"},{"metadata":{"trusted":true,"_uuid":"8319bdad66d7b4b76b4492180adbac9ab1b1d64f"},"cell_type":"code","source":"stores['city'] = le.fit_transform(stores['city'].values)\nstores['state'] = le.fit_transform(stores['state'].values)\nstores['type'] = le.fit_transform(stores['type'].values)\n#stores = reduce_mem_usage(stores)\nstores.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ddb7c599f6a43df71b4fa40c526ff1d9d344b36"},"cell_type":"markdown","source":"## Selecting the training set"},{"metadata":{"trusted":true,"_uuid":"893e86a48daf92f138568054750c8d12eb13c9b9"},"cell_type":"code","source":"df_2017 = df_train.loc[df_train.date>=pd.datetime(2017,1,1)]\ndel df_train\ngc.collect()\n\ndf_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f1396faaaa0c3f852412d43db6de0f5060cbb04"},"cell_type":"markdown","source":"* Setting of the indices like in the training set.\n* Getting rid of unit_sales since we are exclusively interested in promotions here\n* Unstacking the dates\n* Filling the missing information with False. It might not be the best strategy, look at sjv solution:\n> As discussed in the forums, if you naively impute missing onpromotion values with [False], the sales distribution conditioned on onpromotion values will differ in the train and test sets. I didn't find a way to remedy this, but thankfully my teammates were able to come up with a clever solution which improved my score by about .004. The missing values were imputed randomly: [True] with probability p and [False] otherwise. Determining p was a rather tedious task, since we had no data to model it and we had to resort to trial and error leaderboard submissions. We ended up setting p separately for each day, and it was computed as the mean onpromotion rate for each day, scaled by a learnable factor estimated by stochastic leaderboard descent (leaderboard probing…). All credit goes to my teammates for this, and they can probably explain it better than I can."},{"metadata":{"_uuid":"911747bdcb88207556c519745cc1a01e6e153c63"},"cell_type":"markdown","source":"# Creation of promotions dataframe"},{"metadata":{"trusted":true,"_uuid":"f2c6e182db088a736999adbdc2d41cd357eb46c6"},"cell_type":"code","source":"promo_2017_train = df_2017.set_index(\n    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"onpromotion\"]].unstack(\n        level=-1).fillna(False)\n\n\npromo_2017_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f61c9265f3d4358fd86297490df450b70ca2757b"},"cell_type":"markdown","source":"Removing the first useless row"},{"metadata":{"trusted":true,"_uuid":"686ceb1d3fc724e21a180b32cbf47103e68698c9"},"cell_type":"code","source":"promo_2017_train.columns = promo_2017_train.columns.get_level_values(1)\npromo_2017_train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03ee7c2defa642979427eb772ba2e83b1151a372"},"cell_type":"markdown","source":"Doing the same operations for test set"},{"metadata":{"trusted":true,"_uuid":"12afcc17f015ef6e25ee34a671998fe03ff50319"},"cell_type":"code","source":"promo_2017_test = df_test[[\"onpromotion\"]].unstack(level=-1).fillna(False)\npromo_2017_test.columns = promo_2017_test.columns.get_level_values(1)\npromo_2017_test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b65357840799329189de5a5ef97e85e9c78945a"},"cell_type":"markdown","source":"Reindexing of the test set:\n* Removing products that were not here in the train set (good idea?)\n* Adding those that aren't in the test set and filling values with false (refer to the sjv citation above)"},{"metadata":{"trusted":true,"_uuid":"d601b2d887ed7247019631338613f22e0265f39d"},"cell_type":"code","source":"promo_2017_test = promo_2017_test.reindex(promo_2017_train.index).fillna(False)\npromo_2017_test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a639086ca579005659a376a56c0d5a372ba4087d"},"cell_type":"markdown","source":"Concatenating train and test sets and removing them (?)"},{"metadata":{"trusted":true,"_uuid":"bcddb5d06c646712f9de3e81063e85a6b65a0a68"},"cell_type":"code","source":"promo_2017 = pd.concat([promo_2017_train, promo_2017_test], axis=1)\ndel promo_2017_test, promo_2017_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c6b2b1b4ac0056e36848e8e51f29792c1640465"},"cell_type":"markdown","source":"* Setting of the indices like in the training set.\n* Getting rid of promotions since we are exclusively interested in unit_sales here\n* Unstacking the dates\n* Filling the missing information with 0's."},{"metadata":{"trusted":true,"_uuid":"1e1750657c9a2598f09791a0c33e8053445be198"},"cell_type":"code","source":"df_2017 = df_2017.set_index(\n    [\"store_nbr\", \"item_nbr\", \"date\"])[[\"unit_sales\"]].unstack(\n        level=-1).fillna(0)\ndf_2017.columns = df_2017.columns.get_level_values(1)\ndf_2017.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c4a29db6eed6e51066db89908f80c08dd4d1cca7"},"cell_type":"markdown","source":"# Grouping sells and promotions by items through time"},{"metadata":{"_uuid":"bad1392c9afc3c819e99c8286e8b0a1efe56bebf"},"cell_type":"markdown","source":"Remove stores and items that are not in the 2017 set"},{"metadata":{"trusted":true,"_uuid":"83a04e0899c18c00766fb1ebf885c9a7724893f0"},"cell_type":"code","source":"items = items.reindex(df_2017.index.get_level_values(1))\nstores = stores.reindex(df_2017.index.get_level_values(0))\nitems.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e697a002f16041591dbf4fadcc51b8af050bb70c"},"cell_type":"markdown","source":"Grouping sales by item no matter the shop it was sold in"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"50ba4208c6dfe7c90b4d111cbb52b1945d64e8e5"},"cell_type":"code","source":"df_2017_item = df_2017.groupby('item_nbr')[df_2017.columns].sum()\ndf_2017_item.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0095c6bda89753d479b76c51245c956c4488e815"},"cell_type":"markdown","source":"Grouping promotions by item no matter the shop it was sold in"},{"metadata":{"trusted":true,"_uuid":"deb695d6501e87f0bd2c3c2a2786287f21ed72ae"},"cell_type":"code","source":"promo_2017_item = promo_2017.groupby('item_nbr')[promo_2017.columns].sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27c342a9d879f0b23f756c9afa0d240d72e741f3"},"cell_type":"markdown","source":"# Grouping sells and promotions by item class and store through time"},{"metadata":{"trusted":true,"_uuid":"fac444cbee06d92df8301e8941c6eebc66034f75"},"cell_type":"code","source":"df_2017_store_class = df_2017.reset_index()\ndf_2017_store_class['class'] = items['class'].values # par quelle sorcellerie est-ce que ça fonctionne ?\ndf_2017_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e83ef47531372db9eb75a00e68820ca6e97b12e"},"cell_type":"code","source":"df_2017_store_class_index = df_2017_store_class[['class', 'store_nbr']]\ndf_2017_store_class = df_2017_store_class.groupby(['class', 'store_nbr'])[df_2017.columns].sum()\ndf_2017_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aee5f11a644c4b45cdafe63bb6c7213c1aaca40"},"cell_type":"code","source":"df_2017_promo_store_class = promo_2017.reset_index()\ndf_2017_promo_store_class['class'] = items['class'].values\ndf_2017_promo_store_class_index = df_2017_promo_store_class[['class', 'store_nbr']]\ndf_2017_promo_store_class = df_2017_promo_store_class.groupby(['class', 'store_nbr'])[promo_2017.columns].sum()\ndf_2017_promo_store_class.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c8923bb6cffd1f1ace880266b445ac85e534108"},"cell_type":"markdown","source":"# Functions"},{"metadata":{"trusted":true,"_uuid":"a243531b20886e9ce104bf3f0aee0caa3d58cd62"},"cell_type":"code","source":"# Selects df columns which correspond to the \"periods\" days after the \"minus\" day before the date \"dt\"\ndef get_timespan(df, dt, minus, periods, freq='D'):\n    return df[pd.date_range(dt - timedelta(days=minus), periods=periods, freq=freq)]\n\ndef prepare_dataset(df, promo_df, t2017, is_train=True, name_prefix=None):\n    # Création de 6 fenêtres. 3 dans le passé et 3 dans le futur\n    # Le nombre de promotions qu'il y a dans le passé et le futur, à différents intervalles de temps\n    X = {\n        \"promo_14_2017\": get_timespan(promo_df, t2017, 14, 14).sum(axis=1).values,\n        \"promo_60_2017\": get_timespan(promo_df, t2017, 60, 60).sum(axis=1).values,\n        \"promo_140_2017\": get_timespan(promo_df, t2017, 140, 140).sum(axis=1).values,\n        \"promo_3_2017_aft\": get_timespan(promo_df, t2017 + timedelta(days=16), 15, 3).sum(axis=1).values,\n        \"promo_7_2017_aft\": get_timespan(promo_df, t2017 + timedelta(days=16), 15, 7).sum(axis=1).values,\n        \"promo_14_2017_aft\": get_timespan(promo_df, t2017 + timedelta(days=16), 15, 14).sum(axis=1).values,\n    }\n    #print(\"ça démarre!!\")\n    \n    # Moyenne d'items vendus avec ou sans promotion, normale ou avec poids exponentiel\n    for i in [3, 7, 14, 30, 60, 140]:\n        # number of items sold i days before t2017 to t2017\n        tmp1 = get_timespan(df, t2017, i, i)\n        # promotions on items i days before t2017 to t2017 (1 if promotion, else 0)\n        tmp2 = (get_timespan(promo_df, t2017, i, i) > 0) * 1\n        \n        # mean of items sold on promotion in [t2017 - 1; t2017]\n        X['has_promo_mean_%s' % i] = (tmp1 * tmp2.replace(0, np.nan)).mean(axis=1).values\n        # same but exponentially weighted\n        X['has_promo_mean_%s_decay' % i] = (tmp1 * tmp2.replace(0, np.nan) * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n        \n        # mean of items sold without promotion in [t2017 - 1; t2017]\n        X['no_promo_mean_%s' % i] = (tmp1 * (1 - tmp2).replace(0, np.nan)).mean(axis=1).values\n        # same but exponentially weighted\n        X['no_promo_mean_%s_decay' % i] = (tmp1 * (1 - tmp2).replace(0, np.nan) * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n    \n    #print(\"une étape\")\n    # stats d'items vendus\n    for i in [3, 7, 14, 30, 60, 140]:\n        tmp = get_timespan(df, t2017, i, i)\n        # mean of variation of units sold\n        X['diff_%s_mean' % i] = tmp.diff(axis=1).mean(axis=1).values\n        # sum of exponentially weighted units sold\n        X['mean_%s_decay' % i] = (tmp * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n        # stats of units sold\n        X['mean_%s' % i] = tmp.mean(axis=1).values\n        X['median_%s' % i] = tmp.median(axis=1).values\n        X['min_%s' % i] = tmp.min(axis=1).values\n        X['max_%s' % i] = tmp.max(axis=1).values\n        X['std_%s' % i] = tmp.std(axis=1).values\n    \n    #print(\"2 étapes\")\n    # Memes stats avec un décallage d'une semaine dans le passé\n    for i in [3, 7, 14, 30, 60, 140]:\n        tmp = get_timespan(df, t2017 + timedelta(days=-7), i, i)\n        X['diff_%s_mean_2' % i] = tmp.diff(axis=1).mean(axis=1).values\n        X['mean_%s_decay_2' % i] = (tmp * np.power(0.9, np.arange(i)[::-1])).sum(axis=1).values\n        X['mean_%s_2' % i] = tmp.mean(axis=1).values\n        X['median_%s_2' % i] = tmp.median(axis=1).values\n        X['min_%s_2' % i] = tmp.min(axis=1).values\n        X['max_%s_2' % i] = tmp.max(axis=1).values\n        X['std_%s_2' % i] = tmp.std(axis=1).values\n    \n    #print(\"3 étapes\")\n    # Nombre de jours où a eu lieu une vente/promotion dans la fenêtre temporelle, et jours écoulés depuis première/dernière vente/promotion\n    for i in [7, 14, 30, 60, 140]:\n        tmp = get_timespan(df, t2017, i, i)\n        # Number of days a sale has been made\n        X['has_sales_days_in_last_%s' % i] = (tmp > 0).sum(axis=1).values\n        # Number of days since last sales in period\n        X['last_has_sales_day_in_last_%s' % i] = i - ((tmp > 0) * np.arange(i)).max(axis=1).values\n        # Number of days since first sales in period\n        X['first_has_sales_day_in_last_%s' % i] = ((tmp > 0) * np.arange(i, 0, -1)).max(axis=1).values\n\n        tmp = get_timespan(promo_df, t2017, i, i)\n        # Number of days where there was a promotion\n        X['has_promo_days_in_last_%s' % i] = (tmp > 0).sum(axis=1).values\n        # Number of days since there was a promotion in period\n        X['last_has_promo_day_in_last_%s' % i] = i - ((tmp > 0) * np.arange(i)).max(axis=1).values\n        # Number of days since first promotion in period\n        X['first_has_promo_day_in_last_%s' % i] = ((tmp > 0) * np.arange(i, 0, -1)).max(axis=1).values\n    \n    #print(\"4 étapes\")\n    # Nombre de promotions dans les deux semaines à venir, temps avant première et dernière promotion dans la même fenêtre de temps\n    tmp = get_timespan(promo_df, t2017 + timedelta(days=16), 15, 15)\n    X['has_promo_days_in_after_15_days'] = (tmp > 0).sum(axis=1).values\n    X['last_has_promo_day_in_after_15_days'] = i - ((tmp > 0) * np.arange(15)).max(axis=1).values\n    X['first_has_promo_day_in_after_15_days'] = ((tmp > 0) * np.arange(15, 0, -1)).max(axis=1).values\n    \n    # Nombre de ventes le jour i avant aujourd'hui\n    for i in range(1, 16):\n        X['day_%s_2017' % i] = get_timespan(df, t2017, i, 1).values.ravel()\n    \n    #print(\"Presque fini...\")\n    for i in range(7):\n        # mean of sales every same day of week during the month before today\n        X['mean_4_dow{}_2017'.format(i)] = get_timespan(df, t2017, 28-i, 4, freq='7D').mean(axis=1).values\n        # mean of sales every same day of week during the twenty weeks before today\n        X['mean_20_dow{}_2017'.format(i)] = get_timespan(df, t2017, 140-i, 20, freq='7D').mean(axis=1).values\n    \n    # détecte si il y a eu une promotion i jours avant puis après aujourd'hui\n    for i in range(-16, 16):\n        X[\"promo_{}\".format(i)] = promo_df[t2017 + timedelta(days=i)].values.astype(np.uint8)\n    X = pd.DataFrame(X)\n\n    if is_train:\n        # Si il y a entraînement, y devient les 16 jours suivants\n        y = df[\n            pd.date_range(t2017, periods=16)\n        ].values\n        return X, y\n    if name_prefix is not None:\n        X.columns = ['%s_%s' % (name_prefix, c) for c in X.columns]\n    return X\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"39e85b6cd55332a63f89bbde589d6753b8257358"},"cell_type":"code","source":"print(\"Preparing dataset...\")\nt2017 = date(2017, 6, 14)\nnum_days = 6\nX_l, y_l = [], []\nfor i in range(num_days):\n    print('-'*50)\n    print(\"ROUND {} / {}:\".format(i+1, num_days))\n    # décallage d'une semaine à chaque fois (pendant 6 semaines)\n    delta = timedelta(days=7 * i)\n    # Préparation du dataset en distinguant les boutiques (prend du temps!!!)\n    # stats par item par boutique\n    print(\"1/3\")\n    X_tmp, y_tmp = prepare_dataset(df_2017, promo_2017, t2017 + delta)\n    \n    # Préparation du dataset sans distinguer les boutiques (prend peu de temps)\n    # stats générales des items\n    print(\"2/3\")\n    X_tmp2 = prepare_dataset(df_2017_item, promo_2017_item, t2017 + delta, is_train=False, name_prefix='item')\n    X_tmp2.index = df_2017_item.index\n    X_tmp2 = X_tmp2.reindex(df_2017.index.get_level_values(1)).reset_index(drop=True)\n    \n    # stats par type d'objet par boutique\n    print(\"3/3\")\n    X_tmp3 = prepare_dataset(df_2017_store_class, df_2017_promo_store_class, t2017 + delta, is_train=False, name_prefix='store_class')\n    X_tmp3.index = df_2017_store_class.index\n    X_tmp3 = X_tmp3.reindex(df_2017_store_class_index).reset_index(drop=True)\n    \n    #concaténation horizontale des trois X\n    X_tmp = pd.concat([X_tmp, X_tmp2, X_tmp3, items.reset_index(), stores.reset_index()], axis=1)\n    X_l.append(X_tmp)\n    y_l.append(y_tmp)\n    \n    del X_tmp\n    del X_tmp2\n    del X_tmp3\n    del y_tmp\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf59ade0829c590cfc97255cf0478904c37ed458"},"cell_type":"code","source":"X_train = pd.concat(X_l, axis=0)\ny_train = np.concatenate(y_l, axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92fbca7fb37a5432a97e9b7cd11f33f7a221e3ac"},"cell_type":"code","source":"del X_l, y_l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a79f92c18fc6fb475b0daeab1c65a4aa2d102fd1"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2a15ab2eb44d283678bb5a2ae8d6fd60a88426f"},"cell_type":"code","source":"X_val, y_val = prepare_dataset(df_2017, promo_2017, date(2017, 7, 26))\n\nX_val2 = prepare_dataset(df_2017_item, promo_2017_item, date(2017, 7, 26), is_train=False, name_prefix='item')\nX_val2.index = df_2017_item.index\nX_val2 = X_val2.reindex(df_2017.index.get_level_values(1)).reset_index(drop=True)\n\nX_val3 = prepare_dataset(df_2017_store_class, df_2017_promo_store_class, date(2017, 7, 26), is_train=False, name_prefix='store_class')\nX_val3.index = df_2017_store_class.index\nX_val3 = X_val3.reindex(df_2017_store_class_index).reset_index(drop=True)\n\nX_val = pd.concat([X_val, X_val2, X_val3, items.reset_index(), stores.reset_index()], axis=1)\n\nX_test = prepare_dataset(df_2017, promo_2017, date(2017, 8, 16), is_train=False)\n\nX_test2 = prepare_dataset(df_2017_item, promo_2017_item, date(2017, 8, 16), is_train=False, name_prefix='item')\nX_test2.index = df_2017_item.index\nX_test2 = X_test2.reindex(df_2017.index.get_level_values(1)).reset_index(drop=True)\n\nX_test3 = prepare_dataset(df_2017_store_class, df_2017_promo_store_class, date(2017, 8, 16), is_train=False, name_prefix='store_class')\nX_test3.index = df_2017_store_class.index\nX_test3 = X_test3.reindex(df_2017_store_class_index).reset_index(drop=True)\n\nX_test = pd.concat([X_test, X_test2, X_test3, items.reset_index(), stores.reset_index()], axis=1)\n\ndel X_test2, X_val2, df_2017_item, promo_2017_item, df_2017_store_class, df_2017_promo_store_class, df_2017_store_class_index\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7faf6bbd025be60b568dc588b312a57f5b845c4d"},"cell_type":"code","source":"cate_vars = ['family', 'perishable', 'city', 'state', 'type', 'cluster']\nX_val.columns[(X_val.dtypes != 'int16') & (X_val.dtypes != 'int32') & (X_val.dtypes != 'int64') & (X_val.dtypes != 'int8') & (X_val.dtypes != np.float) ]\n#X_train['promo_-16'].dtypes\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(\"Training and predicting models...\")\nparams = {\n    'num_leaves': 80,\n    'objective': 'regression',\n    'min_data_in_leaf': 200,\n    'learning_rate': 0.02,\n    'feature_fraction': 0.8,\n    'bagging_fraction': 0.7,\n    'bagging_freq': 1,\n    'metric': 'l2',\n    'num_threads': 16\n}\n\nMAX_ROUNDS = 5000\nval_pred = []\ntest_pred = []\ncate_vars = []\nfor i in range(16):\n    print(\"=\" * 50)\n    print(\"Step %d\" % (i+1))\n    print(\"=\" * 50)\n    dtrain = lgb.Dataset(\n        X_train, label=y_train[:, i],\n        categorical_feature=cate_vars,\n        weight=pd.concat([items[\"perishable\"]] * num_days) * 0.25 + 1\n    )\n    dval = lgb.Dataset(\n        X_val, label=y_val[:, i], reference=dtrain,\n        weight=items[\"perishable\"] * 0.25 + 1,\n        categorical_feature=cate_vars)\n    bst = lgb.train(\n        params, dtrain, num_boost_round=MAX_ROUNDS,\n        valid_sets=[dtrain, dval], early_stopping_rounds=125, verbose_eval=50\n    )\n    print(\"\\n\".join((\"%s: %.2f\" % x) for x in sorted(\n        zip(X_train.columns, bst.feature_importance(\"gain\")),\n        key=lambda x: x[1], reverse=True\n    )))\n    val_pred.append(bst.predict(\n        X_val, num_iteration=bst.best_iteration or MAX_ROUNDS))\n    test_pred.append(bst.predict(\n        X_test, num_iteration=bst.best_iteration or MAX_ROUNDS))\n\nprint(\"Validation mse:\", mean_squared_error(\n    y_val, np.array(val_pred).transpose()))\n\nweight = items[\"perishable\"] * 0.25 + 1\nerr = (y_val - np.array(val_pred).transpose())**2\nerr = err.sum(axis=1) * weight\nerr = np.sqrt(err.sum() / weight.sum() / 16)\nprint('nwrmsle = {}'.format(err))\n\ny_val = np.array(val_pred).transpose()\ndf_preds = pd.DataFrame(\n    y_val, index=df_2017.index,\n    columns=pd.date_range(\"2017-07-26\", periods=16)\n).stack().to_frame(\"unit_sales\")\ndf_preds.index.set_names([\"store_nbr\", \"item_nbr\", \"date\"], inplace=True)\ndf_preds[\"unit_sales\"] = np.clip(np.expm1(df_preds[\"unit_sales\"]), 0, 1000)\ndf_preds.reset_index().to_csv('lgb_cv.csv', index=False)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7955b54984d81d679eac4522cdd817a8ed3caad"},"cell_type":"code","source":"print(\"Making submission...\")\ny_test = np.array(test_pred).transpose()\ndf_preds = pd.DataFrame(\n    y_test, index=df_2017.index,\n    columns=pd.date_range(\"2017-08-16\", periods=16)\n).stack().to_frame(\"unit_sales\")\ndf_preds.index.set_names([\"store_nbr\", \"item_nbr\", \"date\"], inplace=True)\n\nsubmission = df_test[[\"id\"]].join(df_preds, how=\"left\").fillna(0)\nsubmission[\"unit_sales\"] = np.clip(np.expm1(submission[\"unit_sales\"]), 0, 1000)\nsubmission.to_csv('lgb_sub.csv', float_format='%.4f', index=None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}