{"cells": [{"cell_type": "code", "source": ["import numpy as np\n", "import pandas as pd\n", "from sklearn import preprocessing, linear_model, metrics\n", "import gc; gc.enable()\n", "import random\n", "import matplotlib.pyplot as plt\n", "from datetime import timedelta\n", "from sklearn.neural_network import MLPRegressor\n", "from sklearn.linear_model import TheilSenRegressor, BayesianRidge\n", "from sklearn.ensemble import BaggingRegressor\n", "from sklearn.tree import DecisionTreeRegressor\n", "from sklearn.metrics import mean_squared_log_error, mean_squared_error\n", "import time\n", "import datetime\n", "import seaborn as sns"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "072abefe553feceba0d3e9c721cf502dbed31902", "_cell_guid": "4a396788-cbea-43d0-a452-5699b88c71f7", "collapsed": true}}, {"cell_type": "markdown", "source": ["**Reduce the population**\n", "The idea is to have in the training population only data for the stores and items present in the test sample."], "metadata": {"_uuid": "7298ab16fa77cbf4aa7ffe3ba2cfab4506e682c0", "_cell_guid": "d51e9f73-9132-448f-b42c-a4f8f3cf1466"}}, {"cell_type": "code", "source": ["\n", "# read datasets\n", "dtypes = {'id':'int64', 'item_nbr':'int32', 'store_nbr':'int8', 'onpromotion':str}\n", "data = {\n", "    'tra': pd.read_csv('../input/train.csv', dtype=dtypes, parse_dates=['date']),\n", "    'tes': pd.read_csv('../input/test.csv', dtype=dtypes, parse_dates=['date']),\n", "    'ite': pd.read_csv('../input/items.csv'),\n", "    'sto': pd.read_csv('../input/stores.csv'),\n", "    'trn': pd.read_csv('../input/transactions.csv', parse_dates=['date']),\n", "    'hol': pd.read_csv('../input/holidays_events.csv', dtype={'transferred':str}, parse_dates=['date']),\n", "    'oil': pd.read_csv('../input/oil.csv', parse_dates=['date']),\n", "    }\n", "\n", "\n", "# Sample down\n", "obj_store = data['tes']['store_nbr'].unique()\n", "obj_item = data['tes']['item_nbr'].unique()\n", "train = data['tra'][(data['tra']['date'].dt.year >= 2017) & (data['tra']['date'].dt.month >= 3) & (data['tra']['store_nbr'].isin(obj_store)) & (data['tra']['item_nbr'].isin(obj_item))]\n", "\n", "gc.collect()\n", "train.head()"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "bae61a29f65b672821b20c12820403fe9f8f4a1e", "_cell_guid": "c2db4c10-7c6c-448c-8e3f-a813cc41583e", "collapsed": true}}, {"cell_type": "markdown", "source": ["Lets bring the past information for the test dataset. The idea is to do it in a consistent way so we can do it at any date we chose.\n", "We go back in a weekly basis starting three weeks back (21 days)."], "metadata": {"_uuid": "ecb8c8902f867b6049a658f920be61d0529002fb", "_cell_guid": "d6ed16b3-370a-44cf-b766-8c6bdccb693a"}}, {"cell_type": "code", "source": ["import warnings\n", "warnings.filterwarnings(\"ignore\")\n", "ini_date = pd.to_datetime('2017-08-16')\n", "nw = 8\n", "ts = data['tes'][['date','item_nbr', 'store_nbr']]\n", "ts['unit_sales'] = np.nan\n", "\n", "def back_prop(df1, df2, date):\n", "    dd = str(len(pd.date_range(start=date,end=ini_date, freq='D'))-1)\n", "    df3 =  df2.loc[df2['date'].isin([date + datetime.timedelta(days=x) for x in range(1, 18)]), ['date', 'item_nbr','store_nbr', 'unit_sales']]\n", "    df3['date']= df3['date'] + datetime.timedelta(days=int(dd)-1)  \n", "    df = pd.merge(df1, df3, \n", "    on = ['item_nbr','store_nbr','date'],\n", "    suffixes = ('', dd),\n", "    how= 'left').fillna(0)\n", "    gc.collect()\n", "    return df\n", "\n", "for dt in [ini_date - datetime.timedelta(days=x) for x in range(7*3, 7*nw, 7)]:\n", "    ts = back_prop(ts, train, dt)\n", "gc.collect()\n", "ts.head()"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "c14023bbac379ec66c9aa9a6ddff2f3a9226c240", "_cell_guid": "e0f5214e-aae5-41d7-8221-fd8fdb9f0dd6", "collapsed": true}}, {"cell_type": "markdown", "source": ["**Training Data**\n", "I selected the third Wed of July 2017. This will take implicitely the day of the week effect into account."], "metadata": {"_uuid": "bfb5dc7bf0d5e2ba0299130208be975638a45027", "_cell_guid": "1d9da7d9-7b7e-4b97-bad4-a8268de508b9"}}, {"cell_type": "code", "source": ["import warnings\n", "warnings.filterwarnings(\"ignore\")\n", "ini_date = pd.to_datetime('2017-07-19')\n", "tr = data['tes'][['date','item_nbr', 'store_nbr']]\n", "dif = len(pd.date_range(start=ini_date, end=tr['date'].min(), freq='D'))\n", "tr['date'] = tr['date'] - datetime.timedelta(days=dif-1) \n", "\n", "\n", "def back_prop(df1, df2, date):\n", "    dd = str(len(pd.date_range(start=date,end=ini_date, freq='D'))-1)\n", "    df3 =  df2.loc[df2['date'].isin([date + datetime.timedelta(days=x) for x in range(1, 18)]), ['date', 'item_nbr','store_nbr', 'unit_sales']]\n", "    df3['date']= df3['date'] + datetime.timedelta(days=int(dd)-2)  \n", "    df = pd.merge(df1, df3, \n", "    on = ['date','item_nbr','store_nbr'],\n", "    suffixes = ('', dd),\n", "    how= 'left').fillna(0)\n", "    gc.collect()\n", "    return df\n", "\n", "tr = back_prop(tr, train, ini_date)\n", "for dt in [ini_date - datetime.timedelta(days=x) for x in range(7*3, 7*nw, 7)]:\n", "    tr = back_prop(tr, train, dt)\n", "tr['unit_sales'].mean()\n", "gc.collect()\n", "tr.head()"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "972ed07123c41fc46427b5785820c9d1fb0a70b5", "_cell_guid": "b51a519c-9706-4a08-a0b8-15289f79fd6c", "collapsed": true}}, {"cell_type": "markdown", "source": ["A quick glance of the data."], "metadata": {"_uuid": "e103280710ec98ea66f31b074cfded3330424765", "_cell_guid": "3db7ff9b-0d66-4085-b286-6fc10a3b4271"}}, {"cell_type": "code", "source": ["data_plot = tr[tr['item_nbr'].isin(['105575','105857','112830','116017','116018'])]\n", "data_plot = data_plot[['item_nbr', 'unit_sales', 'unit_sales21', 'unit_sales28', 'unit_sales35', 'unit_sales42']].set_index('item_nbr').stack().reset_index()\n", "data_plot.columns = ['item','lag','sales']\n", "sns.factorplot(x=\"lag\", y=\"sales\", hue= 'item', data = data_plot)\n", "plt.show()"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "fb09eb6b5fec07ea8305edb76ccd392deb047bf5", "_cell_guid": "b84d2746-8cf9-4c7b-a915-f896badb39bf", "collapsed": true}}, {"cell_type": "markdown", "source": ["**Optimal weights**\n", "A simple linear regression without intercept will do."], "metadata": {"_uuid": "9424cc4861beaaa53c098a1f2c77abd61d41dc78", "_cell_guid": "5b036067-e175-486e-8a0c-f3e836be5562"}}, {"cell_type": "code", "source": ["cols = [c for c in tr if c not in ['item_nbr','store_nbr', 'unit_sales', 'date']]\n", "X_train = tr[cols].clip(lower=0)\n", "y_train = tr.unit_sales\n", "\n", "regr = linear_model.LinearRegression(fit_intercept = False)\n", "regr.fit(X_train, y_train)\n", "plt.plot(regr.coef_, )\n", "y_pred = regr.predict(ts[cols].clip(lower=0))\n", "y_pred[0:10]\n", "y_pred[8740:8760]\n"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "ad55d07fcdd9fedc7b0cfb3e7a033518f9bf1646", "_cell_guid": "e574576f-cd68-4ae6-af53-85be7ea2c6f9", "collapsed": true}}, {"cell_type": "markdown", "source": ["**Submission**"], "metadata": {"_uuid": "8529a6350856b79accfabba82f2e7f2c58d398e9", "_cell_guid": "39fe92a6-720a-4acf-b8aa-ebdeff4d802f"}}, {"cell_type": "code", "source": ["sub = pd.read_csv('../input/sample_submission.csv')\n", "sub['unit_sales'] = y_pred\n", "sub.head(20)"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "b8a2bd52b21d85c94ec6edd00e5a309f1aa4e8bd", "_cell_guid": "e1637783-d016-498c-a308-09ea5eb02e4b", "collapsed": true}}, {"cell_type": "code", "source": ["# model submission\n", "sub.to_csv('subma00.csv', index=False)\n", "print('done')"], "execution_count": null, "outputs": [], "metadata": {"_uuid": "4769f8670866067e805bef814a82dc4c6d927037", "_cell_guid": "d57bfbc9-9b03-411e-b553-b77bbdf82f67", "collapsed": true}}], "nbformat_minor": 1, "nbformat": 4, "metadata": {"language_info": {"version": "3.6.3", "nbconvert_exporter": "python", "file_extension": ".py", "mimetype": "text/x-python", "codemirror_mode": {"name": "ipython", "version": 3}, "pygments_lexer": "ipython3", "name": "python"}, "kernelspec": {"language": "python", "name": "python3", "display_name": "Python 3"}}}