{"cells": [{"source": ["### This is a kernel to learn the basics of an end-to-end pipeline for Corparacion Favorita Compeition\n", "#### Source code credit: https://www.kaggle.com/ceshine/mean-baseline-lb-59"], "metadata": {"_cell_guid": "11c9abef-cf43-0665-2143-586625003b3b", "_uuid": "99abd764e0cbaff2d6855b85edabdfed8d4efbd6"}, "cell_type": "markdown"}, {"execution_count": null, "outputs": [], "source": ["import itertools\n", "\n", "import pandas as pd\n", "import numpy as np"], "metadata": {"_cell_guid": "5c5b7178-e8c2-913a-0220-44b52e78d376", "collapsed": true, "_uuid": "9cfdad3296abc70edbbc916e8cbb151a145fe370"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["# usecols: select the columns to include, drop the column named id\n", "# dtype: specify the type of the data for certain series\n", "# converters: can define a lambda function, some unit_sales is negative, so convert that to 0 if negative\n", "# nrows: specify the number of rows to read in\n", "# skiprows: skip over this range of rows\n", "df_train = pd.read_csv(\n", "    '../input/train.csv', usecols=[1, 2, 3, 4, 5], dtype={'onpromotion': str},\n", "    converters={'unit_sales': lambda u: float(u) if float(u) > 0 else 0},\n", "    skiprows=range(1, 124035460)\n", ")\n", "# nobs = 1461580+1"], "metadata": {"_cell_guid": "7c967562-aa64-43c5-8b1f-276fd4528bd3", "collapsed": true, "_uuid": "854de7026fb272e4b999d585484b5a6eb9b96751"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["df_train.tail()"], "metadata": {"_cell_guid": "bb7d0e2d-02ed-482b-a39b-c1e0b3405a4d", "_uuid": "a42644c4f5de9c064bded7a73af96230be0fc43f"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["# log transform\n", "df_train[\"unit_sales\"] = df_train[\"unit_sales\"].apply(np.log1p)"], "metadata": {"_cell_guid": "47193c1f-b9f9-4777-8139-353b18bd6f2d", "collapsed": true, "_uuid": "0d88851caa68b13e6c0da453f2cb8b9909fe8b99"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["# Fill gaps in dates - for example a product has no report in certain store for Jan 2018, but has report\n", "#    for other store in Jan 2018, an entry will be created for that product-store for Jan 2018\n", "#    with value np.nan (NaN)\n", "# Improved with the suggestion from Paulo Pinto\n", "u_dates = df_train.date.unique()\n", "u_stores = df_train.store_nbr.unique()\n", "u_items = df_train.item_nbr.unique()\n", "df_train.set_index([\"date\", \"store_nbr\", \"item_nbr\"], inplace=True)\n", "df_train = df_train.reindex(\n", "    pd.MultiIndex.from_product(\n", "        (u_dates, u_stores, u_items),\n", "        names=[\"date\", \"store_nbr\", \"item_nbr\"]\n", "    )\n", ")"], "metadata": {"_cell_guid": "d6506be9-19f4-4544-91cc-1ab8595501f9", "collapsed": true, "_uuid": "0f85ca81a37311b4f359fd908c978a512dc4326c"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["# now instead of the default row id, the rows are now labeled by date, store_nbr and item_nbr\n", "df_train.head()"], "metadata": {"_cell_guid": "c289c595-29a7-4c0f-99b3-e04e4ae7088a", "_uuid": "b020950ad058e7b5a8b8343a33629df15fa47b2c"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["# Fill NAs\n", "df_train.loc[:, \"unit_sales\"].fillna(0, inplace=True)\n", "# Assume missing entris imply no promotion\n", "df_train.loc[:, \"onpromotion\"].fillna(\"False\", inplace=True)\n", "df_train.head()"], "metadata": {"_cell_guid": "e8bc0aa1-1a15-4e82-8a39-b16d3677b863", "_uuid": "1a6454d7cc294f957f5c1b903c294d47f5b983c7"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["# Calculate means \n", "df_mean = df_train.groupby(\n", "    ['item_nbr', 'store_nbr', 'onpromotion']\n", ")['unit_sales'].mean().to_frame('unit_sales')\n", "df_mean.head()\n", "# we see that now the date has disappeared, the new df only has one column,\n", "# with index specified in the groupby"], "metadata": {"_cell_guid": "dc44c67f-7f35-4cd4-a0c5-151a695f7342", "_uuid": "50b4b9497d41e9175fb411741a930da02dd85f92"}, "cell_type": "code"}, {"execution_count": null, "outputs": [], "source": ["# Inverse transform\n", "df_mean[\"unit_sales\"] = df_mean[\"unit_sales\"].apply(np.expm1)\n", "# this creates a master mean reference table that is date-free, now can use that to\n", "# propagate the submission\n", "\n", "# Create submission\n", "pd.read_csv(\n", "    \"../input/test.csv\", usecols=[0, 2, 3, 4], dtype={'onpromotion': str}\n", ").set_index(\n", "    ['item_nbr', 'store_nbr', 'onpromotion']\n", ").join(\n", "    df_mean, how='left'\n", ").fillna(0).to_csv(\n", "    'mean.csv.gz', float_format='%.2f', index=None, compression=\"gzip\"\n", ")"], "metadata": {}, "cell_type": "code"}], "nbformat_minor": 1, "nbformat": 4, "metadata": {"kernelspec": {"language": "python", "display_name": "Python 3", "name": "python3"}, "language_info": {"codemirror_mode": {"version": 3, "name": "ipython"}, "version": "3.6.3", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "file_extension": ".py", "pygments_lexer": "ipython3"}, "_change_revision": 0, "_is_fork": false}}