{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "4ed55089-3b16-6d61-28e7-586c10057368"
      },
      "source": [
        "## Fluctuating demands\n",
        "I had the idea that demand for an item on a given week should be correlated with the demand for that product the week before. I'm sure many of you have had the same thought.     \n",
        "\n",
        "In this notebook we'll see a couple examples of the change on demand for each item stocked by a client. Then we'll plot the distributions and see the trends."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7f2cefbc-1187-a28c-f930-a4f2f997d441"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "a0a2a634-0f2b-af46-9d45-80e8d87df832"
      },
      "outputs": [],
      "source": [
        "types = {'Semana':np.uint8, 'Cliente_ID':np.uint32,\n",
        "         'Producto_ID':np.uint16, 'Demanda_uni_equil':np.uint32}\n",
        "\n",
        "df = pd.read_csv('../input/train.csv', usecols=types.keys(),\n",
        "                 dtype=types)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "2b497b24-ac02-cd2a-b84d-5277fe62c61e"
      },
      "source": [
        "Before looking at the change in demand, we should check out the distribution of item demands. It's spiked at 0 with a tail extending to 5000."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "089c9996-5193-d987-e3c2-2fa37041f160"
      },
      "outputs": [],
      "source": [
        "# Histogram of target variable\n",
        "df.Demanda_uni_equil.hist(bins=100)\n",
        "plt.xlabel('Demand per week')\n",
        "plt.ylabel('Number of clients');"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "37c438f1-b3db-1f48-e64f-433420c93f37"
      },
      "outputs": [],
      "source": [
        "# Log scale histogram of target variable\n",
        "df.Demanda_uni_equil.hist(bins=100, log=True)\n",
        "plt.xlabel('Demand per week')\n",
        "plt.ylabel('Number of clients');"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "aa3b757c-96f1-08e9-7bd4-2cc96f3b20e1"
      },
      "outputs": [],
      "source": [
        "demand_sorted = df.Demanda_uni_equil.sort_values(ascending=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "52f4febf-83de-f946-0b59-59a40c406ce5"
      },
      "outputs": [],
      "source": [
        "print('plotting {0:.2f} % of data'.\\\n",
        "      format(100*(demand_sorted < 30).sum()/len(demand_sorted)))\n",
        "demand_sorted[demand_sorted < 30].hist(bins=30)\n",
        "plt.xlabel('Demand per week')\n",
        "plt.ylabel('Number of clients');"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "760b4b25-17a9-0994-e2bd-fd713238e737"
      },
      "source": [
        "As promised, lets look at the week-to-week changes in item demand."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7a2d7610-bc76-1c3c-2988-93e25f07681f"
      },
      "outputs": [],
      "source": [
        "def demandVar(c_id, df, percent=False):\n",
        "    ''' Get the amounts by which product demand changed\n",
        "    week-to-week for a given set of client ids. Returned\n",
        "    object is a pandas dataframe with NaN entries where\n",
        "    the product was not ordered the week before or after.\n",
        "    '''\n",
        "    \n",
        "    for week in range(4,10):\n",
        "        try:\n",
        "            \n",
        "            vals_a = df[(df.Cliente_ID.values == c_id)&\\\n",
        "                        (df.Semana.values == week-1)].Demanda_uni_equil.values\n",
        "            prod_a = df[(df.Cliente_ID.values == c_id)&\\\n",
        "                        (df.Semana.values == week-1)].Producto_ID.values\n",
        "            dict_a = {p: v for p, v in zip(prod_a, vals_a)}\n",
        "            \n",
        "            vals_b = df[(df.Cliente_ID.values == c_id)&\\\n",
        "                        (df.Semana.values == (week))].Demanda_uni_equil.values\n",
        "            prod_b = df[(df.Cliente_ID.values == c_id)&\\\n",
        "                        (df.Semana.values == week)].Producto_ID.values\n",
        "            dict_b = {p: v for p, v in zip(prod_b, vals_b)}\n",
        "            \n",
        "            dict_merge = {}\n",
        "            for key in np.unique(np.concatenate((prod_a, prod_b))):\n",
        "                try:\n",
        "                    if percent:\n",
        "                        try:\n",
        "                            # Calculate percent difference\n",
        "                            dict_merge[key] = (dict_b[key].astype(int) - dict_a[key].astype(int))\\\n",
        "                                              /(dict_b[key].astype(int) + dict_a[key].astype(int))\n",
        "                        except:\n",
        "                            # If dividing by zero assign 0% change\n",
        "                            dict_merge[key] = 0.0\n",
        "                    else:\n",
        "                        dict_merge[key] = dict_b[key].astype(int) - dict_a[key].astype(int)\n",
        "                except:\n",
        "                    # The product was not on the previous form\n",
        "                    # or was removed from the current one\n",
        "                    dict_merge[key] = np.nan\n",
        "            \n",
        "            if week==4:\n",
        "                df_return = pd.DataFrame({'week_3-4': list(dict_merge.values())},\n",
        "                                         index=list(dict_merge.keys()))\n",
        "            else:\n",
        "                df_new = pd.DataFrame({'week_'+str(week-1)+'-'+str(week): list(dict_merge.values())}, \n",
        "                                       index=list(dict_merge.keys()))\n",
        "                df_return = pd.merge(df_return, df_new, how='outer', left_index=True, right_index=True)\n",
        "                \n",
        "        except:\n",
        "            print('No week {}-{} data found'.\\\n",
        "                 format(week-1, week))\n",
        "\n",
        "    return df_return"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "ee76246f-38cf-33e3-f0eb-d74bbfdb7371"
      },
      "source": [
        "Generating a set of client id's to look at."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ee7d6923-ffde-c543-2e9d-90964d8272f7"
      },
      "outputs": [],
      "source": [
        "c_ids = [df.Cliente_ID.values[int(i)] for i in np.linspace(0, len(df)-1, 5)]\n",
        "c_ids"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "4c9c8a2c-b4b1-8bdf-e2ef-92ecbf69abf9"
      },
      "source": [
        "Below we see the differences for client id=15766, where the dataframe index is the product id.  NaN entries represent products that were not present on the inventory order sheet for both or one of the compared weeks."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "bae1dc32-ec19-7277-3662-0295bdfb649a"
      },
      "outputs": [],
      "source": [
        "var = demandVar(c_id=c_ids[0], df=df)\n",
        "var"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "2832fe7d-6b53-899c-3652-481030cff103"
      },
      "source": [
        "The `demandVar` function can also calcualte the percent differences in product demand, as seen below for client id=2465653."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "7234616a-7c00-92fb-f85c-438347febe7c"
      },
      "outputs": [],
      "source": [
        "var = demandVar(c_id=c_ids[1], df=df, percent=True)\n",
        "var"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "_cell_guid": "e7d6fe12-e4b0-4f6d-e308-cbf654f685f3"
      },
      "source": [
        "Now we'll plot distributions using larger samples of data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "3943fcc9-4db2-f413-f94a-278a576db8c9"
      },
      "outputs": [],
      "source": [
        "def get_vars(c_ids, percent=False):\n",
        "    ''' Return a list of variations in the demand\n",
        "    week-to-week on individual products for a set of clients. '''\n",
        "    return_list = [[] for _ in range(len(c_ids))]\n",
        "    for i, c_id in enumerate(c_ids):\n",
        "        var = demandVar(c_id, df, percent)\n",
        "        for col in var.columns:\n",
        "            return_list[i] += list(var[col].dropna())\n",
        "        \n",
        "    return return_list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d397f1af-c628-058f-378c-f827f992ffcb"
      },
      "outputs": [],
      "source": [
        "var_list = get_vars(c_ids)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "43419b59-436b-c336-3cf9-7cbeacea332c"
      },
      "outputs": [],
      "source": [
        "colors = ['blue', 'red', 'green', 'turquoise', 'brown']\n",
        "\n",
        "fig, ax = plt.subplots(1,2)\n",
        "plt.suptitle('Change in demand on individual poducts for 5 clients')\n",
        "\n",
        "for i in range(5):\n",
        "    ax[0].hist(var_list[i], color=colors[i],\n",
        "             normed=True, alpha=0.5, bins=20)\n",
        "ax[0].set_ylim(0,0.2)\n",
        "ax[0].set_xlabel('Change in demand')\n",
        "ax[0].set_ylabel('Normed frequency')\n",
        "\n",
        "for i in range(5):\n",
        "    ax[1].hist(var_list[i], color=colors[i],\n",
        "             normed=True, alpha=0.5, bins=20)\n",
        "ax[1].set_ylim(0,1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "ac20fad9-9fd2-5013-518b-488ace3cf31e"
      },
      "outputs": [],
      "source": [
        "var_list = get_vars(c_ids, percent=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "d1e16ccf-1b1f-95dd-1dce-7e494558566b"
      },
      "outputs": [],
      "source": [
        "colors = ['blue', 'red', 'green', 'turquoise', 'brown']\n",
        "\n",
        "fig, ax = plt.subplots(1,2)\n",
        "plt.suptitle('Percent change in demand on individual poducts for 5 clients')\n",
        "\n",
        "for i in range(5):\n",
        "    ax[0].hist(var_list[i], color=colors[i],\n",
        "             normed=True, alpha=0.5, bins=20)\n",
        "ax[0].set_ylim(0,2)\n",
        "ax[0].set_xlabel('Change in demand')\n",
        "ax[0].set_ylabel('Normed frequency')\n",
        "\n",
        "for i in range(5):\n",
        "    ax[1].hist(var_list[i], color=colors[i],\n",
        "               normed=True, alpha=0.5, bins=20)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "1aa63b0a-2941-bd97-115f-e90d0679128f"
      },
      "outputs": [],
      "source": [
        "%%time\n",
        "c_ids = [df.Cliente_ID.values[int(i)] for i in np.linspace(0, len(df)-1, 100)]\n",
        "var_list, p_var_list = get_vars(c_ids), get_vars(c_ids, percent=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "79314886-f208-da16-23c0-e87e143059ec"
      },
      "outputs": [],
      "source": [
        "fig, ax = plt.subplots(1, 2)\n",
        "plt.suptitle('Change in demand on individual poducts for 100 clients')\n",
        "\n",
        "# Plot historgram for flattened list\n",
        "ax[0].hist([x for row in var_list for x in row], color='red',\n",
        "            normed=True, alpha=0.5, bins=100)\n",
        "ax[0].set_xlim(-100,100)\n",
        "ax[0].set_xlabel('Change in demand')\n",
        "ax[0].set_ylabel('Normed frequency')\n",
        "\n",
        "ax[1].hist([x for row in var_list for x in row], color='red',\n",
        "            normed=True, alpha=0.5, bins=500)\n",
        "ax[1].set_xlim(-10,10);"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "75041b4c-2c78-2d5f-9564-72ef2c6b9a9f"
      },
      "outputs": [],
      "source": [
        "fig, ax = plt.subplots()\n",
        "plt.suptitle('Percent change in demand on individual poducts for 100 clients')\n",
        "\n",
        "# Plot historgram for flattened list\n",
        "ax.hist([x for row in p_var_list for x in row], color='red',\n",
        "            normed=True, alpha=0.5, bins=100)\n",
        "# ax[0].set_xlim(-100,100)\n",
        "ax.set_xlabel('Change in demand')\n",
        "ax.set_ylabel('Normed frequency');"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "_cell_guid": "8e82b252-306a-43d3-3ef5-5b784f48021f"
      },
      "outputs": [],
      "source": ""
    }
  ],
  "metadata": {
    "_change_revision": 0,
    "_is_fork": false,
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.5.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}