{"nbformat_minor": 1, "metadata": {"kernelspec": {"display_name": "Python 3", "language": "python", "name": "python3"}, "language_info": {"nbconvert_exporter": "python", "codemirror_mode": {"version": 3, "name": "ipython"}, "name": "python", "pygments_lexer": "ipython3", "version": "3.6.3", "mimetype": "text/x-python", "file_extension": ".py"}}, "cells": [{"source": ["import numpy as np\n", "import pandas as pd\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "18719494-3e65-461d-9747-78257e3ca673", "_uuid": "41d021b144226c8aaea236872a9b15a91750e30b"}}, {"source": ["e00 = check_output([\"ls\", \"../input/ensemble\"]).decode(\"utf8\")\n", "e01 = check_output([\"ls\", \"../input/ensemble-grocery-01\"]).decode(\"utf8\")\n", "print(e00)\n", "print(e01)"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "83371c30-b18c-4f5c-ae57-5f68862efecd", "_uuid": "31d971733a6b570ee32241f36569e872237dabc1"}}, {"source": ["mods = e00.split('\\n')\n", "mods.pop()  # Get rid of empty string at end\n", "preds = [ pd.read_csv('../input/ensemble/' + m, index_col=0) for m in mods ]\n", "moremods = e01.split('\\n')\n", "moremods.pop()\n", "preds += [ pd.read_csv('../input/ensemble-grocery-01/' + m, index_col=0) for m in moremods ]\n", "mods += moremods"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {}}, {"source": ["df = pd.DataFrame(index=preds[0].index)\n", "modnames = []\n", "for m,p in zip(mods,preds):\n", "    name = m[:-4]\n", "    if len(name)>20:\n", "        name = name[:13] # Too damn long\n", "    modnames += [name]\n", "    p.columns = [name]\n", "    df = df.join( p )\n", "print( modnames )\n", "df.head()"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {}}, {"source": ["test = pd.read_csv('../input/favorita-grocery-sales-forecasting/test.csv', \n", "                   parse_dates=['date'], index_col=0)\n", "test.head()"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "e68e5074-efeb-40af-af6b-a884314161c7", "_uuid": "5a3b4797953077ad6d4fea88042e24b317decea9"}}, {"source": ["df = test[['date']].join(df)"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "3a6656ac-20d5-42db-bf4a-c065ee7e083d", "_uuid": "af89c54d8804b0b839aabd5a47ee5eec229f772f"}}, {"source": ["# Dates of public test set\n", "public_dates = ['2017-08-16', '2017-08-17', '2017-08-18', '2017-08-19', '2017-08-20']\n", "# Dates of private test set exactly one week after those of public test set\n", "private_public_dows = ['2017-08-23', '2017-08-24', '2017-08-25', '2017-08-26', '2017-08-27']\n", "\n", "# Dates of public test set that don't correspond to holidays in the subsequent week\n", "public_dates_nohol = ['2017-08-16', '2017-08-18', '2017-08-19', '2017-08-20']\n", "# Non-holdiay dates of private test set exactly one week after those of public test set\n", "private_public_dows_nohol = ['2017-08-23', '2017-08-25', '2017-08-26', '2017-08-27']\n", "\n", "# Dates of private test set that occur before end of first 7 days of full test set\n", "early_private_dates = ['2017-08-21', '2017-08-22']\n", "# Dates of private test set exactly one week after those\n", "private_private_dows = ['2017-08-28', '2017-08-29']\n", "\n", "# Last days of private test set\n", "last_private_dates = ['2017-08-30', '2017-08-31']\n", "# Days exactly 2 weeks before the last days\n", "public_last_dows = ['2017-08-16', '2017-08-17']"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "200bf483-3430-44cb-92ed-e85a2abc4cec", "_uuid": "b5ef5277e1ebbe98c2f00bd444164cb92591eed8"}}, {"source": ["def compare_results(results, earlier_dates, later_dates):\n", "    early = results[results.date.isin(earlier_dates)].describe()\n", "    late = results[results.date.isin(later_dates)].describe()\n", "    print( '\\nIf these are dramatically different with no good reason why,')\n", "    print(  '   there might be a problem:\\n')\n", "    print( early.join(late, lsuffix='_early', rsuffix='_late') )"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "09e41539-e68e-4faa-80ea-b5c5d70d3f5b", "_uuid": "086a9d03fbdf3247a3594b62f2f7670608c452ca"}}, {"source": ["for m in modnames:\n", "    print( '\\n\\n\\nModel: ' + m )\n", "    results = df[['date', m]]\n", "    print( '\\n\\nComparing results for public test data with subsequent week...')\n", "    compare_results(results, public_dates, private_public_dows)\n", "    print( '\\n\\nComparing results for public test data with non-holidays in subsequent week...')\n", "    compare_results(results, public_dates_nohol, private_public_dows_nohol)\n", "    print( '\\n\\nComparing results for early private test data with a week later...')\n", "    compare_results(results, early_private_dates, private_private_dows)\n", "    print( '\\n\\nComparing results for end of private data with corresponding days in public data...')\n", "    compare_results(results, public_last_dows, last_private_dates)"], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"_cell_guid": "ca627e88-03ea-4da1-94a7-1a0933253882", "_uuid": "3e52ec3adbcfecf15e2a30046bed8713c5e69ceb"}}, {"source": ["The \"split verification\" results are of course bad by design: private predictions are zero, which was the whole point.  The ETS results really do seem to have a problem.  (Among other things, the means are very different in all the comprisions, and just ridiculous on the last days.)  The rest mostly look OK, I think, but you can judge for yourself."], "cell_type": "markdown", "metadata": {}}, {"source": [], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "adaba4b5-b881-48b8-bb24-3273d865a0c3", "_uuid": "45e774827e7c1c85db037fcfc534feb20622a5af"}}, {"source": [], "cell_type": "code", "outputs": [], "execution_count": null, "metadata": {"collapsed": true, "_cell_guid": "85a59549-0751-4bee-ab64-3a12773ba358", "_uuid": "63b9f16078f4fe2b7d8a3fec9ce373707c2a7505"}}], "nbformat": 4}