{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np\nimport pandas as pd\nimport os\nimport matplotlib.pyplot as plt\nfrom scipy.stats import gaussian_kde\nimport time\nimport seaborn as sns \n%matplotlib inline\npal = sns.color_palette()\nftypes_train = dict(row_id=np.int32, x=np.float32, y=np.float32, accuracy=np.int32, time=np.int32, place_id=np.int64)\nftypes_test = dict(row_id=np.int32, x=np.float32, y=np.float32, accuracy=np.int32, time=np.int32)\ndf_train = pd.read_csv(\"../input/train.csv\", dtype=ftypes_train)\ndf_test = pd.read_csv(\"../input/test.csv\", dtype=ftypes_test)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Add some time fields\ndf_train[\"hour\"]       = (df_train[\"time\"]%(60*24))//60.\ndf_train[\"dayofweek\"]  = np.ceil((df_train[\"time\"]%(60*24*7))//(60.*24))\ndf_train[\"day\"]  = np.ceil((df_train[\"time\"]/(60*24)))\ndf_train[\"week\"] = np.ceil((df_train[\"time\"]/(60*24*7)))\ndf_test[\"hour\"]        = (df_test[\"time\"]%(60*24))//60.\ndf_test[\"dayofweek\"]   = np.ceil((df_test[\"time\"]%(60*24*7))//(60.*24))\ndf_test[\"day\"]   = np.ceil((df_test[\"time\"]/(60*24)))\ndf_test[\"week\"]  = np.ceil((df_test[\"time\"]/(60*24*7)))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "It's been reported elsewhere that accuracy varies as a function of time, let's quickly plot the time series, averaged in weeks."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Group by week and get average/median/std dev\ndf_train_wkaccuracy = df_train.groupby(\"week\").agg({\"accuracy\":[np.mean, np.median, np.std]}).reset_index()\ndf_test_wkaccuracy = df_test.groupby(\"week\").agg({\"accuracy\":[np.mean, np.median, np.std]}).reset_index()\ndf_train_wkaccuracy.columns = [\"week\", \"acc_mean\", \"acc_median\", \"acc_std\"]\ndf_test_wkaccuracy.columns = [\"week\", \"acc_mean\", \"acc_median\", \"acc_std\"]\ndf_test_wkaccuracy.head()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Plots\nx1 = df_train_wkaccuracy[\"week\"]\ny1a = df_train_wkaccuracy[\"acc_mean\"]\ny1b = df_train_wkaccuracy[\"acc_median\"]\ny1c = df_train_wkaccuracy[\"acc_std\"]\n\nx2 = df_test_wkaccuracy[\"week\"]\ny2a = df_test_wkaccuracy[\"acc_mean\"]\ny2b = df_test_wkaccuracy[\"acc_median\"]\ny2c = df_test_wkaccuracy[\"acc_std\"]\n\nplt.figure(0, figsize=(12,12))\n\nplt.subplot(311)\nplt.plot(x1, y1a, c=pal[0], lw=3, label=\"Train\")\nplt.plot(x2, y2a, c=pal[1], lw=3, label=\"Test\")\nplt.ylabel(\"Average accuracy\")\nplt.legend(loc=2)\n\nplt.subplot(312)\nplt.plot(x1, y1b, c=pal[0], lw=3, label=\"Train\")\nplt.plot(x2, y2b, c=pal[1], lw=3, label=\"Test\")\nplt.ylabel(\"Median accuracy\")\nplt.legend(loc=2)\nplt.ylim(56,66)\n\nplt.subplot(313)\nplt.plot(x1, y1c, c=pal[0], lw=3, label=\"Train\")\nplt.plot(x2, y2c, c=pal[1], lw=3, label=\"Test\")\nplt.xlabel(\"Week number\")\nplt.ylabel(\"Std accuracy\")\nplt.legend(loc=2)\n\nplt.tight_layout()\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "It's very interesting that after ~50 weeks the median accuracy flatlines at 64, with a small blip at 65, then back to 64 for the test data.\n\nWe can also examine the distribution of accuracy closer, and find some intersting things"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "plt.figure(0, figsize=(12,8))\n\nplt.subplot(211)\nplt.hist(df_train[\"accuracy\"], bins=250, range=[0,250], color=pal[0], label=\"Train\")\nplt.ylabel(\"Count\")\nplt.title(\"Accuracy distribution\")\nplt.legend()\n\nplt.subplot(212)\nplt.hist(df_test[\"accuracy\"].values, bins=250, range=[0,250], color=pal[1], label=\"Test\")\nplt.xlabel(\"Accuracy\")\nplt.ylabel(\"Count\")\nplt.legend()\n\nplt.tight_layout()\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "There's three main peaks, as well as 5 weird peaky outliers, let's find out what values they are at:"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# The peaks are at very discrete accuracy levels\ncounts, bins = np.histogram(df_train[\"accuracy\"], bins=np.arange(0.5,251.5,1), range=[1,250])\nbinsc = bins[:-1] + np.diff(bins)/2.\ni1 = np.where(counts==counts[0:7].max())[0][0]\ni2 = np.where(counts==counts[7:15].max())[0][0]\ni3 = np.where(counts==counts[25:50].max())[0][0]\ni4 = np.where(counts==counts[50:100].max())[0][0]\ni5 = np.where(counts==counts[150:200].max())[0][0]\na1, c1 = binsc[i1], counts[i1]\na2, c2 = binsc[i2], counts[i2]\na3, c3 = binsc[i3], counts[i3]\na4, c4 = binsc[i4], counts[i4]\na5, c5 = binsc[i5], counts[i5]\nprint (\"Peaks at:\", a1, a2, a3, a4, a5)\nprint (\"Counts are:\", c1, c2, c3, c4, c5)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "We can define these regions and peaks, so we can investigate them independently to see if they behave differently in any ways.\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "plt.figure(0, figsize=(12,4))\nplt.hist(df_train[\"accuracy\"].values, bins=250, range=[0,250])\nplt.axvline(x=40, c=pal[2], ls='--')\nplt.axvline(x=125, c=pal[2], ls='--')\nplt.axvline(x=200, c=pal[2], ls='--')\nplt.text(10, 1250000, \"region 1\", color=pal[2], size=18)\nplt.text(75, 1250000, \"region 2\", color=pal[2], size=18)\nplt.text(155, 1250000, \"region 3\", color=pal[2], size=18)\nplt.text(215, 1250000, \"region 4\", color=pal[2], size=18)\nplt.text(a1-2, 500000, \"p1\", color=pal[3], size=15)\nplt.text(a2-2, 610000, \"p2\", color=pal[3], size=15)\nplt.text(a3-2, 210000, \"p3\", color=pal[3], size=15)\nplt.text(a4-2, 1250000, \"p4\", color=pal[3], size=15)\nplt.text(a5-2, 300000, \"p5\", color=pal[3], size=15)\nplt.xlabel(\"Accuracy\")\nplt.ylabel(\"Count\")\nplt.title(\"Accuracy distribution and splits\")\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "The question is, do these different region/peaks correlate with anything else?\n\nLet's split the train set up into various subsets representing these regions/peaks, and see if the distribution of other variables change."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Split the accuracy up into different components, and check some distributions of other things\n# Groups are the main distribution regions, and separately the weird maxima\nacc_r1 = df_train[(df_train[\"accuracy\"]>=0) & (df_train[\"accuracy\"]<40)]\nacc_r2 = df_train[(df_train[\"accuracy\"]>=40) & (df_train[\"accuracy\"]<125)]\nacc_r3 = df_train[(df_train[\"accuracy\"]>=125) & (df_train[\"accuracy\"]<200)]\nacc_r4 = df_train[(df_train[\"accuracy\"]>=200)]\nacc_p1 = df_train[(df_train[\"accuracy\"]==a1)]\nacc_p2 = df_train[(df_train[\"accuracy\"]==a2)]\nacc_p3 = df_train[(df_train[\"accuracy\"]==a3)]\nacc_p4 = df_train[(df_train[\"accuracy\"]==a4)]\nacc_p5 = df_train[(df_train[\"accuracy\"]==a5)]\nacc = [acc_r1, acc_r2, acc_r3, acc_r4, acc_p1, acc_p2, acc_p3, acc_p4, acc_p5]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "plt.figure(0, figsize=(16,14))\nfor i in range(len(acc)):\n    pd_acc = acc[i]\n    plt.subplot(9, 5, (i*5)+1)\n    plt.hist(pd_acc[\"x\"].values, bins=50)\n    plt.xlabel(\"x\")\n    plt.gca().get_yaxis().set_ticks([]) \n    \n    plt.subplot(9, 5, (i*5)+2)\n    plt.hist(pd_acc[\"y\"].values, bins=50)\n    plt.xlabel(\"y\")\n    plt.gca().get_yaxis().set_ticks([]) \n\n    plt.subplot(9, 5, (i*5)+3)\n    plt.hist(pd_acc[\"time\"].values, bins=50)\n    plt.xlabel(\"time\")\n    plt.gca().get_xaxis().set_ticks([]) \n    plt.gca().get_yaxis().set_ticks([]) \n\n    plt.subplot(9, 5, (i*5)+4)\n    plt.hist(pd_acc[\"hour\"].values, bins=24)\n    plt.xlabel(\"hour\")\n    plt.gca().get_yaxis().set_ticks([]) \n\n    plt.subplot(9, 5, (i*5)+5)\n    plt.hist(pd_acc[\"dayofweek\"].values, bins=7)\n    plt.xlabel(\"dayofweek\")\n    plt.gca().get_yaxis().set_ticks([]) \n    \nplt.tight_layout()\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "Everything remains constant _except_ non-cyclic time.\n\nParticularly interesting is that peaks 1 and 2 (the 5th and 6th rows) don't appear much at the end, but peak 3 (7th row) is the reverse.\n\nLet's take raw time and split it up into various even parts, and see how the _accuracy_ distribution varies.\n\nFor this, it might be nicer to plot density plots, and compare them against the overall distribution (represented in the following charts as the light grey shaded area).\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Sample the train set for the next part\ndf_train_sample = df_train.sample(n=1000000)\n\n# Check accuracy profiles as a function of time\n# Split the dataset up into 8 equal parts of 100,000 minutes (~2 months)\nt1 = df_train_sample[(df_train_sample[\"time\"]>=0)      & (df_train_sample[\"time\"]<100000)]\nt2 = df_train_sample[(df_train_sample[\"time\"]>=100000) & (df_train_sample[\"time\"]<200000)]\nt3 = df_train_sample[(df_train_sample[\"time\"]>=200000) & (df_train_sample[\"time\"]<300000)]\nt4 = df_train_sample[(df_train_sample[\"time\"]>=300000) & (df_train_sample[\"time\"]<400000)]\nt5 = df_train_sample[(df_train_sample[\"time\"]>=400000) & (df_train_sample[\"time\"]<500000)]\nt6 = df_train_sample[(df_train_sample[\"time\"]>=500000) & (df_train_sample[\"time\"]<600000)]\nt7 = df_train_sample[(df_train_sample[\"time\"]>=600000) & (df_train_sample[\"time\"]<700000)]\nt8 = df_train_sample[(df_train_sample[\"time\"]>=700000) & (df_train_sample[\"time\"]<800000)]\ntimes = [t1,t2,t3,t4,t5,t6,t7,t8]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Produce density estimates\nkde_t_overall = gaussian_kde(df_train_sample[\"accuracy\"].values)\nkdes = []\nfor t in times:\n    kdes.append(gaussian_kde(t[\"accuracy\"].values))\n\nrangeX = np.linspace(0, 250, 100)\ny_overall = kde_t_overall(rangeX)\nys = []\nfor k in kdes:\n    ys.append(k(rangeX))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# Plot them\nplt.figure(0, figsize=(12,12))\nfor i in range(8):\n    plt.subplot(4,2,i+1)\n    # Overall accuracy distribution\n    plt.plot(rangeX, y_overall, color='k', alpha=0.1)\n    plt.gca().fill_between(rangeX, 0, y_overall, facecolor='k', alpha=0.1)\n    \n    # Time period N distribution\n    plt.plot(rangeX, ys[i], color=pal[0], alpha=0.5)\n    plt.gca().fill_between(rangeX, 0, ys[i], facecolor=pal[0], alpha=0.5)\n    \n    plt.title(\"Time period \" + str(i))\n    plt.ylabel(\"Density\")\n    plt.xlabel(\"Accuracy\")\n    plt.gca().get_yaxis().set_ticks([])  \n    \nplt.tight_layout()\nplt.show()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "This aligns with the general statement \"accuracy increases with time\", but actually shows how the _distribution_ changes with time. You see the biggest change is that as time progresses, region 1 (as defined above) reduces in size and the values move upward.\n\nWhat does this mean though? Knowing accuracy distribution as a function of time (in this form) is useless for predictions..."
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}