{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport sqlite3 as sql\n\nimport csv\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.\nimport matplotlib.pyplot as plt\n\ndf_train = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')\n\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#df_train['time'].describe()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train['time']\ntime = time % (24*60)#*60#*60*10\n\nn, bins, patches = plt.hist(time, 50)\nplt.title('What is Time?')\nplt.xlabel('Time')\nplt.ylabel('Frequency')\nplt.grid(True)\nplt.show()\n#So, 1 of 3 things is happening: Nothing happens cyclically with time; time isn't based on hours, minutes, seconds, or sub-seconds; or they've given us data that aggregates to the same use for each hour.\n#Option 3 sounds most promising, so let's dive into that"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#df_train['place_id'].value_counts().head(10) #get the top places to breakout time\noffset=0 # This can be adjusted if we figure out what time midnight is\n#Let's take a look at how each individual place breaks down with time"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==8772469670]['time']\n\ntimeToTest=24*60#*60#*60*10\n\ntime = (time+offset) % timeToTest\n\nn, bins, patches = plt.hist(time, 50)\nplt.title('What is Time?')\nplt.xlabel('Time')\nplt.ylabel('Frequency')\nplt.grid(True)\nplt.show()\ntime = df_train[df_train['place_id']==1623394281]['time']"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# check for cycles over the course of a day (top) and week (bottom)\ndef plotDayAndWeek(time,offset,clumpHours):\n    fig, axs = plt.subplots(2,1);\n    for j in range(0,2):\n        timeToTest=24*60*(j*6+1)\n        timeC = (time+offset) % timeToTest\n        n, bins, patches = axs[j].hist(timeC/60., 24 / clumpHours * (j*6+1));\n        plt.xlabel('Time (Hours)');\n        plt.ylabel('Frequency');\n        fig.show();"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==8772469670]['time']\nclumpHours = 1  # this could be used to reduce the number of histogram bins\n# later on we'll want to know what are the likely places at a given time\n# in that case, a \"given time\" probably doesn't need minute-by-minute resolution\n# in fact, too much resolution in time would be noisier \nplotDayAndWeek(time,offset,clumpHours)\n# looks like this place is open for the first 12 hours the cycle, so cycle prob starts around 6am?\n# third weekly peak is the biggest... maybe taco tuesday? which would put sunday 6am on left"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==1623394281]['time']\nplotDayAndWeek(time,offset,clumpHours)\n# looks like a nighttime / weekend place, where weekend can mean early morning\n# nighttime might even mean 6 in the morning\n# could be a place that everyone goes to on a weekday night and party all night\n# are there people on this site that do that?!  We might not have the right expertise."
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==1308450003]['time']\nplotDayAndWeek(time,offset,clumpHours)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==4823777529]['time']\nplotDayAndWeek(time,offset,clumpHours)\n# I wouldn't want to work at this place"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#Strong case for this dataset being in minutes.\n# weekly cycles add supporting evidence\n# still don't know what the two big dips in the middle of time range...\n\n#Let's see how much time this data has been collected for\nprint('Time since start of data collection: ' + str(round(df_train['time'].max()/(24*60*365.25),2)) + ' Years.')\n# the long time period also makes sense given a steady overall rise of checkins over time"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "# check for cycles over the course of a year - but 7-day week should still dominate\ndef plotYear(time,offset):\n    timeToTest=24*60*365\n    timeC = (time+offset) % timeToTest\n    n, bins, patches = plt.hist(timeC/(60.*24.*7.), 52*7);\n    plt.xlabel('Time (Weeks)');\n    plt.ylabel('Frequency');\n    plt.show();"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==8772469670]['time']\nplotYear(time,offset)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==1623394281]['time']\nplotYear(time,offset)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==1308450003]['time']\nplotYear(time,offset)\n# have to check if this place actually closed after 35ish weeks into the first year\n# if so, a popular place would account for 0 checkins in the test set.\n# specifically, did checkins come back in the new year, or were they done after 35 weeks?"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==1308450003]['time']\ntimeC = time /(60.*24.*7.)\nplt.plot(np.sort(timeC))\n# the very few-and-far between checkins for this place after 35ish weeks (y-axis) means two things\n# 1) the place closed after 30ish weeks - exclude it from test set predicitons\n# 2) some checkins to a place can be considered erroneous\n#    in the sense that the person didn't actually go to that place\n\n# late-stage tweaking may include a \"closed for business\" detector\"\n# other places might have few checkins, but also be new, so should be scaled higher than the raw count\n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "time = df_train[df_train['place_id']==4823777529]['time']\nplotYear(time,offset)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}