{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "%matplotlib inline\n\nimport matplotlib.pyplot as plt\nimport matplotlib.mlab as mlab\nimport matplotlib.cm as cm\nfrom matplotlib.colors import LogNorm\n\nimport pandas as pd\nimport numpy as np\nimport datetime"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### Load the CSV files specifying which is the datetime column and the parse function\ndf_train = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')\n\n### Convert the timestamp column to a datetime value \ndf_train[\"time\"] = pd.to_datetime(df_train[\"time\"])\ndf_test[\"time\"] = pd.to_datetime(df_train[\"time\"])"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### Let's see some context info about the dataset\n\nprint(\"Size of training data: %d\" % df_train.shape[0])\nprint(\"Size of testing data: %d\" % df_test.shape[0])\nprint(\"Number of different places: %d\" % len(df_train['place_id'].unique()))\nprint(\"\\nDataset Description:\")\nprint(df_train.describe())\nprint(\"\\nInfo about the time period:\\n\" + str(df_train['time'].describe()))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### I add some extra columns to expand the model with time related fields.\n### This will be useful to plot some charts to understand the data.\ndf_train['hour'] = df_train['time'].dt.hour\ndf_train['weekday'] = df_train['time'].dt.dayofweek"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### Let's visualize which days of the week are the most \"active\"\nn, bins, patches = plt.hist(df_train['weekday'], 50)\nplt.title('Day of the week')\nplt.xlabel('Day of week')\nplt.ylabel('Frequency')\nplt.grid(True)\nplt.show()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### We realize that there are some days with more data than the others\n### Maybe this is because there's 10 days of data. \n### To discover more info about the dataset, we need to have uniform data. \n### We drop the days 8,9 and 10\n\ndf_train = df_train[df_train['time']<'1970-01-08 00:00:00']\n\nprint(\"Size of training data: %d\" % df_train.shape[0])\nprint(\"Number of different places: %d\" % len(df_train['place_id'].unique()))\nprint(\"\\nDataset Description:\")\nprint(df_train.describe())\nprint(\"\\nInfo about the time period:\\n\" + str(df_train['time'].describe()))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### Visually check whether the weekday data is more uniform\nn, bins, patches = plt.hist(df_train['weekday'], 50)\nplt.title('Day of the week')\nplt.xlabel('Day of week')\nplt.ylabel('Frequency')\nplt.grid(True)\nplt.show()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### And the same exploration by weekday\n### based on localization info\n\nfig = plt.figure(figsize=(15,15))\n\nfor day in range(7):\n    sp = fig.add_subplot(3,3,day+1)\n\n    x = df_train[df_train['weekday'] == day]['x']\n    y = df_train[df_train['weekday'] == day]['y']\n\n    sp.hist2d(x, y, bins=20, norm=LogNorm())\n    sp.set_title('x and y location histogram day %d' % day)\n    sp.set_xlabel('x')\n    sp.set_ylabel('y')\n\nplt.show()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### To finalize this initial exploration, we plot some locations to find some patterns.\n\ngrouped = df_train['place_id'].value_counts().reset_index()\nids = grouped['index'][:10]\n\ncolors = cm.rainbow(np.linspace(0, 1, len(ids)))\n\nplt.figure(figsize=(10,10))\nfor id, c in zip(ids, colors):\n    x = df_train[df_train['place_id'] == id]['x']\n    y = df_train[df_train['place_id'] == id]['y']\n    plt.scatter(x, y, color=c)\n    \nplt.grid(True)\nplt.xlim(-0.1,10)\nplt.ylim(-0.1,10)\nplt.show()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "### Interesting!"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}