{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np \nimport pandas as pd \nimport matplotlib\n\nfrom subprocess import check_output\n\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n%matplotlib inline "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test = pd.read_csv(\"../input/test.csv\", index_col=\"row_id\")\ntest.head()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train = pd.read_csv(\"../input/train.csv\", index_col=\"row_id\")\ntrain.head()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print(train.shape) \nprint(test.shape)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "### What does stats say about the data ? "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_descriptive_stats = train.describe()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_descriptive_stats"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test_descriptive_stats = test.describe()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test_descriptive_stats"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train.corr()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test.corr()"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "from sklearn.cross_validation import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(train[train.columns.difference([\"place_id\"])], train[\"place_id\"], test_size=0.33, random_state=42)\nprint(X_train.shape)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train.tail(100000).plot.hexbin(x='x', y='y');"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## what is the distribution of places vs number of rows in the training data ? \ncompute the number of rows in the train set associated with a given place_id"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "places_vs_datarows_distribution = train[\"place_id\"].value_counts()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "create a dataframe to showcase the distribution of the place_id in the train data. "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "places = places_vs_datarows_distribution.index \nplaces_vs_datarows_distribution = places_vs_datarows_distribution.to_frame(\"nbr_of_rows\") \nplaces_vs_datarows_distribution[\"place_id\"] = places\ndel places "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "places_vs_datarows_distribution.head()"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "what is the total number of unique place_id in the train set? "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "len(pd.unique(train[\"place_id\"].values.ravel()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "how many place\\_id values have 100 or less rows in the training data? "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "less_than_100_place_id = places_vs_datarows_distribution[places_vs_datarows_distribution[\"nbr_of_rows\"] <= 100]\nlen(pd.unique(less_than_100_place_id[\"place_id\"].values.ravel()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "how many place\\_id values have 10 or less rows in the training data? "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "less_than_10_place_id = places_vs_datarows_distribution[places_vs_datarows_distribution[\"nbr_of_rows\"] <= 10]\nlen(pd.unique(less_than_10_place_id[\"place_id\"].values.ravel()))"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "how does the distribution look like ? "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "places_vs_datarows_distribution.hist(column=\"nbr_of_rows\",bins=100,figsize=(10,10), xlabelsize=15, ylabelsize=15)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## How many outliers do we have? \n"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "\nfrom scipy import stats\nno_outliers = train[(np.abs(stats.zscore(train)) < 3).all(axis=1)]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print(train.shape)\nprint(no_outliers.shape)"
 },
 {
  "cell_type": "markdown",
  "metadata": {},
  "source": "## what is the time interval "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "train_descriptive_stats[\"time\"]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "test_descriptive_stats[\"time\"]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#assuming time is in minutes  \n#how long does it span over train set? \ntrain_nbr_of_minutes = (train_descriptive_stats[\"time\"].loc[\"max\"]-train_descriptive_stats[\"time\"].loc[\"min\"]) "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print(\"train hours {0}\".format(train_nbr_of_minutes/(60)))\nprint(\"train days {0}\".format(train_nbr_of_minutes/(60*24)))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#how long does time span in test data\ntest_nbr_of_minutes = (test_descriptive_stats[\"time\"].loc[\"max\"]-test_descriptive_stats[\"time\"].loc[\"min\"])"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print(\"test hours {0}\".format(test_nbr_of_minutes/(60)))\nprint(\"test days {0}\".format(test_nbr_of_minutes/(60*24)))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#split the to grids of 2kmx2km\nfrom itertools import product \ndef split_to_grids(grid_size, data):\n    analysis_per_grid = pd.DataFrame()\n    grid_id = []\n    grid_places_count = []\n    for i in product(range(grid_size,11,grid_size),repeat=2):\n        grid_id.append(i)\n        dt = data[(data[\"x\"]>=i[0]-grid_size)&(data[\"x\"]<i[0])&(data[\"y\"]>=i[1]-grid_size)&(data[\"y\"]<i[1])]\n        count = dt[\"place_id\"].count()\n        grid_places_count.append(count) \n        print(\"nbr of places {0}\".format(count))\n    analysis_per_grid[\"grid_id\"] = grid_id\n    analysis_per_grid[\"places_count\"] = grid_places_count\n    return analysis_per_grid"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "analysis_per_grid = split_to_grids(2,train)"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "analysis_per_grid"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}