{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "\n__author__ = 'Michael Hartman'\n\n'''Inspired by several scripts at:\nhttps://www.kaggle.com/c/facebook-v-predicting-check-ins/scripts\nSpecial thanks to Sandro for starting the madness. :-)\nhttps://www.kaggle.com/svpons/facebook-v-predicting-check-ins/grid-plus-classifier\n'''\nimport time\nimport gc\nimport numpy as np\nimport pandas as pd\nfrom datetime import timedelta\nfrom xgboost.sklearn import XGBClassifier\n#from nolearn.dbn import DBN\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.ensemble import RandomForestClassifier\n\n# Found at: https://www.kaggle.com/rshekhar2/facebook-v-predicting-check-ins/xgboost-cv-example-with-small-bug\ndef mapkprecision(truthvalues, predictions):\n    '''\n    This is a faster implementation of MAP@k valid for numpy arrays.\n    It is only valid when there is one single truth value. \n\n    m ~ number of observations\n    k ~ MAP at k -- in this case k should equal 3\n\n    truthvalues.shape = (m,) \n    predictions.shape = (m, k)\n    '''\n    z = (predictions == truthvalues[:, None]).astype(np.float32)\n    weights = 1./(np.arange(predictions.shape[1], dtype=np.float32) + 1.)\n    z = z * weights[None, :]\n    return np.mean(np.sum(z, axis=1))\n\ndef load_data(data_name):\n    types = {'row_id': np.dtype(np.int32),\n         'x': np.dtype(float),\n         'y' : np.dtype(float),\n         'accuracy': np.dtype(np.int16),\n         'place_id': np.int64,\n         'time': np.dtype(np.int32)}\n    df = pd.read_csv(data_name, dtype=types, index_col = 0, na_filter=False)\n    return df\n\ndef process_one_cell(df_cell_train, df_cell_test, fw, th, n_neighbors):\n    \n    # Remove infrequent places\n    df_cell_train = remove_infrequent_places(df_cell_train, th).copy()\n    \n    # Store row_ids for test\n    row_ids = df_cell_test.index\n    \n    # Preparing data\n    y = df_cell_train.place_id.values\n    X = df_cell_train.drop(['place_id'], axis=1).values\n    \n    #Applying the classifier\n    \n    clf = KNeighborsClassifier(n_neighbors=n_neighbors,\n                            weights=calculate_distance, p=1, \n                            n_jobs=2, leaf_size=20) \n    \n    #clf = XGBClassifier(n_estimators = 30)\n    clf.fit(X, y)\n    y_pred = clf.predict_proba(df_cell_test.values)\n    y_pred_labels = np.argsort(y_pred, axis=1)[:,:-4:-1]\n    pred_labels = clf.classes_[y_pred_labels]\n    cell_pred = np.column_stack((row_ids, pred_labels)).astype(np.int64) \n    \n    return cell_pred\n    \ndef calculate_distance(distances):\n    return distances ** -2.2\n    \n# Generate a dictionary of the time limits so it doesn't have to be \n# recalculated each loop\ndef create_time_dict(t_cuts, time_factor, time_aug):\n    \n    t_slice = 24 / t_cuts\n    time_dict = dict()\n    for t in range(t_cuts):\n        \n        t_min = 2 * np.pi * (t * t_slice * 12 / 288)\n        t_max = 2 * np.pi * (((t + 1) * t_slice * 12 - 1) / 288)\n        sin_t_start = np.round(np.sin(t_min)+1, 4) * time_factor\n        sin_t_stop = np.round(np.sin(t_max)+1, 4) * time_factor\n        cos_t_start = np.round(np.cos(t_min)+1, 4) * time_factor\n        cos_t_stop = np.round(np.cos(t_max)+1, 4) * time_factor\n        #print(t, (sin_t_start, sin_t_stop, cos_t_start, cos_t_stop))\n        sin_t_min = min((sin_t_start, sin_t_stop))\n        sin_t_max = max((sin_t_start, sin_t_stop))\n        cos_t_min = min((cos_t_start, cos_t_stop))\n        cos_t_max = max((cos_t_start, cos_t_stop))\n\n        time_dict[t] = [sin_t_min, sin_t_max, cos_t_min, cos_t_max]\n        t_min = 2 * np.pi * ((t * t_slice - time_aug) * 12 / 288)\n        t_max = 2 * np.pi * ((((t + 1) * t_slice + time_aug)* 12 - 1) / 288)\n        sin_t_start = np.round(np.sin(t_min)+1, 4) * time_factor\n        sin_t_stop = np.round(np.sin(t_max)+1, 4) * time_factor\n        cos_t_start = np.round(np.cos(t_min)+1, 4) * time_factor\n        cos_t_stop = np.round(np.cos(t_max)+1, 4) * time_factor\n        sin_t_min = min((sin_t_start, sin_t_stop, sin_t_min))\n        sin_t_max = max((sin_t_start, sin_t_stop, sin_t_max))\n        cos_t_min = min((cos_t_start, cos_t_stop, cos_t_min))\n        cos_t_max = max((cos_t_start, cos_t_stop, cos_t_max))\n        time_dict[t] += [sin_t_min, sin_t_max, cos_t_min, cos_t_max]\n        \n    return time_dict\n\ndef process_grid(df_train, df_test, x_cuts, y_cuts, t_cuts,\n                 x_border_aug, y_border_aug, time_aug, fw, th, n_neighbors):\n    preds_list = []\n    x_slice = df_train['x'].max() / x_cuts\n    y_slice = df_train['y'].max() / y_cuts\n    time_max = df_train['minute_sin'].max()\n    time_factor = time_max / 2\n    time_dict = create_time_dict(t_cuts, time_factor, time_aug)\n\n    for i in range(x_cuts):\n        row_start_time = time.time()\n        x_min = x_slice * i\n        x_max = x_slice * (i+1)\n        x_max += int((i+1) == x_cuts) # expand edge at end\n\n        mask = (df_test['x'] >= x_min)\n        mask = mask & (df_test['x'] < x_max)      \n        df_col_test = df_test[mask]\n        x_min -= x_border_aug\n        x_max += x_border_aug\n        mask = (df_train['x'] >= x_min)\n        mask = mask & (df_train['x'] < x_max)\n        df_col_train = df_train[mask]\n\n        for j in range(y_cuts):\n            y_min = y_slice * j\n            y_max = y_slice * (j+1)\n            y_max += int((j+1) == y_cuts) # expand edge at end\n\n            mask = (df_col_test['y'] >= y_min)\n            mask= mask & (df_col_test['y'] < y_max)\n            df_row_test = df_col_test[mask]\n            y_min -= y_border_aug\n            y_max += y_border_aug\n            mask = (df_col_train['y'] >= y_min)\n            mask = mask & (df_col_train['y'] < y_max)\n            df_row_train = df_col_train[mask]\n\n            for t in range(t_cuts):\n                #print(df_row_test.shape, df_row_train.shape)\n                t_lim = time_dict[t]\n                mask = df_row_test['minute_sin'].between(t_lim[0], t_lim[1])\n                mask = mask & df_row_test['minute_cos'].between(t_lim[2], t_lim[3])\n                df_cell_test = df_row_test[mask].copy()\n                mask = df_row_train['minute_sin'].between(t_lim[4], t_lim[5])\n                mask = mask & df_row_train['minute_cos'].between(t_lim[6], t_lim[7])\n                df_cell_train = df_row_train[mask].copy()\n                cell_pred = process_one_cell(df_cell_train.copy(), \n                                             df_cell_test.copy(), \n                                             fw, th, n_neighbors)\n                preds_list.append(cell_pred)\n        elapsed = (time.time() - row_start_time)\n        print('Row', i, 'completed in:', timedelta(seconds=elapsed))\n    preds = np.vstack(preds_list)\n    return preds\n\n# Thank you Alex!\n# From: https://www.kaggle.com/drarfc/facebook-v-predicting-check-ins/fastest-way-to-write-the-csv\ndef generate_submission(preds):    \n    print('Writing submission file')\n    print('Pred shape:', preds.shape)\n    with open('KNN_submission.csv', \"w\") as out:\n        out.write(\"row_id,place_id\\n\")\n        rows = ['']*8607230\n        n=0\n        for num in range(8607230):\n            rows[n]='%d,%d %d %d\\n' % (preds[num,0],preds[num,1],preds[num,2],preds[num,3])\n            n=n+1\n        out.writelines(rows)\n\ndef validation_split(df, val_start_day):\n    day = df['time']//1440\n    df_val = df.loc[(day>=val_start_day)].copy()\n    df = df.loc[(day<val_start_day)].copy()\n    return df, df_val\n\ndef remove_infrequent_places(df, th=5):\n    place_counts = df.place_id.value_counts()\n    mask = (place_counts[df.place_id.values] >= th).values\n    df = df.loc[mask]\n    return df\n    \ndef prepare_data(datapath, val_start_day):\n    val_label = None\n    df_train = load_data(datapath + 'train.csv')\n    if val_start_day > 0:\n        # Create validation data\n        df_train, df_test = validation_split(df_train, val_start_day)\n        val_label = df_test['place_id']\n        df_test.drop(['place_id'], axis=1, inplace=True)\n        print('Feature engineering on train')\n        df_train = feature_engineering(df_train)\n        print('Feature engineering on validation')\n        df_test = feature_engineering(df_test)\n    else:\n        print('Feature engineering on train')\n        df_train = feature_engineering(df_train)\n        df_test = load_data(datapath + 'test.csv') \n        print('Feature engineering on test')\n        df_test = feature_engineering(df_test)\n    df_train = apply_weights(df_train, fw)\n    df_test = apply_weights(df_test, fw)\n    return df_train, df_test, val_label\n        \n\ndef apply_weights(df, fw):\n    df['accuracy'] *= fw[0]\n    df['day_of_year_sin'] *= fw[1]\n    df['day_of_year_cos'] *= fw[1]\n    df['minute_sin'] *= fw[2]\n    df['minute_cos'] *= fw[2]\n    df['weekday_sin'] *= fw[3]\n    df['weekday_cos'] *= fw[3]\n    df.x *= fw[4]\n    df.y *= fw[5]\n    df['year'] *= fw[6]\n    return df\n    \ndef feature_engineering(df):\n    minute = 2*np.pi*((df[\"time\"]//5)%288)/288\n    df['minute_sin'] = (np.sin(minute)+1).round(4)\n    df['minute_cos'] = (np.cos(minute)+1).round(4)\n    del minute\n    day = 2*np.pi*((df['time']//1440)%365)/365\n    df['day_of_year_sin'] = (np.sin(day)+1).round(4)\n    df['day_of_year_cos'] = (np.cos(day)+1).round(4)\n    del day\n    weekday = 2*np.pi*((df['time']//1440)%7)/7\n    df['weekday_sin'] = (np.sin(weekday)+1).round(4)\n    df['weekday_cos'] = (np.cos(weekday)+1).round(4)\n    del weekday\n    df['year'] = (((df['time'])//525600))\n    df.drop(['time'], axis=1, inplace=True)\n    df['accuracy'] = np.log10(df['accuracy'])\n    return df\n    "
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "print('Starting...')\nstart_time = time.time()\n# Global variables\ndatapath = '../input/'\n# Change val_start_day to zero to generate predictions\nval_start_day = 441 # Day at which to cut validation\nth = 5 # Threshold at which to cut places from train\nfw = [0.6, 0.32935, 0.56515, 0.2670, 22, 52, 0.51785]\n\n# Defining the size of the grid\nx_cuts = 20 # number of cuts along x \ny_cuts = 20 # number of cuts along y\n#TODO: More general solution for t_cuts. For now must be 4.\nt_cuts = 4 # number of cuts along time. \nx_border_aug = 0.04 # expansion of x border on train \ny_border_aug = 0.015 # expansion of y border on train\ntime_aug = 2\nn_neighbors = 37"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "df_train, df_test, val_label = prepare_data(datapath, val_start_day)\ngc.collect()\n\nelapsed = (time.time() - start_time)\nprint('Data prepared in:', timedelta(seconds=elapsed))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "preds = process_grid(df_train, df_test, x_cuts, y_cuts, t_cuts,\n                     x_border_aug, y_border_aug, time_aug, \n                     fw, th, n_neighbors)\nelapsed = (time.time() - start_time)\nprint('Predictions made in:', timedelta(seconds=elapsed))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "#del df_train, df_test\n\nif val_start_day > 0:\n    preds = preds[preds[:, 0] > 0] # only use rows predicted\n    labels = val_label.loc[preds[:, 0]].values\n    score = mapkprecision(labels, preds[:, 1:])\n    print('Final score:', score)\nelse:\n    generate_submission(preds)\nelapsed = (time.time() - start_time)\nprint('Task completed in:', timedelta(seconds=elapsed))"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "preds[1]"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": ""
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}