{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np\nfrom sklearn.grid_search import GridSearchCV\nfrom sklearn.neighbors import KNeighborsClassifier"
 },
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "def my_scorer(clf, X, y_true):\n    y_pred_proba = clf.predict_proba(X)\n    class_labels = clf.classes_\n\n    # Sort the classifications based on highest probability,\n    # making sure to keep track of class labels properly\n    p_pairs = [sorted([(c, p) for c, p in zip(class_labels, y_pred_proba[j])],\n                      key=lambda x: x[1], reverse=True)\n                for j in range(y_pred_proba.shape[0])]\n    \n    # Get the top 3 predictions by selecting the class\n    # label piece of the tuple (element 0) for each sample\n    y_top_3 = [[p[i][0] for i in range(3)]\n               for p in p_pairs]\n    \n    # Calculate the MAP@3 score,\n    # where the sum over P(k) is equal to\n    # 1 if correct prediction is 1st\n    # 1/2 if correct prediction is 2nd\n    # 1/3 if correct prediction is 3rd\n    MAP_score = [[(y == y_true[j])/(i+1) for i, y in enumerate(y_sample)]\n                 for j, y_sample in enumerate(y_top_3)]\n    MAP_score = np.sum(MAP_score, axis=1).mean()\n    \n    return MAP_score\n\ngs = GridSearchCV(estimator=KNeighborsClassifier(),\n                  param_grid=[{'n_neighbors': [2, 4, 6]}],\n                  cv=5,\n                  scoring=my_scorer)"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}