{"cells":[{"cell_type":"markdown","metadata":{"_cell_guid":"31ae9e5f-9955-d971-c2c5-141a7c290be5"},"source":"# Matthews correlation coefficient\n\nSee https://en.wikipedia.org/wiki/Matthews_correlation_coefficient\n\nAuthor: [CPMP](https://www.kaggle.com/cpmpml)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"0a633cf2-8deb-f33a-a30e-a435661c7072"},"outputs":[],"source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"3d3eb384-ce43-a4c3-c4ca-a37e8e4c1346"},"outputs":[],"source":"%matplotlib inline\n\nfrom sklearn.metrics import matthews_corrcoef\n\nimport matplotlib.pyplot as plt"},{"cell_type":"markdown","metadata":{"_cell_guid":"10575e78-0a7f-d203-b973-5c18a7950f08"},"source":"We compile the code with Numba.  If you don't have it installed, then just comment out the import and the @jit lines below."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d62f429d-342e-bc25-d97a-6a1ff2ef8bdc"},"outputs":[],"source":"from numba import jit\n\n@jit\ndef mcc(tp, tn, fp, fn):\n    sup = tp * tn - fp * fn\n    inf = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n    if inf==0:\n        return 0\n    else:\n        return sup / np.sqrt(inf)\n\n@jit\ndef eval_mcc(y_true, y_prob, show=False):\n    idx = np.argsort(y_prob)\n    y_true_sort = y_true[idx]\n    n = y_true.shape[0]\n    nump = 1.0 * np.sum(y_true) # number of positive\n    numn = n - nump # number of negative\n    tp = nump\n    tn = 0.0\n    fp = numn\n    fn = 0.0\n    best_mcc = 0.0\n    best_id = -1\n    mccs = np.zeros(n)\n    for i in range(n):\n        # all items with idx <= i are predicted negative while others are predicted positive\n        if y_true_sort[i] == 1:\n            tp -= 1.0\n            fn += 1.0\n        else:\n            fp -= 1.0\n            tn += 1.0\n        new_mcc = mcc(tp, tn, fp, fn)\n        mccs[i] = new_mcc\n        if new_mcc >= best_mcc:\n            best_mcc = new_mcc\n            best_id = i\n    if show:\n        best_proba = y_prob[idx[best_id]]\n        y_pred = (y_prob > best_proba).astype(int)\n        score = matthews_corrcoef(y_true, y_pred)\n        print(score, best_mcc)\n        plt.plot(mccs)\n        return best_proba, best_mcc, y_pred\n    else:\n        return best_mcc"},{"cell_type":"markdown","metadata":{"_cell_guid":"e39bdb23-7b6d-ae71-5e80-0e74e6ee4e25"},"source":"Let's see what this gives on an example.  We also compute the MCC using scikit-learn function to make sure we compute the same."},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"d6164f6f-5b4b-50f5-40c6-8d68a1cfb07c"},"outputs":[],"source":"y_prob0 = np.random.rand(1000000)\ny_prob = y_prob0 + 0.4 * np.random.rand(1000000) - 0.02\n\n\ny_true = (y_prob0 > 0.6).astype(int)\nbest_proba, best_mcc, y_pred = eval_mcc(y_true, y_prob, True)"},{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"2f8eded0-bb73-7124-a68e-065bed3edc0f"},"outputs":[],"source":""}],"metadata":{"_change_revision":0,"_is_fork":false,"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.5.2"}},"nbformat":4,"nbformat_minor":0}