{"cells":[{"metadata":{"_cell_guid":"fc88eb5d-2c0f-4d72-9dd8-9bcd3143cd76","_uuid":"518a4a059bea00f32ee4f46d001ec8d6fdc7f7ab"},"cell_type":"markdown","source":"This kernel provides local CV score of evaluation metric.  \nCalculation takes about 5 min in this kernel and 2min in my local.  \n\nThis kernel is based on  \nKeras U-Net starter - LB 0.277  \n<https://www.kaggle.com/keegil/keras-u-net-starter-lb-0-277>  \nAlternative Metrics Kernel  \n<https://www.kaggle.com/glenslade/alternative-metrics-kernel>  \n\nI hope this kernel helps you.\n","outputs":[],"execution_count":null},{"metadata":{"_cell_guid":"a6c733e1-2c4a-45a4-a635-f4660dfdb0c2","_uuid":"81b73b5739132bc4fb4335cd59ebe429b9d983cb","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom subprocess import check_output\nprint(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"42ad794b-3734-4ac4-baa9-a1ed34cbb0c7","_uuid":"dc73619878f5afcb131d30a99725d56dff638536","trusted":true,"collapsed":true},"cell_type":"code","source":"# prepare functions for calculation\n\nimport skimage.io\nimport skimage.segmentation\nfrom skimage.transform import resize\nimport os\nfrom multiprocessing import Pool\n\n\ndef load_y_true(id, TRAIN_PATH):\n    file = TRAIN_PATH + \"{}/images/{}.png\".format(id, id)\n    mfile = TRAIN_PATH + \"{}/masks/*.png\".format(id)\n\n    image = skimage.io.imread(file)\n\n    masks = skimage.io.imread_collection(mfile).concatenate()\n    height, width, _ = image.shape\n    num_masks = masks.shape[0]\n    y_true = np.zeros((num_masks, height, width), np.bool)\n    y_true[:, :, :] = masks[:, :, :] // 255  # Change ground truth mask to zeros and ones\n    return y_true\n\n\ndef load_y_trues(TRAIN_PATH='input/stage1_train/', ng=[]):\n    train_ids = sorted(next(os.walk(TRAIN_PATH))[1])\n    for item in ng:\n        train_ids.remove(item)\n    y_trues = []\n    for i in range(len(train_ids)):\n        id = train_ids[i]\n        y_trues.append(load_y_true(id, TRAIN_PATH))\n    return y_trues\n\n\ndef IOU(y_pred, y_true):\n    \"\"\"\n    calcurate IOU of 1 image and its prediction\n    :param y_pred: numpy array. shape (mask, height, width)\n    :param y_true:\n    :return: list of scores. len = number of true masks\n    \"\"\"\n    num_true = len(y_true)\n    num_pred = len(y_pred)\n    iou = []\n    for pr in range(num_pred):\n        bol = 0  # best overlap\n        bun = 1e-9  # corresponding best union\n        for tr in range(num_true):\n            olap = y_pred[pr] * y_true[tr]  # Intersection points\n            osz = np.sum(olap)  # Add the intersection points to see size of overlap\n            if osz > bol:  # Choose the match with the biggest overlap\n                bol = osz\n                bun = np.sum(np.maximum(y_pred[pr], y_true[tr]))  # Union formed with sum of maxima\n        iou.append(bol / bun)\n    return iou\n\ndef mAP2(args):\n    \"\"\"\n    return np array with 43 values.\n    [0:2] = number of prediction masks, number of true masks\n    [1:6] = TP, FP, FN, Prec. with threshold 0.50\n    [6:10] = TP, FP, FN, Prec. with threshold 0.55\n    ...\n    [38:42] = TP, FP, FN, Prec. with threshold 0.95\n    [42] = mean Prec.\n    :param y_pred:\n    :param y_true:\n    :return: np array shape = 41,\n    \"\"\"\n    y_pred, train_id, train_path  = args\n    y_true = load_y_true(train_id, train_path)\n    num_true = len(y_true)\n    num_pred = len(y_pred)\n    # print(y_pred.dtype, y_true.dtype)\n    iou = IOU(y_pred, y_true)\n    output = np.zeros(43, np.float64)\n    p_all = 0\n    thresholds = np.arange(0.5, 1.0, 0.05)\n    for i in range(thresholds.shape[0]):\n        t = thresholds[i]\n        matches = iou > t\n        tp = np.count_nonzero(matches)  # True positives\n        fp = num_pred - tp  # False positives\n        fn = num_true - tp  # False negatives\n        p = tp / (tp + fp + fn)\n        p_all += p\n        output[i*4+2:(i+1)*4+2] = np.array([tp, fp, fn, p])\n    output[0] = num_pred\n    output[1] = num_true\n    output[42] = p_all/10\n    return output\n\ndef mAP(args):\n    \"\"\"\n    return np array with 43 values.\n    [0:2] = number of prediction masks, number of true masks\n    [1:6] = TP, FP, FN, Prec. with threshold 0.50\n    [6:10] = TP, FP, FN, Prec. with threshold 0.55\n    ...\n    [38:42] = TP, FP, FN, Prec. with threshold 0.95\n    [42] = mean Prec.\n    :param y_pred:\n    :param y_true:\n    :return: np array shape = 41,\n    \"\"\"\n    y_pred, y_true = args\n    num_true = len(y_true)\n    num_pred = len(y_pred)\n    # print(y_pred.dtype, y_true.dtype)\n    iou = IOU(y_pred, y_true)\n    output = np.zeros(43, np.float64)\n    p_all = 0\n    thresholds = np.arange(0.5, 1.0, 0.05)\n    for i in range(thresholds.shape[0]):\n        t = thresholds[i]\n        matches = iou > t\n        tp = np.count_nonzero(matches)  # True positives\n        fp = num_pred - tp  # False positives\n        fn = num_true - tp  # False negatives\n        p = tp / (tp + fp + fn)\n        p_all += p\n        output[i*4+2:(i+1)*4+2] = np.array([tp, fp, fn, p])\n    output[0] = num_pred\n    output[1] = num_true\n    output[42] = p_all/10\n    return output\n\n\ndef valid_score(y_preds, y_trues):\n    \"\"\"\n    calculate a validation IOU score of predction and some related values\n    :param y_preds: list of np array. len = number of images.\n                    each np array's shape = (number of nuclei, height, width)\n                    each height and width must be same with its original input image\n    :return: pd.Dataframe with shape = (number of images, 43)\n    \"\"\"\n    pool = Pool(1) # 16 107sec\n    \n    list_mAP = pool.map(mAP,\n                       [(y_preds[i], y_trues[i])\n                        for i in range(len(y_preds))])\n    pool.terminate()\n    scores = np.array(list_mAP)\n    cols = ['pred_masks', 'true_masks',\n            'TP_0.50', 'FP_0.50', 'FN_0.50', 'Prec_0.50',\n            'TP_0.55', 'FP_0.55', 'FN_0.55', 'Prec_0.55',\n            'TP_0.60', 'FP_0.60', 'FN_0.60', 'Prec_0.60',\n            'TP_0.65', 'FP_0.65', 'FN_0.65', 'Prec_0.65',\n            'TP_0.70', 'FP_0.70', 'FN_0.70', 'Prec_0.70',\n            'TP_0.75', 'FP_0.75', 'FN_0.75', 'Prec_0.75',\n            'TP_0.80', 'FP_0.80', 'FN_0.80', 'Prec_0.80',\n            'TP_0.85', 'FP_0.85', 'FN_0.85', 'Prec_0.85',\n            'TP_0.90', 'FP_0.90', 'FN_0.90', 'Prec_0.90',\n            'TP_0.95', 'FP_0.95', 'FN_0.95', 'Prec_0.95',\n            'mAP',\n            ]\n    scores = pd.DataFrame(scores, columns=cols)\n    return scores\n\n\ndef valid_score2(y_preds, train_ids, train_path):\n    \"\"\"\n    calculate a validation IOU score of predction and some related values\n    :param y_preds: list of np array. len = number of images.\n                    each np array's shape = (number of nuclei, height, width)\n                    each height and width must be same with its original input image\n    :return: pd.Dataframe with shape = (number of images, 43)\n    \"\"\"\n    pool = Pool() # 16 107sec\n    \n    list_mAP = pool.map(mAP2,\n                       [(y_preds[i], train_ids[i], train_path)\n                        for i in range(len(y_preds))])\n    pool.terminate()\n    scores = np.array(list_mAP)\n    cols = ['pred_masks', 'true_masks',\n            'TP_0.50', 'FP_0.50', 'FN_0.50', 'Prec_0.50',\n            'TP_0.55', 'FP_0.55', 'FN_0.55', 'Prec_0.55',\n            'TP_0.60', 'FP_0.60', 'FN_0.60', 'Prec_0.60',\n            'TP_0.65', 'FP_0.65', 'FN_0.65', 'Prec_0.65',\n            'TP_0.70', 'FP_0.70', 'FN_0.70', 'Prec_0.70',\n            'TP_0.75', 'FP_0.75', 'FN_0.75', 'Prec_0.75',\n            'TP_0.80', 'FP_0.80', 'FN_0.80', 'Prec_0.80',\n            'TP_0.85', 'FP_0.85', 'FN_0.85', 'Prec_0.85',\n            'TP_0.90', 'FP_0.90', 'FN_0.90', 'Prec_0.90',\n            'TP_0.95', 'FP_0.95', 'FN_0.95', 'Prec_0.95',\n            'mAP',\n            ]\n    scores = pd.DataFrame(scores, columns=cols)\n    return scores","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f6a2d58d-f17d-4292-bdd2-a45397ba4539","_uuid":"41d37b1c99785ad04bfe20e9808de350ddb58273","trusted":true},"cell_type":"code","source":"# load sample cv prediction\nfrom skimage.morphology import label\nimport time\n\nstart_time = time.time()\n\ny_preds = np.load(\"../input/y-preds-samplenpy/y_preds_sample.npy\")\nprint(\"y pred loading done. {:1.3f} sec\".format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"12ac2a91-0d4a-492e-a583-9c9d19ac6172","_uuid":"f016c5495ef869f5854c586d02d8d8ab2a9aac6a","trusted":true},"cell_type":"code","source":"# resize prediction\n# ignore bad train data\nng = [\"12aeefb1b522b283819b12e4cfaf6b13c1264c0aadac3412b4edd2ace304cb40\",\n      \"7b38c9173ebe69b4c6ba7e703c0c27f39305d9b2910f46405993d2ea7a963b80\",]\nsizes_train = []\nTRAIN_PATH = '../input/data-science-bowl-2018/stage1_train/'\ntrain_ids = sorted(next(os.walk(TRAIN_PATH))[1])\nfor item in ng:\n    train_ids.remove(item)\n\nfor i in range(len(train_ids)):\n    id = train_ids[i]\n    file = TRAIN_PATH + \"{}/images/{}.png\".format(id,id)\n    image = skimage.io.imread(file)\n    sizes_train.append((image.shape[0], image.shape[1]))\n\ny_preds_resized = []\nfor i in range(len(y_preds)):\n    y_preds_resized.append(resize(np.squeeze(y_preds[i]),\n                           (sizes_train[i][0], sizes_train[i][1]),\n                           mode='constant', preserve_range=True))\n\nprint(\"y pred resizing done. {:1.3f} sec\".format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c4c69c12-d0d3-44c2-8661-3d91d58958f0","_uuid":"f150b1ac09c53c77acec8046a4585b2921b057ed","trusted":true},"cell_type":"code","source":"# split mask\ny_preds_labeled = []\nCUTOFF = 0.5\nfor i in range(len(y_preds)):\n    y_label_base = label(y_preds_resized[i] > CUTOFF)\n    y_pred_labeled = np.zeros([np.max(y_label_base) + 1, y_label_base.shape[0], y_label_base.shape[1]], dtype=np.bool)\n    for k in range(y_pred_labeled.shape[0]):\n        y_pred_labeled[k] = y_label_base == k\n    y_preds_labeled.append(y_pred_labeled)\n   \nprint(\"y pred mask splitting done. {:1.3f} sec\".format(time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f9ec8bf4-76e5-4dbb-a382-d5c8a7320440","_uuid":"ec2a48b69f1dc75327697dec3d5b323ae3d29786","trusted":true,"collapsed":true},"cell_type":"code","source":"# load y true\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a898666d-71b0-4193-a935-7cf522ad5c80","_uuid":"a4620526b56fc99530aaad9751faf61ad462c30a","trusted":true},"cell_type":"code","source":"# scoring\n\n# faster but requires more RAM\n# y_trues = load_y_true(TRAIN_PATH=TRAIN_PATH, ng=ng)\n# score = valid_score(y_preds_labeled, y_trues)\n\nscore = valid_score2(y_preds_labeled, train_ids, TRAIN_PATH)\n\nprint(\"valid score\", np.mean(score['mAP']))\nprint(\"scoring done.\", time.time() - start_time)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}