{"cells":[{"metadata":{},"cell_type":"markdown","source":"This notebook shows how to compute score in local.\nDemo is given with simply literal matching.\n\nAny comments/suggestions are welcome!"},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nimport numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# train\ntrain_path = f'../input/coleridge-cv-data/cv_1_train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\ntrain = pd.read_csv(train_path)\n\n# test\ntest_path = f'../input/coleridge-cv-data/cv_1_test.csv'\npaper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\ntest = pd.read_csv(test_path)\n\n# paper\npapers = {}\nfor paper_id in train['Id'].unique():\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper\n\nfor paper_id in test['Id'].unique():\n    with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Literal matching"},{"metadata":{},"cell_type":"markdown","source":"### Prepare literals"},{"metadata":{"trusted":true},"cell_type":"code","source":"text_to_pred = {}\ndataset_texts = set()\n\nfor id, pub_title, dataset_title, dataset_label, cleaned_label in train.itertuples(index=False):\n    for title, label, cleaned_label in zip(dataset_title.split('|'), \n                                           dataset_label.split('|'), \n                                           cleaned_label.split('|')):\n        text_to_pred[title] = cleaned_label\n        dataset_texts.add(title)\n        text_to_pred[label] = cleaned_label\n        dataset_texts.add(label)\n    \nprint(f'text_to_pred size: {len(text_to_pred)}')\nprint(f'dataset_texts size: {len(dataset_texts)}')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Match and predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = []\nfor id in test['Id']:\n    paper = papers[id]\n    paper = str(paper)\n    \n    pred = set()\n    for dataset_text in dataset_texts:\n        if dataset_text in paper:\n            pred.add(text_to_pred[dataset_text])\n    \n    preds.append('|'.join(pred))\n\ntest['PredictionString'] = preds\n\ndisplay(test.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compute score"},{"metadata":{"trusted":true},"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\ndef compute_score(y_true, y_pred, beta=0.5):\n    TP, FP, FN = 0, 0, 0\n    \n    for truth, pred in zip(y_true, y_pred):\n        true_datasets = truth.split('|')\n        # Predicted strings for each publication are sorted alphabetically \n        # and processed in that order.\n        pred_datasets = sorted(pred.split('|'))\n        \n        for true_dataset in true_datasets:\n            if len(pred_datasets):\n                match_scores = [jaccard_similarity(true_dataset, pred_dataset) \n                                for pred_dataset in pred_datasets]\n                # The prediction with the highest score for a given ground truth \n                # is matched with that ground truth.\n                match_index = np.argmax(match_scores)\n\n                if match_scores[match_index] >= 0.5:\n                    # Any matched predictions where the Jaccard score meets or\n                    # exceeds the threshold of 0.5 are counted as true positives (TP),\n                    TP += 1\n                else:\n                    # the remainder as false positives (FP).\n                    FP += 1\n                \n                del(pred_datasets[match_index])\n            else:\n                # Any ground truths with no nearest predictions are counted as \n                # false negatives (FN).\n                FN += 1\n        # Any unmatched predictions are counted as false positives (FP).\n        FP += len(pred_datasets)\n    \n    precision = TP / (TP + FP)\n    recall = TP / (TP + FN)\n    f_score = (1 + beta**2)*(precision*recall)/((beta**2)*precision + recall)\n    \n    return f_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"compute_score(test['cleaned_label'], test['PredictionString'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}