{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport simplejson\nimport re\nimport pydash\nimport sys\nimport os\nfrom collections import defaultdict\nfrom typing import *\nfrom joblib import Parallel, delayed\nfrom glob import glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(text: str) -> str:               \n    return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_texts(texts: List[str]) -> List[str]: return [ clean_text(text) for text in texts ] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame([[1, 1.5]], columns=['int', 'float'])\nrow = next(df.iterrows())\nrow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lookup_indf(df):\n    lookup=defaultdict(set)\n    for _,row in df.iterrows():\n        label = clean_text(row['dataset_title'])  # was: row['cleaned_label']\n        lookup[ label ] |= set(clean_texts([ \n            row['dataset_label'], \n            row['dataset_title'], \n            row['pub_title'],\n            # row['cleaned_label'], \n    ]))\n    return lookup","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"next(iter(lookup_indf(train).items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_json(index: str, test_train=\"test\") -> Dict:\n    filename = f\"../input/coleridgeinitiative-show-us-the-data/{test_train}/{index}.json\"\n    with open(filename) as f:\n        json = simplejson.load(f)\n    return json\n        \ndef json2text(index: str, test_train=\"test\") -> str:\n    json  = read_json(index, test_train)\n    texts = [\n        row[\"section_title\"] + \" \" + row[\"text\"] \n        for row in json\n    ]\n    texts = clean_texts(texts)\n    text  = \" \".join(texts)\n    return text\n\n\ndef extract_label(text: str, lookup: Dict[str, Set[str]]) -> str:\n    labels = []\n    for label, values in lookup.items():\n        for value in values:\n            if value in text:                \n                labels += [ clean_text(value) ]\n            \n    label = \"|\".join(set(labels))  # multi label support\n    # label = Counter(labels).most_common(1)[0][0] if len(labels) else \"\"  # single most-popular label\n    # print('extract_label', labels, '->', label)\n    return label","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_accuracy(df, limit=sys.maxsize) -> float:\n    limit   = 100 if os.environ.get('KAGGLE_KERNEL_RUN_TYPE', 'Localhost') == 'Interactive' else limit\n    lookup  = lookup_indf(df)\n    labels  = Parallel(-1)(\n        delayed(extract_label)(json2text(index, \"train\"), lookup)\n        for index in df['Id'][:limit]\n    )\n    correct   = 0\n    expecteds = df['cleaned_label'][:limit]\n    for label, expected in zip(labels, expecteds):\n        expected_set = set(expected.split(\"|\"))\n        label_set    = set(label.split(\"|\"))\n        matches      = expected_set & label_set\n        correct     += len(matches) / len(label_set)\n\n    # correct = np.count_nonzero( np.array(labels) == expecteds )\n    total   = len(expecteds)\n    return correct / total\n\ntrain_accuracy(train, 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def generate_submission():\n    submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)\n    lookup  = lookup_indf(train)\n    indexes = submission_df.index\n    labels  = Parallel(-1)(\n        delayed(extract_label)(json2text(index, \"test\"), lookup)\n        for index in indexes\n    )\n    submission_df['PredictionString'] = labels\n    return submission_df\n\nsubmission_df = generate_submission()\nsubmission_df.to_csv('submission.csv')\n!head submission.csv\nsubmission_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}