{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import transformers\nprint(transformers.__version__)\nfrom transformers import *","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:28.997105Z","iopub.status.busy":"2021-06-15T03:01:28.996307Z","iopub.status.idle":"2021-06-15T03:01:37.834023Z","shell.execute_reply":"2021-06-15T03:01:37.832746Z","shell.execute_reply.started":"2021-05-25T03:35:28.114081Z"},"papermill":{"duration":8.867674,"end_time":"2021-06-15T03:01:37.834254","exception":false,"start_time":"2021-06-15T03:01:28.96658","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Asthetics\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\nimport gc\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport json\nimport os\nimport random\nfrom tqdm.autonotebook import tqdm\nimport string\nimport re\nfrom functools import partial\n\ntqdm.pandas()","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2021-06-15T03:01:37.892675Z","iopub.status.busy":"2021-06-15T03:01:37.891699Z","iopub.status.idle":"2021-06-15T03:01:37.89465Z","shell.execute_reply":"2021-06-15T03:01:37.894203Z","shell.execute_reply.started":"2021-05-25T03:35:28.127408Z"},"papermill":{"duration":0.037127,"end_time":"2021-06-15T03:01:37.894793","exception":false,"start_time":"2021-06-15T03:01:37.857666","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RANDOM_SEED = 42\ndef seed_everything(seed=RANDOM_SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\nseed_everything()","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:37.946899Z","iopub.status.busy":"2021-06-15T03:01:37.945935Z","iopub.status.idle":"2021-06-15T03:01:37.948558Z","shell.execute_reply":"2021-06-15T03:01:37.949026Z","shell.execute_reply.started":"2021-05-25T03:35:28.138655Z"},"papermill":{"duration":0.031864,"end_time":"2021-06-15T03:01:37.949196","exception":false,"start_time":"2021-06-15T03:01:37.917332","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'\ntrain_df.head(10)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.003653Z","iopub.status.busy":"2021-06-15T03:01:38.003031Z","iopub.status.idle":"2021-06-15T03:01:38.155783Z","shell.execute_reply":"2021-06-15T03:01:38.154893Z","shell.execute_reply.started":"2021-05-25T03:35:28.150279Z"},"papermill":{"duration":0.182797,"end_time":"2021-06-15T03:01:38.155953","exception":false,"start_time":"2021-06-15T03:01:37.973156","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infer_limit = 999999\nis_real_run = len(os.listdir('../input/coleridgeinitiative-show-us-the-data/test')) > 10\nis_real_run","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.21026Z","iopub.status.busy":"2021-06-15T03:01:38.209433Z","iopub.status.idle":"2021-06-15T03:01:38.218682Z","shell.execute_reply":"2021-06-15T03:01:38.219161Z","shell.execute_reply.started":"2021-05-25T03:35:28.22034Z"},"papermill":{"duration":0.038073,"end_time":"2021-06-15T03:01:38.219341","exception":false,"start_time":"2021-06-15T03:01:38.181268","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.278208Z","iopub.status.busy":"2021-06-15T03:01:38.277226Z","iopub.status.idle":"2021-06-15T03:01:38.280206Z","shell.execute_reply":"2021-06-15T03:01:38.279714Z","shell.execute_reply.started":"2021-05-25T03:35:28.229355Z"},"papermill":{"duration":0.036131,"end_time":"2021-06-15T03:01:38.280345","exception":false,"start_time":"2021-06-15T03:01:38.244214","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.339129Z","iopub.status.busy":"2021-06-15T03:01:38.338172Z","iopub.status.idle":"2021-06-15T03:01:38.398004Z","shell.execute_reply":"2021-06-15T03:01:38.397566Z","shell.execute_reply.started":"2021-05-25T03:35:28.240472Z"},"papermill":{"duration":0.091072,"end_time":"2021-06-15T03:01:38.398155","exception":false,"start_time":"2021-06-15T03:01:38.307083","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef clean_text_v2(txt):\n    return re.sub('[^A-Za-z0-9\\(\\)]+', ' ', str(txt).lower().strip())","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.457658Z","iopub.status.busy":"2021-06-15T03:01:38.456732Z","iopub.status.idle":"2021-06-15T03:01:38.45958Z","shell.execute_reply":"2021-06-15T03:01:38.459149Z","shell.execute_reply.started":"2021-05-25T03:35:28.29973Z"},"papermill":{"duration":0.036117,"end_time":"2021-06-15T03:01:38.459708","exception":false,"start_time":"2021-06-15T03:01:38.423591","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [clean_text(x) for x in train_df['dataset_label'].unique()]\ntemp_2 = [clean_text(x) for x in train_df['dataset_title'].unique()]\ntemp_3 = [clean_text(x) for x in train_df['cleaned_label'].unique()]\n\nempty_ids = []\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlabels_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = clean_text(row['text'])\n    row_id = row['Id']\n    cleaned_labels = []\n    for known_label in existing_labels:\n        if known_label in sample_text:\n            cleaned_labels.append(clean_text(known_label))\n#     cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    labels_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)\n#     if len(labels_list[-1]) == 0:\n    empty_ids.append(row_id)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.519003Z","iopub.status.busy":"2021-06-15T03:01:38.518095Z","iopub.status.idle":"2021-06-15T03:01:38.619842Z","shell.execute_reply":"2021-06-15T03:01:38.619411Z","shell.execute_reply.started":"2021-05-25T03:35:28.310887Z"},"papermill":{"duration":0.135781,"end_time":"2021-06-15T03:01:38.620022","exception":false,"start_time":"2021-06-15T03:01:38.484241","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# existing_labels","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.674435Z","iopub.status.busy":"2021-06-15T03:01:38.673543Z","iopub.status.idle":"2021-06-15T03:01:38.676218Z","shell.execute_reply":"2021-06-15T03:01:38.676636Z","shell.execute_reply.started":"2021-05-25T03:35:28.420939Z"},"papermill":{"duration":0.032358,"end_time":"2021-06-15T03:01:38.676805","exception":false,"start_time":"2021-06-15T03:01:38.644447","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if not is_real_run:\n    empty_ids = train_df.Id.values[:30]\n    data_dir = \"train\"\nelse:\n    empty_ids = empty_ids[:infer_limit]\n    data_dir = \"test\"","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.733439Z","iopub.status.busy":"2021-06-15T03:01:38.732533Z","iopub.status.idle":"2021-06-15T03:01:38.735677Z","shell.execute_reply":"2021-06-15T03:01:38.735213Z","shell.execute_reply.started":"2021-05-25T03:35:28.426695Z"},"papermill":{"duration":0.033873,"end_time":"2021-06-15T03:01:38.735837","exception":false,"start_time":"2021-06-15T03:01:38.701964","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import *\nimport torch\nfrom torch import nn\nimport torch.nn.functional as F\n\nimport logging\nimport math\nimport os\n\nimport torch\nfrom torch import nn\nfrom torch.nn import CrossEntropyLoss, MSELoss","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.794152Z","iopub.status.busy":"2021-06-15T03:01:38.793284Z","iopub.status.idle":"2021-06-15T03:01:38.796504Z","shell.execute_reply":"2021-06-15T03:01:38.796049Z","shell.execute_reply.started":"2021-05-25T03:35:28.44261Z"},"papermill":{"duration":0.035093,"end_time":"2021-06-15T03:01:38.796648","exception":false,"start_time":"2021-06-15T03:01:38.761555","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def convert_lines(tokenizer, df, max_sequence_length = 512, is_test=False):\n    pad_token_idx = tokenizer.pad_token_id or tokenizer.eos_token_id\n    cls_token_idx = tokenizer.cls_token_id or tokenizer.eos_token_id\n    sep_token_idx = tokenizer.sep_token_id or tokenizer.eos_token_id\n    outputs = np.zeros((len(df), max_sequence_length))\n    type_outputs = np.zeros((len(df), max_sequence_length))\n    position_outputs = np.zeros((len(df), 2))\n    offset_outputs = np.ones((len(df),))\n    extracted = []\n    for idx, row in tqdm(df.iterrows(), total=len(df)): \n        input_ids_1 = tokenizer.encode(row.text,add_special_tokens=False)\n#         print(len(input_ids_1))\n        input_ids = [cls_token_idx, ] +input_ids_1 + [sep_token_idx, ]\n        token_type_ids = [0,]*len(input_ids)\n        if len(input_ids) > max_sequence_length: \n            input_ids = input_ids[:max_sequence_length]\n            input_ids[-1] = sep_token_idx\n            token_type_ids = token_type_ids[:max_sequence_length]\n        else:\n            input_ids = input_ids + [pad_token_idx, ]*(max_sequence_length - len(input_ids))\n            token_type_ids = token_type_ids + [pad_token_idx, ]*(max_sequence_length - len(token_type_ids))\n        assert len(input_ids) == len(token_type_ids)\n        outputs[idx,:max_sequence_length] = np.array(input_ids)\n        type_outputs[idx,:] = token_type_ids\n        if is_test:\n            continue\n        selected_text = row.label.strip()\n        if len(selected_text) == 0 or len(row.text) == 0:\n            start_idx, end_idx = (0,0)\n            position_outputs[idx,:] = [0, 0]\n        else:\n            if \" \"+selected_text in row.text:\n                input_ids_2 = tokenizer.encode(\" \"+selected_text,add_special_tokens=False)\n            else:\n                input_ids_2 = tokenizer.encode(selected_text,add_special_tokens=False)\n            for i in range(len(input_ids_2)):\n                start_idx, end_idx = contains(input_ids_2[:len(input_ids_2)-i], input_ids_1) #[:max_sequence_length - len(input_ids_0) - 2])\n                if start_idx is not None:\n                    if i > 1:\n                        print(input_ids_2, i)\n                    break\n            if start_idx is None:\n                start_idx = 0\n                end_idx = 0\n            position_outputs[idx,:] = [start_idx + 1, end_idx + 1]\n            if max(position_outputs[idx,:]) >= max_sequence_length:\n                position_outputs[idx,:] = 0,0\n    if is_test:\n        return outputs, type_outputs\n    else:\n        return outputs, type_outputs, position_outputs, offset_outputs, df\n    \n\ndef find_best_combinations(start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, valid_start= 0, valid_end=512):\n    best = (valid_start, valid_end - 1)\n    best_score = -9999\n#    print(valid_end, start_top_index, end_top_index)\n    for i in range(len(start_top_log_probs)):\n        for j in range(end_top_log_probs.shape[0]):\n            if valid_start <= start_top_index[i] < valid_end and valid_start <= end_top_index[j,i] < valid_end and start_top_index[i] <= end_top_index[j,i]:\n                score = start_top_log_probs[i] * end_top_log_probs[j,i]\n                if score > best_score:\n                    best = (start_top_index[i],end_top_index[j,i])\n                    best_score = score\n    return best\n\ndef jaccard_similarity(str1: str, str2: str) -> float:\n    a = set(str1.split()) \n    b = set(str2.split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.868359Z","iopub.status.busy":"2021-06-15T03:01:38.867266Z","iopub.status.idle":"2021-06-15T03:01:38.869873Z","shell.execute_reply":"2021-06-15T03:01:38.870345Z","shell.execute_reply.started":"2021-05-25T03:35:28.454566Z"},"papermill":{"duration":0.049256,"end_time":"2021-06-15T03:01:38.870497","exception":false,"start_time":"2021-06-15T03:01:38.821241","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = RobertaTokenizer.from_pretrained('../input/roberta-base-config/')\n\nwindow_size = 24\nmax_sequence_length = 256\ntest_df = pd.DataFrame()\ntexts = []\nlabels = []\nids = []\nfor idx in tqdm(empty_ids):\n    x = json.load(open(f\"../input/coleridgeinitiative-show-us-the-data/{data_dir}/{idx}.json\",\"rt\"))\n    article = \"\"\n    for section in x:\n        raw_text = \" \".join(section[\"text\"].replace(\"\\n\", \" \").split())\n        article += raw_text\n        article += \" \"\n#     article =  clean_text(article)\n    input_ids = tokenizer.encode(article, add_special_tokens=False)\n    n_samples = math.ceil(len(input_ids)/(max_sequence_length - window_size))\n    for sample_idx in range(n_samples):\n        start = max(0, (max_sequence_length - window_size)*sample_idx)\n        end = start + max_sequence_length\n        curr_ids = input_ids[start: end]\n        curr_text = tokenizer.decode(curr_ids)\n        texts.append(curr_text)\n        ids.append(idx)\ntest_df[\"id\"] = ids\ntest_df[\"text\"] = texts\ntest_df = test_df.fillna(\"\")","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:38.928578Z","iopub.status.busy":"2021-06-15T03:01:38.927964Z","iopub.status.idle":"2021-06-15T03:01:41.11461Z","shell.execute_reply":"2021-06-15T03:01:41.114169Z","shell.execute_reply.started":"2021-05-25T03:35:28.476151Z"},"papermill":{"duration":2.219752,"end_time":"2021-06-15T03:01:41.114738","exception":false,"start_time":"2021-06-15T03:01:38.894986","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp ../input/gpt-eval-script/*.py .","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:41.183042Z","iopub.status.busy":"2021-06-15T03:01:41.168412Z","iopub.status.idle":"2021-06-15T03:01:41.870691Z","shell.execute_reply":"2021-06-15T03:01:41.870184Z","shell.execute_reply.started":"2021-05-25T03:35:30.339073Z"},"papermill":{"duration":0.731539,"end_time":"2021-06-15T03:01:41.870817","exception":false,"start_time":"2021-06-15T03:01:41.139278","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunk_size = 200000\nn_chunks = math.ceil(len(test_df)/chunk_size)\n\ntokenizer = GPT2Tokenizer.from_pretrained('../input/gpt2-config/')\nconfig = GPT2Config.from_pretrained('../input/gpt2-config/', output_hidden_states=True)\nX_test, X_type_test = convert_lines(tokenizer,test_df,is_test=True, max_sequence_length=256)\nnp.save(\"X_test_gpt.npy\",X_test)\n\n# tokenizer = RobertaTokenizer.from_pretrained('../input/roberta-base-config/')\n# config = RobertaConfig.from_pretrained('../input/roberta-base-config/', output_hidden_states=True)\n# X_test, X_type_test = convert_lines(tokenizer,test_df,is_test=True)\n# np.save(\"X_test_roberta.npy\",X_test)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:41.927468Z","iopub.status.busy":"2021-06-15T03:01:41.926818Z","iopub.status.idle":"2021-06-15T03:01:43.797372Z","shell.execute_reply":"2021-06-15T03:01:43.796852Z","shell.execute_reply.started":"2021-05-25T03:35:30.985592Z"},"papermill":{"duration":1.901346,"end_time":"2021-06-15T03:01:43.797504","exception":false,"start_time":"2021-06-15T03:01:41.896158","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python eval.py --input_path ./X_test_gpt.npy --ckpt_path ../input/gpt-test/gpt2-medium_extra_data_13.bin --config_path ../input/gpt-medium-config --output_path o1.txt --model gpt --batch_size 8 --beam_size 10 --sigmoid_decoding --threshold 0.8\n!ls -halt","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:01:44.761216Z","iopub.status.busy":"2021-06-15T03:01:44.760089Z","iopub.status.idle":"2021-06-15T03:02:48.000213Z","shell.execute_reply":"2021-06-15T03:02:47.999641Z","shell.execute_reply.started":"2021-05-25T03:35:33.352339Z"},"papermill":{"duration":63.280562,"end_time":"2021-06-15T03:02:48.000354","exception":false,"start_time":"2021-06-15T03:01:44.719792","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"selected_texts = []\nfor i in [1]:\n    data = [x.strip() for x in open(f\"o{i}.txt\").readlines()]\n#     test_df[f\"preds_{i}\"] = [clean_text(x) for x in data]\n    selected_texts.extend(data)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.244597Z","iopub.status.busy":"2021-06-15T03:02:48.243832Z","iopub.status.idle":"2021-06-15T03:02:48.247525Z","shell.execute_reply":"2021-06-15T03:02:48.246896Z","shell.execute_reply.started":"2021-05-25T03:35:54.426815Z"},"papermill":{"duration":0.035715,"end_time":"2021-06-15T03:02:48.247656","exception":false,"start_time":"2021-06-15T03:02:48.211941","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"good_words = [\" Dataset \",\" Datasets \", \" Database \",\" Databases \", \" Data \", \" Survey \",\" Study \",\" Studies \",\" Surveys \"]\nbad_words = [\" are \", \" is \", \" was \", \" were \"]\ndef has_good_words(x):\n    if \".\" in x or \"!\" in x or \"?\" in x or (not x.split()[0].isalpha()):\n        return False\n    x = f\" {x} \"\n    for w in bad_words:\n        if w in x:\n            return False\n    for w in good_words:\n        if w in x:\n            return True\n    return False\ndef is_long_pred(x):\n    long_enough = len(x) > 10 and len(x) < 100 and len(x.split()) > 3 \n    return long_enough and (\",\" not in x)\n\ntemp_df = pd.DataFrame()\ntemp_df[\"preds\"] = [x for x in selected_texts if len(x) > 0]\npred_counts = temp_df.preds.value_counts()\nunique_preds =  [clean_text(x) for x in pred_counts.index[pred_counts.values > 3] if is_long_pred(x)]\nmore_preds = [clean_text(x) for x in pred_counts.index[pred_counts.values <= 3] if has_good_words(x) and is_long_pred(x)]\nunique_preds.extend(more_preds)\nunique_preds = list(set(unique_preds))\n\nblack_list = []\nfor pred in unique_preds:\n    for label in existing_labels:\n        if jaccard_similarity(pred,label) >= 0.5:\n            black_list.append(pred)\n            break\nblack_list = list(set(black_list))\n#     print(black_list, preds)\nfor x in black_list:\n    unique_preds.remove(x)\nbest_preds = list(set(unique_preds))","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.325661Z","iopub.status.busy":"2021-06-15T03:02:48.324661Z","iopub.status.idle":"2021-06-15T03:02:48.327985Z","shell.execute_reply":"2021-06-15T03:02:48.327446Z","shell.execute_reply.started":"2021-05-25T03:35:54.437446Z"},"papermill":{"duration":0.05345,"end_time":"2021-06-15T03:02:48.328128","exception":false,"start_time":"2021-06-15T03:02:48.274678","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(best_preds[:20], more_preds[:20], len(best_preds), len(more_preds))","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.387378Z","iopub.status.busy":"2021-06-15T03:02:48.386575Z","iopub.status.idle":"2021-06-15T03:02:48.390124Z","shell.execute_reply":"2021-06-15T03:02:48.390612Z","shell.execute_reply.started":"2021-05-25T03:35:54.454912Z"},"papermill":{"duration":0.035657,"end_time":"2021-06-15T03:02:48.390763","exception":false,"start_time":"2021-06-15T03:02:48.355106","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport re\n\n\ndef find_all_pred_in_text(normed_text, all_unique_preds):\n    preds = []\n    preds_indexs = []\n    for pred in all_unique_preds:\n        if pred in normed_text:\n            preds.append(pred)\n    unique_preds = []  # unique in terms of index.\n    for pred in preds:\n        matchs = re.finditer(pred, normed_text)\n        for match in matchs:\n            start_index = match.start()\n            end_index = match.end()\n            preds_indexs.append([start_index, end_index])\n            unique_preds.append(pred)\n    group_idxs = []\n    for i in range(len(preds_indexs)):\n        for j in range(len(preds_indexs)):\n            if i != j:\n                start_i, end_i = preds_indexs[i]\n                start_j, end_j = preds_indexs[j]\n                if start_i <= end_j and end_i <= end_j and start_i >= start_j:\n                    group_idxs.append([i, j])\n    unique_preds = np.array(unique_preds)\n    for group_idx in group_idxs:\n        unique_preds[group_idx[0]] = unique_preds[group_idx[1]]\n    return np.unique(unique_preds)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.455198Z","iopub.status.busy":"2021-06-15T03:02:48.454231Z","iopub.status.idle":"2021-06-15T03:02:48.457047Z","shell.execute_reply":"2021-06-15T03:02:48.456561Z","shell.execute_reply.started":"2021-05-25T03:35:54.470682Z"},"papermill":{"duration":0.039633,"end_time":"2021-06-15T03:02:48.457171","exception":false,"start_time":"2021-06-15T03:02:48.417538","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"extra_dict = dict()\ntest_df[\"orig_text\"] = test_df.text.copy()\ntest_df.text = test_df.text.apply(clean_text)\nfor idx in sample_sub[\"Id\"].unique():\n    sub_df = sample_sub[sample_sub[\"Id\"] == idx]\n    sub_texts = clean_text(sub_df.text.values[0])\n    preds = []\n    for pred in best_preds:\n        if pred in sub_texts and pred not in preds:\n            preds.append(pred)\n    black_list = []\n    preds = list(find_all_pred_in_text(sub_texts, preds))\n    for pred in preds:\n        for label in existing_labels:\n            if jaccard_similarity(pred,label) >= 0.5:\n                black_list.append(pred)\n                break\n    black_list = list(set(black_list))\n    for x in black_list:\n        preds.remove(x)\n    extra_dict[idx] = '|'.join(preds)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.54166Z","iopub.status.busy":"2021-06-15T03:02:48.536441Z","iopub.status.idle":"2021-06-15T03:02:48.632248Z","shell.execute_reply":"2021-06-15T03:02:48.631707Z","shell.execute_reply.started":"2021-05-25T03:35:54.482891Z"},"papermill":{"duration":0.14777,"end_time":"2021-06-15T03:02:48.632394","exception":false,"start_time":"2021-06-15T03:02:48.484624","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = \"\"\nsubmission['PredictionString'] = submission.apply(lambda row: extra_dict.get(row.Id, row.PredictionString),axis=1)\n# submission.to_csv(\"submission.csv\",index=False)\n# !head submission.csv","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.757486Z","iopub.status.busy":"2021-06-15T03:02:48.75669Z","iopub.status.idle":"2021-06-15T03:02:48.763786Z","shell.execute_reply":"2021-06-15T03:02:48.763324Z","shell.execute_reply.started":"2021-05-25T03:35:54.607483Z"},"papermill":{"duration":0.041948,"end_time":"2021-06-15T03:02:48.763935","exception":false,"start_time":"2021-06-15T03:02:48.721987","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_valid_acronym(label, acronym):\n    guess_acronym = ' '.join([w[0] for w in label.split()])\n    js = jaccard_similarity(guess_acronym, ' '.join([c for c in acronym.split()[0]]))\n    if js >= 0.5:\n        return True\n    return False\n\ndef is_last_word_acronym(label):\n    words = label.split()\n    last_word = words[-1]\n    label = \" \".join(words[:-1])\n    if check_valid_acronym(label, last_word):\n        return True\n    return False\n\ndef find_all_acronyms_candidates(row):\n    string = row.text\n    all_labels = row.PredictionString.split(\"|\")\n    curr_preds = row.PredictionString\n    for label in all_labels:\n        if label != \"\":\n            acronyms_candidates = re.findall(f\"{label} \\((.*?)\\)\", string)\n            acronyms_candidates = np.unique([ac for ac in acronyms_candidates if len(ac.split()) >= 1])\n            if is_last_word_acronym(label):\n                acronyms_candidates = np.unique(np.append(acronyms_candidates, label.split()[-1]))\n            if len(acronyms_candidates) > 0:\n                for ac in acronyms_candidates:\n                    index_of_label = np.array([i for i in range(len(string)) if string.startswith(f'{label} ({ac})', i)]) + len(f\"{label} (\")\n                    index_of_ac = np.array([i for i in range(len(string)) if string.startswith(f'{ac}', i)])\n                    if len(list(set(index_of_ac) - set(index_of_label))) != 0:\n                        if check_valid_acronym(label, ac):\n                            curr_preds += f\"|{ac}\"\n    curr_preds = list(set(curr_preds.split(\"|\")))\n    return \"|\".join(curr_preds)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.83205Z","iopub.status.busy":"2021-06-15T03:02:48.831032Z","iopub.status.idle":"2021-06-15T03:02:48.834375Z","shell.execute_reply":"2021-06-15T03:02:48.833898Z","shell.execute_reply.started":"2021-05-25T03:35:54.623667Z"},"papermill":{"duration":0.042875,"end_time":"2021-06-15T03:02:48.834512","exception":false,"start_time":"2021-06-15T03:02:48.791637","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub[\"text\"] = sample_sub.text.apply(clean_text_v2)\nsample_sub.PredictionString = submission.PredictionString.copy()","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:48.922185Z","iopub.status.busy":"2021-06-15T03:02:48.921253Z","iopub.status.idle":"2021-06-15T03:02:48.93081Z","shell.execute_reply":"2021-06-15T03:02:48.930306Z","shell.execute_reply.started":"2021-05-25T03:35:54.638743Z"},"papermill":{"duration":0.068978,"end_time":"2021-06-15T03:02:48.93097","exception":false,"start_time":"2021-06-15T03:02:48.861992","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.PredictionString = sample_sub.apply(find_all_acronyms_candidates,axis=1)\nsample_sub.PredictionString = sample_sub.PredictionString.apply(lambda x: \"|\".join([i for i in x.split(\"|\") if len(i.strip()) > 0]))\nprint(sample_sub.PredictionString.values[:5])","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:49.008302Z","iopub.status.busy":"2021-06-15T03:02:49.003113Z","iopub.status.idle":"2021-06-15T03:02:49.169315Z","shell.execute_reply":"2021-06-15T03:02:49.16883Z","shell.execute_reply.started":"2021-05-25T03:35:54.67276Z"},"papermill":{"duration":0.211113,"end_time":"2021-06-15T03:02:49.169449","exception":false,"start_time":"2021-06-15T03:02:48.958336","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub[[\"Id\",\"PredictionString\"]].to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.execute_input":"2021-06-15T03:02:49.230426Z","iopub.status.busy":"2021-06-15T03:02:49.229545Z","iopub.status.idle":"2021-06-15T03:02:49.348003Z","shell.execute_reply":"2021-06-15T03:02:49.347465Z","shell.execute_reply.started":"2021-05-25T03:35:55.400191Z"},"papermill":{"duration":0.15119,"end_time":"2021-06-15T03:02:49.348161","exception":false,"start_time":"2021-06-15T03:02:49.196971","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"papermill":{"duration":0.027862,"end_time":"2021-06-15T03:02:49.404186","exception":false,"start_time":"2021-06-15T03:02:49.376324","status":"completed"},"tags":[]},"execution_count":null,"outputs":[]}]}