{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"MAX_SAMPLE = None","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-17T03:59:16.226081Z","iopub.execute_input":"2021-06-17T03:59:16.226436Z","iopub.status.idle":"2021-06-17T03:59:16.230202Z","shell.execute_reply.started":"2021-06-17T03:59:16.226394Z","shell.execute_reply":"2021-06-17T03:59:16.229222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n# !pip install -q ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n# !pip install -q ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n# !pip install -q ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n\nimport os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport torch\nfrom transformers import AutoTokenizer, DataCollatorForLanguageModeling, \\\nAutoModelForMaskedLM, Trainer, TrainingArguments, pipeline, AutoConfig\n\nsns.set()\nrandom.seed(123)\nnp.random.seed(456)\n\nfrom IPython.display import clear_output\nclear_output()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:59:16.442023Z","iopub.execute_input":"2021-06-17T03:59:16.442558Z","iopub.status.idle":"2021-06-17T03:59:16.450731Z","shell.execute_reply.started":"2021-06-17T03:59:16.442525Z","shell.execute_reply":"2021-06-17T03:59:16.449892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_checkpoint = \"bert-base-cased\"\n\n# MAX_LENGTH = 64\n# OVERLAP = 20\n\n# DATASET_SYMBOL = '$' # this symbol represents a dataset name\n# NONDATA_SYMBOL = '#' # this symbol represents a non-dataset name","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:59:25.331259Z","iopub.execute_input":"2021-06-17T03:59:25.331772Z","iopub.status.idle":"2021-06-17T03:59:25.335203Z","shell.execute_reply.started":"2021-06-17T03:59:25.331739Z","shell.execute_reply":"2021-06-17T03:59:25.334414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n\ntrain = pd.read_csv(train_path)\ntrain = train[:MAX_SAMPLE]\n# Group by publication, training labels should have the same form as expected output.\n# train = train.groupby('Id').agg({\n#     'pub_title': 'first',\n#     'dataset_title': '|'.join,\n#     'dataset_label': '|'.join,\n#     'cleaned_label': '|'.join\n# }).reset_index()    \ntrain.drop_duplicates(subset='Id', keep='last', inplace=True)\n\nprint('train size: ', len(train))\ntrain.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:59:25.777555Z","iopub.execute_input":"2021-06-17T03:59:25.778233Z","iopub.status.idle":"2021-06-17T03:59:25.883050Z","shell.execute_reply.started":"2021-06-17T03:59:25.778185Z","shell.execute_reply":"2021-06-17T03:59:25.882377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n    \npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\ndef read_json_pub(filename, train_data_path=paper_train_folder, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:59:27.190090Z","iopub.execute_input":"2021-06-17T03:59:27.190609Z","iopub.status.idle":"2021-06-17T03:59:27.202848Z","shell.execute_reply.started":"2021-06-17T03:59:27.190576Z","shell.execute_reply":"2021-06-17T03:59:27.201805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"annotated_datasets = {}\nannotated_datasets['2100032a-7c33-4bff-97ef-690822c43466'] = [\n    'Cardiovascular Health Study (CHS)',\n]\n\nannotated_datasets['2f392438-e215-4169-bebf-21ac4ff253e1'] = [\n    \"OECD's online Education Database\",\n]\n\nannotated_datasets['3f316b38-1a24-45a9-8d8c-4e05a42257c6'] = [\n    'North Carolina Emergency Management Spatial Data Download',\n    'Sea, Lake, and Overland Surges from Hurricanes (SLOSH) basin models',\n]\n\nannotated_datasets['8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60'] = [\n    '2010 Nielsen Homescan Survey',\n]\n\nall_annotated_datasets = []\nfor key, val in annotated_datasets.items():\n    all_annotated_datasets = all_annotated_datasets + val \n    \nprint(len(all_annotated_datasets))\nall_annotated_datasets[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T03:59:28.371180Z","iopub.execute_input":"2021-06-17T03:59:28.371535Z","iopub.status.idle":"2021-06-17T03:59:28.381361Z","shell.execute_reply.started":"2021-06-17T03:59:28.371503Z","shell.execute_reply":"2021-06-17T03:59:28.380259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp2 = pd.read_csv('../input/coleridge-additional-gov-datasets-22000popular/data_set_800_with20000popular.csv')\ntmp3 = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-17T04:00:10.467029Z","iopub.execute_input":"2021-06-17T04:00:10.467412Z","iopub.status.idle":"2021-06-17T04:00:10.627084Z","shell.execute_reply.started":"2021-06-17T04:00:10.467373Z","shell.execute_reply":"2021-06-17T04:00:10.626154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp2 = [x for x in tmp2['title'].unique() if len(str(x).split()) > 2]\ntmp2 = [x for x in tmp2 if len(str(x).split()) < 8]\ntmp3_ = [x for x in tmp3['cleaned_label'].unique() if len(str(x).split()) > 0]\ntmp3_ += [x for x in tmp3['dataset_title'].unique()]\ntmp3 = [clean_text(x) for x in np.unique(tmp3_)]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T04:00:11.348666Z","iopub.execute_input":"2021-06-17T04:00:11.349042Z","iopub.status.idle":"2021-06-17T04:00:11.415185Z","shell.execute_reply.started":"2021-06-17T04:00:11.349014Z","shell.execute_reply":"2021-06-17T04:00:11.413853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp4 = pd.read_csv('../input/bigger-govt-dataset-list/data_set_26897.csv')\ntmp4 = [x for x in tmp4['title'].unique() if len(str(x).split()) > 2]\ntmp4 = [x for x in tmp4 if len(str(x).split()) < 8]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T04:00:31.321310Z","iopub.execute_input":"2021-06-17T04:00:31.321775Z","iopub.status.idle":"2021-06-17T04:00:31.459838Z","shell.execute_reply.started":"2021-06-17T04:00:31.321732Z","shell.execute_reply":"2021-06-17T04:00:31.459025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_datasets = np.unique(tmp2 + tmp3 + tmp4 + all_annotated_datasets)\nall_datasets = np.unique([clean_text(x) for x in all_datasets])\nprint(len(all_datasets))\nall_datasets[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T04:00:55.092153Z","iopub.execute_input":"2021-06-17T04:00:55.092548Z","iopub.status.idle":"2021-06-17T04:00:55.181002Z","shell.execute_reply.started":"2021-06-17T04:00:55.092506Z","shell.execute_reply":"2021-06-17T04:00:55.180200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# literal_preds = []\nto_append = []\ntrain['ext_cleaned_label'] = ''\nfor paper_id in tqdm(train['Id'].values):\n    to_append = [paper_id,'']\n    large_string = str(read_json_pub(paper_id))\n    clean_string = text_cleaning(large_string)\n    for query_string in all_datasets:\n        if query_string in clean_string:\n            if to_append[1]!='' and clean_text(query_string) not in to_append[1]:\n                to_append[1]=to_append[1]+'|'+clean_text(query_string)\n            if to_append[1]=='':\n                to_append[1]=clean_text(query_string)\n#     literal_preds.append(*to_append[1:])\n    train.loc[train['Id']==paper_id, 'ext_cleaned_label'] = to_append[1]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T04:01:07.013905Z","iopub.execute_input":"2021-06-17T04:01:07.014470Z","iopub.status.idle":"2021-06-17T04:01:16.357527Z","shell.execute_reply.started":"2021-06-17T04:01:07.014403Z","shell.execute_reply":"2021-06-17T04:01:16.356077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('./train_ext_data.csv', index=False)\npd.read_csv('./train_ext_data.csv').head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T04:01:19.508889Z","iopub.execute_input":"2021-06-17T04:01:19.509279Z","iopub.status.idle":"2021-06-17T04:01:19.761957Z","shell.execute_reply.started":"2021-06-17T04:01:19.509246Z","shell.execute_reply":"2021-06-17T04:01:19.761146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}