{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Asthetics\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\n# Basic\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport json\nimport os\nimport random\nfrom tqdm.autonotebook import tqdm\nimport string\nimport re\nfrom functools import partial\nfrom pprint import pprint\n\n# Visualizations\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\nfrom wordcloud import WordCloud, STOPWORDS\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os, glob, pickle, time, gc, copy, sys\nimport yaml\nimport pandas_profiling as pdp\nimport warnings\nfrom tqdm import tqdm\nimport re\nimport json\nfrom transformers import RobertaTokenizer, RobertaModel\nimport nltk\n\ntqdm.pandas() #tqdm is used to show any code running with a progress bar. \npd.set_option('display.max_columns', 100) # 表示できる列数","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T12:53:27.262201Z","iopub.execute_input":"2021-06-22T12:53:27.262629Z","iopub.status.idle":"2021-06-22T12:53:31.217974Z","shell.execute_reply.started":"2021-06-22T12:53:27.262572Z","shell.execute_reply":"2021-06-22T12:53:31.216957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!nvidia-smi","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:53:31.220344Z","iopub.execute_input":"2021-06-22T12:53:31.220626Z","iopub.status.idle":"2021-06-22T12:53:31.915934Z","shell.execute_reply.started":"2021-06-22T12:53:31.220598Z","shell.execute_reply":"2021-06-22T12:53:31.914882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -la /kaggle/input/download-spacy","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:53:31.918156Z","iopub.execute_input":"2021-06-22T12:53:31.918543Z","iopub.status.idle":"2021-06-22T12:53:32.583613Z","shell.execute_reply.started":"2021-06-22T12:53:31.918499Z","shell.execute_reply":"2021-06-22T12:53:32.582584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U spacy[cuda110] --no-index --find-links /kaggle/input/download-spacy/spacy_cuda110/ # gpu\n# !pip install -U spacy --no-index --find-links /kaggle/input/download-spacy/spacy/ # cpu","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:53:32.58561Z","iopub.execute_input":"2021-06-22T12:53:32.586004Z","iopub.status.idle":"2021-06-22T12:53:59.323507Z","shell.execute_reply.started":"2021-06-22T12:53:32.585961Z","shell.execute_reply":"2021-06-22T12:53:59.322503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install /kaggle/input/download-spacy/en_core_web_lg-3.0.0-py3-none-any.whl/en_core_web_lg-3.0.0-py3-none-any.whl\n# roberta\n!pip install spacy-transformers --no-index --find-links /kaggle/input/download-spacy/spacy-transformers\n!pip install /kaggle/input/download-spacy/en_core_web_trf-3.0.0-py3-none-any.whl/en_core_web_trf-3.0.0-py3-none-any.whl","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:53:59.326946Z","iopub.execute_input":"2021-06-22T12:53:59.327223Z","iopub.status.idle":"2021-06-22T12:55:27.931753Z","shell.execute_reply.started":"2021-06-22T12:53:59.327193Z","shell.execute_reply":"2021-06-22T12:55:27.930629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nprint(spacy.prefer_gpu())\n# nlp = spacy.load('en_core_web_lg') # CPU optimized, not transformer\n# nlp = spacy.load('en_core_web_trf') # RoBERTa\n# finetuned_ner_nlp = spacy.load(\"../input/spacy-ner-aug-lexeme/model-best\") # lexeme table + data augmentation using the gov data\nfinetuned_ner_nlp = spacy.load(\"../input/spacynerlexeme0620/model-best\") # added lexeme table","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:55:27.940691Z","iopub.execute_input":"2021-06-22T12:55:27.948631Z","iopub.status.idle":"2021-06-22T12:55:58.77982Z","shell.execute_reply.started":"2021-06-22T12:55:27.948576Z","shell.execute_reply":"2021-06-22T12:55:58.7789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path, output='text'):\n \"\"\"\n Function to read json file and then return the text data from them and append to the dataframe\n \"\"\"\n json_path = os.path.join(train_files_path, (filename+'.json'))\n headings = []\n contents = []\n combined = []\n with open(json_path, 'r') as f:\n json_decode = json.load(f)\n for data in json_decode:\n headings.append(data.get('section_title'))\n contents.append(data.get('text'))\n combined.append(data.get('section_title'))\n combined.append(data.get('text'))\n \n all_headings = ' '.join(headings)\n all_contents = ' '.join(contents)\n all_data = '. '.join(combined)\n \n if output == 'text':\n return all_contents\n elif output == 'head':\n return all_headings\n else:\n return all_data\n \ndef clean_text(txt):\n return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef jaccard(str1, str2): \n a = set(str1.lower().split()) \n b = set(str2.lower().split())\n c = a.intersection(b)\n return float(len(c)) / (len(a) + len(b) - len(c))\n\n\ndef calc_score(y_true, y_pred, beta=0.5):\n y_true = y_true.split('|')\n y_pred = y_pred.split('|')\n# print(y_true)\n# print(y_pred)\n TP = 0\n FP = 0\n FN = 0\n for i in range(len(y_pred)):\n FP += 1\n for j in range(len(y_true)):\n if jaccard(y_true[j], y_pred[i])>=0.5:\n FP -= 1\n break\n for i in range(len(y_true)):\n FN += 1\n for j in range(len(y_pred)):\n if jaccard(y_true[i], y_pred[j])>=0.5:\n FN -= 1\n TP += 1\n break\n return TP, FP, FN\n\ndef detect_duplicated(x):\n for i in range(len(df_train_reduced)):\n if x==df_train_reduced['text'][i]:\n return df_train_reduced['Id'][i]\n return 'no dup'\n\ndef pickle_save(path, df):\n with open(path, 'wb') as f:\n pickle.dump(df, f)\n\ndef pickle_load(path):\n with open(path, 'rb') as f:\n df = pickle.load(f)\n return df\n\ndef ri(df):\n return df.reset_index(drop=True)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:55:58.781172Z","iopub.execute_input":"2021-06-22T12:55:58.781515Z","iopub.status.idle":"2021-06-22T12:55:58.797741Z","shell.execute_reply.started":"2021-06-22T12:55:58.78148Z","shell.execute_reply":"2021-06-22T12:55:58.796873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def det_acronym_ver1(text, keywords, TH_LEN_CHAR = 3):\n ans = []\n # text = re.sub(\"-\", \" \", text)\n words = text.split()\n for i, word in enumerate(words):\n if word[0]!='(' or word[-1]!=')': continue # (XXX)の形でなければスルー\n acronym_cand = word[1:-1]\n if acronym_cand.lower()==acronym_cand: continue # 大文字が一つもないならスルー\n len_acronym_cand = len(acronym_cand)\n if len_acronym_cand0:\n df_tmps = pd.DataFrame(df_tmps, columns=['chunk'])\n df_tmps['sentence_idx'] = i\n df_ner.append(df_tmps)\n if len(df_ner)>0:\n df_ner = pd.concat(df_ner).reset_index(drop=True)\n else:\n df_ner = pd.DataFrame(columns=['chunk'])\n return df_ner\n\ndef monitor_loop(i, len_data, start, verbose):\n if (i+1)%verbose==0:\n print(\"{}/{}, sec: {:.1f}\".format(i+1, len_data, time.time()-starttime))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:55:58.816268Z","iopub.execute_input":"2021-06-22T12:55:58.816805Z","iopub.status.idle":"2021-06-22T12:55:58.833394Z","shell.execute_reply.started":"2021-06-22T12:55:58.816759Z","shell.execute_reply":"2021-06-22T12:55:58.83258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def check_no_capital(x):\n try:\n if x[1:]==x[1:].lower():\n return True\n return False\n except:\n return False\n\ndef delete_the(x):\n try:\n x_split = x.split()\n if x_split[0]=='the':\n return ' '.join(x_split[1:])\n return x\n except:\n return x\n\ndef check_including_acronym(x):\n try:\n x_split = re.split(\"[ ']\" , x)\n for word in x_split:\n if word.upper()==word and word.lower()!=word: # 大文字だけの単語があるか\n return True\n return False\n except:\n return True \n \ndef get_match(x, ref_labels, threshold=0.5):\n for label in ref_labels:\n if jaccard(x, label)>=threshold:\n return label\n return 'no_match'\n\ndef jaccard(str1, str2): \n a = set(str1.lower().split()) \n b = set(str2.lower().split())\n c = a.intersection(b)\n if len(a)==0 or len(b)==0:\n return 0\n return float(len(c)) / (len(a) + len(b) - len(c))\n\ndef delete_keywords(x, ref):\n try:\n x_split = x.split()\n x_new = []\n for item in x_split:\n if item.lower() not in ref:\n x_new.append(item)\n x_new = ' '.join(x_new)\n return x_new\n except:\n return ''\n\n\ndef check_head_capital(x):\n try:\n if len(x)==0:\n return False\n return x[0]==x[0].upper() and x[0]!=x[0].lower() \n except:\n return False\n\ndef detect_black(x, ref):\n try:\n for item in ref:\n if item in x:\n return True\n return False\n except:\n return True\n \ndef detect_label(x, ref_label, ref_target):\n try:\n predict = []\n for i in range(len(ref_label)):\n if ref_label[i] in x:\n predict.append(ref_target[i])\n return np.array(predict, np.int64)\n except:\n return np.array([], np.int64)\n \ndef get_label(x, ref_label, ref_target):\n try:\n predict = []\n for i in range(len(x)):\n predict.append(ref_label[ref_target==x[i]][0])\n predict = np.unique(predict).tolist()\n predict = '|'.join(predict)\n return predict\n except:\n return ''\n \ndef detect_acronym(x, ref_label, ref_acronym, ref_target, th=[1,2]):\n try:\n ans = []\n for i in range(len(ref_label)):\n if x.count(ref_label[i])>=th[0] and x.count(ref_acronym[i])>=th[1]:\n ans.append(ref_target[i])\n # print(i)\n return ans\n except:\n return []\n \ndef detect_keywords(x, ref):\n try:\n for keyword in ref:\n if keyword in x:\n return True\n return False\n except:\n return False","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:55:58.834968Z","iopub.execute_input":"2021-06-22T12:55:58.83535Z","iopub.status.idle":"2021-06-22T12:55:58.857449Z","shell.execute_reply.started":"2021-06-22T12:55:58.835312Z","shell.execute_reply":"2021-06-22T12:55:58.856518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keywords = [\n 'study',\n 'studies',\n 'data',\n 'survey',\n 'panel',\n 'census',\n 'cohort',\n 'longitudinal',\n 'registry', \n]\n\nkeywords2 = [\n 'study',\n 'studies',\n 'data',\n 'survey',\n 'panel',\n 'census',\n 'cohort',\n 'longitudinal',\n 'registry',\n 'the',\n]\nkeywords3 = [\n 'study',\n 'studies',\n 'dataset',\n 'database',\n 'survey',\n 'panel',\n 'census',\n 'cohort',\n 'longitudinal',\n 'registry',\n]\nkeywords4 = [\n 'system',\n 'center',\n 'centre',\n 'committee',\n 'documentation',\n 'entry',\n 'assimilation',\n 'explorer',\n 'regulation',\n 'portal',\n 'format',\n 'data science',\n 'analysis',\n 'management',\n 'agreement',\n 'branch',\n 'acquisition',\n 'request',\n 'task force',\n 'program',\n 'operator',\n 'office',\n 'data view',\n 'data language',\n 'mission',\n 'alliance',\n 'data model',\n 'data structure',\n 'corporation',\n]\n\nwhite_list = [\n 'ipeds',\n]\n\nkeywords5 = keywords + ['of', 'the', 'national', 'education']\n\nng_list = [\n 'national longitudinal survey',\n 'education longitudinal survey',\n 'census bureau',\n 'data appendix',\n 'data file user',\n 'supplementary data',\n 'data supplement',\n 'major field of study'\n]\nblack_list = [\n 'USGS',\n 'GWAS',\n 'ECLS',\n 'DAS',\n 'NCDC',\n 'NDBC',\n 'UDS',\n 'GTD',\n 'ISC',\n 'DGP',\n 'EDC',\n 'FDA',\n 'TSE',\n 'DEA',\n 'CDA',\n 'IDB',\n 'NGDC',\n 'JODC',\n 'EDM',\n 'FADN',\n 'LRD',\n 'DBDM',\n 'DMC',\n 'WSC',\n ###count4##\n]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:55:58.858595Z","iopub.execute_input":"2021-06-22T12:55:58.859227Z","iopub.status.idle":"2021-06-22T12:55:58.87235Z","shell.execute_reply.started":"2021-06-22T12:55:58.85918Z","shell.execute_reply":"2021-06-22T12:55:58.871288Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data loading","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\nprint(df_train.shape)\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:55:58.873685Z","iopub.execute_input":"2021-06-22T12:55:58.87406Z","iopub.status.idle":"2021-06-22T12:55:59.027496Z","shell.execute_reply.started":"2021-06-22T12:55:58.874021Z","shell.execute_reply":"2021-06-22T12:55:59.026641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_reduced = pd.read_csv(\"../input/coleridge-ext/df_train_reduced.csv\")\nprint(df_train_reduced.shape)\ndf_train_reduced.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:55:59.028897Z","iopub.execute_input":"2021-06-22T12:55:59.029279Z","iopub.status.idle":"2021-06-22T12:56:20.522646Z","shell.execute_reply.started":"2021-06-22T12:55:59.02924Z","shell.execute_reply":"2021-06-22T12:56:20.521778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\nprint(df_test.shape)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:20.525462Z","iopub.execute_input":"2021-06-22T12:56:20.525734Z","iopub.status.idle":"2021-06-22T12:56:20.547802Z","shell.execute_reply.started":"2021-06-22T12:56:20.525706Z","shell.execute_reply":"2021-06-22T12:56:20.546913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files_path = \"../input/coleridgeinitiative-show-us-the-data/test\"\ndf_test['text'] = df_test['Id'].progress_apply(lambda x: read_append_return(x, train_files_path=test_files_path))\ndf_test['clean_text'] = df_test['text'].progress_apply(clean_text)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:20.54921Z","iopub.execute_input":"2021-06-22T12:56:20.549633Z","iopub.status.idle":"2021-06-22T12:56:20.6289Z","shell.execute_reply.started":"2021-06-22T12:56:20.549592Z","shell.execute_reply":"2021-06-22T12:56:20.627764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# find train data in test data\ndf_test['dup_id'] = df_test['text'].progress_apply(lambda x: detect_duplicated(x))\ndf_test['dup'] = df_test['dup_id']!='no dup'\nif len(df_test)==4:\n df_test['dup'][0] = False\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:20.630562Z","iopub.execute_input":"2021-06-22T12:56:20.630959Z","iopub.status.idle":"2021-06-22T12:56:20.967323Z","shell.execute_reply.started":"2021-06-22T12:56:20.630917Z","shell.execute_reply":"2021-06-22T12:56:20.966349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_traintest = pd.concat([df_train_reduced, df_test[df_test['dup']==False]]).reset_index(drop=True)\nprint(df_traintest.shape)\ndf_traintest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:20.9688Z","iopub.execute_input":"2021-06-22T12:56:20.96946Z","iopub.status.idle":"2021-06-22T12:56:21.014074Z","shell.execute_reply.started":"2021-06-22T12:56:20.969398Z","shell.execute_reply":"2021-06-22T12:56:21.013065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(df_test)==4:\n df_test_reduced = df_train_reduced.iloc[:1000][['Id', 'text', 'clean_text']].copy()\n df_test_reduced['text'].iloc[-1] = \" \"\n df_test_reduced['clean_text'].iloc[-1] = \" \"\nelse:\n df_test_reduced = df_test[df_test['dup']==False].reset_index(drop=True).iloc[:][['Id', 'text', 'clean_text']]\n\nprint(df_test_reduced.shape)\ndf_test_reduced.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:21.015586Z","iopub.execute_input":"2021-06-22T12:56:21.01597Z","iopub.status.idle":"2021-06-22T12:56:21.03495Z","shell.execute_reply.started":"2021-06-22T12:56:21.015928Z","shell.execute_reply":"2021-06-22T12:56:21.033769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(df_test)==4:\n df_traintest = df_train_reduced.iloc[:100].copy()\n# df_traintest['text'].iloc[-1] = \"\"\n# df_traintest['clean_text'].iloc[-1] = \"\"\n# df_traintest = df_train_reduced\nprint(df_traintest.shape)\ndf_traintest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:21.036685Z","iopub.execute_input":"2021-06-22T12:56:21.037109Z","iopub.status.idle":"2021-06-22T12:56:21.057614Z","shell.execute_reply.started":"2021-06-22T12:56:21.037065Z","shell.execute_reply":"2021-06-22T12:56:21.056622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train label","metadata":{}},{"cell_type":"code","source":"df_label = df_train['cleaned_label'].value_counts().reset_index()\ndf_label.columns = ['cleaned_label', 'count']\ndf_label['-count'] = -df_label['count']\ndf_label = df_label.sort_values(['-count', 'cleaned_label']).reset_index(drop=True)\ndf_label['target'] = np.arange(len(df_label))\ndf_label['cleaned_label'] = df_label['cleaned_label'].apply(clean_text)\n\ndf_tmp1 = df_train[['dataset_label', 'cleaned_label', 'dataset_title']]\ndf_tmp2 = df_train[['dataset_title', 'cleaned_label', 'dataset_title']]\ndf_tmp3 = df_train[['cleaned_label', 'cleaned_label', 'dataset_title']]\ndf_tmp1.columns = ['label', 'cleaned_label', 'dataset_title']\ndf_tmp2.columns = ['label', 'cleaned_label', 'dataset_title']\ndf_tmp3.columns = ['label', 'cleaned_label', 'dataset_title']\ndf_label2 = pd.concat([df_tmp1, df_tmp2, df_tmp3])\n\ndf_label2['label'] = df_label2['label'].apply(lambda x: x.lower())\ndf_label2['cleaned_label_my'] = df_label2['label'].apply(clean_text)\ndf_label2['cleaned_label'] = df_label2['cleaned_label'].apply(clean_text)\ndf_label2 = df_label2[df_label2['label'].duplicated()==False].reset_index(drop=True)\ndf_label2 = pd.merge(df_label2, df_label, on='cleaned_label', how='left')\nprint(df_label2.shape)\nprint(np.sum(pd.isna(df_label2['target'])))\n# print(df_label2[df_label2['target']==1])\ndf_label2 = df_label2.sort_values(['-count', 'label']).reset_index(drop=True)\ndf_label2['target_relabeled'] = df_label2['target']\ndf_label2['target_relabeled'][df_label2['cleaned_label']!=df_label2['cleaned_label_my']] =\\\n np.arange(np.sum(df_label2['cleaned_label']!=df_label2['cleaned_label_my'])) + df_label2['target'].max()+1\nprint(\"not match: \", np.sum(df_label2['cleaned_label']!=df_label2['cleaned_label_my']))\nprint(\"unique cleaned_label_my: \", len(df_label2['cleaned_label_my'].unique()))\nprint(df_label2['target'].max())\nprint(df_label2['target_relabeled'].max())\ndf_label_train = df_label2\ndf_label_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:21.059133Z","iopub.execute_input":"2021-06-22T12:56:21.059523Z","iopub.status.idle":"2021-06-22T12:56:21.59434Z","shell.execute_reply.started":"2021-06-22T12:56:21.059485Z","shell.execute_reply":"2021-06-22T12:56:21.593552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label_train['label-keywords'] = df_label_train['cleaned_label_my'].apply(\n lambda x: delete_keywords(x, keywords5))\ndf_label_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:21.596356Z","iopub.execute_input":"2021-06-22T12:56:21.59675Z","iopub.status.idle":"2021-06-22T12:56:21.613815Z","shell.execute_reply.started":"2021-06-22T12:56:21.596709Z","shell.execute_reply":"2021-06-22T12:56:21.612751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['target'] = df_train['cleaned_label'].progress_apply(lambda x: df_label2['target'][df_label2['cleaned_label']==x.strip()].values[0])\ndf_tmp = df_train.groupby('Id')['target'].agg(lambda x: list(x)).reset_index()\ndf_tmp.columns = ['Id', 'targets']\ndf_tmp['targets'] = df_tmp['targets'].apply(lambda x: np.array([x]).flatten())\ndf_train_reduced2 = pd.merge(df_train_reduced, df_tmp, on='Id', how='left')\ndf_train_reduced2.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:21.615513Z","iopub.execute_input":"2021-06-22T12:56:21.615993Z","iopub.status.idle":"2021-06-22T12:56:26.432612Z","shell.execute_reply.started":"2021-06-22T12:56:21.615949Z","shell.execute_reply":"2021-06-22T12:56:26.431564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train label acronym","metadata":{}},{"cell_type":"code","source":"acronym_list = [\n ['National Education Longitudinal Study', 'nels'],\n# ['NOAA Tide Gauge', 'nan'],\n ['Sea, Lake, and Overland Surges from Hurricanes', 'slosh'],\n ['Coastal Change Analysis Program', 'c-cap'],\n ['Aging Integrated Database (AGID)', 'agid'],\n [\"Alzheimer's Disease Neuroimaging Initiative (ADNI)\", 'adni'],\n ['Baltimore Longitudinal Study of Aging (BLSA)', 'blsa'],\n ['Agricultural Resource Management Survey', 'arms'],\n ['Beginning Postsecondary Student', 'bps'],\n [\"The National Institute on Aging Genetics of Alzheimer's Disease Data Storage Site (NIAGADS)\", 'niagads'],\n ['Common Core of Data', 'ccd'],\n# ['Survey of Industrial Research and Development', 'nan'],\n ['Baccalaureate and Beyond', 'b&b'],\n ['International Best Track Archive for Climate Stewardship', 'IBTrACS'],\n ['National Teacher and Principal Survey', 'ntps'],\n ['Higher Education Research and Development Survey', 'herd'],\n ['Survey of Earned Doctorates', 'sed'],\n ['School Survey on Crime and Safety', 'ssocs'],\n ['World Ocean Database', 'wod'],\n ['Program for the International Assessment of Adult Competencies', 'piaac'],\n ['Early Childhood Longitudinal Study', 'ecls'],\n# ['Survey of Graduate Students and Postdoctorates in Science and Engineering', 'nan'],\n ['Trends in International Mathematics and Science Study', 'timss'],\n ['Education Longitudinal Study', 'els'],\n ['Optimum Interpolation Sea Surface Temperature', 'oisst'],\n ['National Assessment of Education Progress', 'naep'],\n ['High School Longitudinal Study', 'hsls'],\n ['Survey of Doctorate Recipients', 'sdr'],\n ['Rural-Urban Continuum Codes', 'rucc'],\n# ['Survey of Science and Engineering Research Facilities', 'nan'],\n# ['FFRDC Research and Development Survey', 'nan'],\n# ['Survey of State Government Research and Development', 'nan'],\n ['Advanced National Seismic System (ANSS) Comprehensive Catalog (ComCat)', 'comcat'],\n# ['Census of Agriculture', 'nan'],\n ['North American Breeding Bird Survey (BBS)', 'bbs'],\n ['COVID-19 Open Research Dataset (CORD-19)', 'cord-19'],\n ['Complexity Science Hub COVID-19 Control Strategies List (CCCSL)', 'cccsl'],\n# ['Our World in Data COVID-19 dataset', 'nan'],\n ['COVID-19 Precision Medicine Analytics Platform Registry (JH-CROWN)', 'jh-crown'],\n ['Characterizing Health Associated Risks, and Your Baseline Disease In SARS-COV-2 (CHARYBDIS)', 'charybdis'],\n# ['COVID-19 Deaths data', 'nan'],\n# ['SARS-CoV-2 genome sequence', 'nan'],\n# ['COVID-19 Image Data Collection', 'nan'],\n ['RSNA International COVID-19 Open Radiology Database (RICORD)', 'ricord'],\n# ['CAS COVID-19 antiviral candidate compounds dataset', 'nan'],\n]\ndf_label_train_acronym = pd.DataFrame(acronym_list, columns=['dataset_title', 'acronym'])\ndf_label_train_acronym['acronym_clean'] = df_label_train_acronym['acronym'].apply(clean_text)\ndf_label_train_acronym['target'] = np.arange(len(df_label_train_acronym))+df_label_train['target'].max()+1\ndf_label_train_acronym","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:26.437838Z","iopub.execute_input":"2021-06-22T12:56:26.438109Z","iopub.status.idle":"2021-06-22T12:56:26.463613Z","shell.execute_reply.started":"2021-06-22T12:56:26.438081Z","shell.execute_reply":"2021-06-22T12:56:26.462382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TH_LEN_SELF = 4\ndf_label_train_acronym2 = pd.merge(df_label_train_acronym, df_label_train[['cleaned_label_my', 'dataset_title']], on='dataset_title', how='left')\nprint(df_label_train_acronym2.shape)\ndf_label_train_acronym2['base'] = df_label_train_acronym2['cleaned_label_my']\ndf_label_train_acronym2['label'] = df_label_train_acronym2['acronym_clean']\ndf_label_train_acronym2['len'] = df_label_train_acronym2['acronym_clean'].apply(lambda x: len(x))\ndf_label_train_acronym2 = ri(df_label_train_acronym2[df_label_train_acronym2['len']>=TH_LEN_SELF])\nprint(df_label_train_acronym2.shape)\ndf_label_train_acronym2.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:26.467333Z","iopub.execute_input":"2021-06-22T12:56:26.46775Z","iopub.status.idle":"2021-06-22T12:56:26.497499Z","shell.execute_reply.started":"2021-06-22T12:56:26.46771Z","shell.execute_reply":"2021-06-22T12:56:26.496563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label_train_ref = df_label_train.copy()\nprint(df_label_train_ref['dataset_title'].unique().shape)\ndf_agg = df_label_train_ref.groupby('dataset_title')['target_relabeled'].agg(min).reset_index()\ndf_agg.columns = ['dataset_title', 'target_group']\ndf_label_train_ref = pd.merge(df_label_train_ref, df_agg, on='dataset_title', how='left')\ndf_label_train_ref = df_label_train_ref[df_label_train_ref['dataset_title']!='National Education Longitudinal Study']\ndf_label_train_ref = df_label_train_ref[df_label_train_ref['dataset_title']!='Education Longitudinal Study']\ndf_label_train_ref = ri(df_label_train_ref)\nprint(df_label_train_ref['target_group'].unique().shape)\nprint(df_label_train_ref.shape)\ndf_label_train_ref.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:26.498995Z","iopub.execute_input":"2021-06-22T12:56:26.499497Z","iopub.status.idle":"2021-06-22T12:56:26.535459Z","shell.execute_reply.started":"2021-06-22T12:56:26.499451Z","shell.execute_reply":"2021-06-22T12:56:26.534499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detect Acronym ver1","metadata":{}},{"cell_type":"code","source":"df_test_reduced['det_acronym'] = df_test_reduced['text'].progress_apply(lambda x: det_acronym_ver1(x, keywords))\ndf_test_reduced.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:26.537021Z","iopub.execute_input":"2021-06-22T12:56:26.537642Z","iopub.status.idle":"2021-06-22T12:56:27.010523Z","shell.execute_reply.started":"2021-06-22T12:56:26.537602Z","shell.execute_reply":"2021-06-22T12:56:27.009487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = []\nfor i in range(len(df_test_reduced)):\n if df_test_reduced['det_acronym'][i] is not None:\n tmp += df_test_reduced['det_acronym'][i]\ndf_label_acronym_ver1_test = pd.DataFrame(tmp)\ndf_label_acronym_ver1_test.columns = ['acronym', 'base']\ndf_label_acronym_ver1_test['base_acronym'] = df_label_acronym_ver1_test['base']+\"|\"+df_label_acronym_ver1_test['acronym']\n\ndf_label_acronym_ver1_test_tmp = df_label_acronym_ver1_test[df_label_acronym_ver1_test['base_acronym'].duplicated()==False]\ndf_agg = df_label_acronym_ver1_test.groupby('base_acronym')['base'].agg(len).reset_index()\ncols_tmp = df_agg.columns.tolist()\ncols_tmp[-1] = 'count_BA'\ndf_agg.columns = cols_tmp\ndf_label_acronym_ver1_test = pd.merge(df_label_acronym_ver1_test_tmp, df_agg, on='base_acronym', how='left')\n\nprint(df_label_acronym_ver1_test.shape)\n# BA unique 325\ndf_label_acronym_ver1_test.head(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:27.012057Z","iopub.execute_input":"2021-06-22T12:56:27.012429Z","iopub.status.idle":"2021-06-22T12:56:27.065042Z","shell.execute_reply.started":"2021-06-22T12:56:27.012389Z","shell.execute_reply":"2021-06-22T12:56:27.064158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label_acronym_ver1_train = pd.read_csv(\"../input/coleridge-ext/df_label_acronym_ver1_all_210619_02.csv\")\nprint(df_label_acronym_ver1_train.shape)\ndf_label_acronym_ver1_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:27.06635Z","iopub.execute_input":"2021-06-22T12:56:27.066735Z","iopub.status.idle":"2021-06-22T12:56:27.09946Z","shell.execute_reply.started":"2021-06-22T12:56:27.066695Z","shell.execute_reply":"2021-06-22T12:56:27.098462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(df_test)==4:\n df_label_acronym_ver1_test['acronym'].iloc[-1] = 'DUMMY'\n df_label_acronym_ver1_test['base'].iloc[-1] = 'dummy'\n df_label_acronym_ver1_test['base_acronym'].iloc[-1] = 'dummy|DUMMY'\ndf_label_acronym_ver1_test['test'] = True\ndf_tmp = df_label_acronym_ver1_train.copy()\ndf_tmp['count_BA_train'] = df_tmp['count_BA']\ndf_label_acronym_ver1_test_tmp = pd.merge(df_label_acronym_ver1_test, df_tmp[['base_acronym', 'count_BA_train']], on='base_acronym', how='left')\nprint(df_label_acronym_ver1_test_tmp.shape)\ndf_label_acronym_ver1_train_tmp = pd.merge(df_label_acronym_ver1_train, df_label_acronym_ver1_test[['base_acronym', 'test']], on='base_acronym', how='left')\ndf_label_acronym_ver1_train_tmp = ri(df_label_acronym_ver1_train_tmp[df_label_acronym_ver1_train_tmp['test']!=True])\nprint(df_label_acronym_ver1_train_tmp.shape)\ndf_label_acronym_ver1 = ri(pd.concat([df_label_acronym_ver1_test_tmp, df_label_acronym_ver1_train_tmp], axis=0))\nprint(df_label_acronym_ver1.shape)\ndf_label_acronym_ver1['cleaned_label'] = df_label_acronym_ver1['base'].apply(clean_text)\ndf_label_acronym_ver1['target'] = np.arange(len(df_label_acronym_ver1))\\\n +df_label_train_acronym['target'].max()+1\ndf_label_acronym_ver1.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:27.100935Z","iopub.execute_input":"2021-06-22T12:56:27.101288Z","iopub.status.idle":"2021-06-22T12:56:27.150457Z","shell.execute_reply.started":"2021-06-22T12:56:27.101251Z","shell.execute_reply":"2021-06-22T12:56:27.149326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label_acronym_ver1['match_train'] = df_label_acronym_ver1['base'].apply(\n get_match, ref_labels=df_label_train['cleaned_label_my'])\ndf_label_acronym_ver1['label-keywords'] = df_label_acronym_ver1['base'].apply(\n lambda x: delete_keywords(x, keywords5))\ndf_label_acronym_ver1['len_label-keywords'] = df_label_acronym_ver1['label-keywords'].apply(\n lambda x: len(x.split()))\ndf_label_acronym_ver1['match_train-keywords'] = df_label_acronym_ver1['label-keywords'].apply(lambda x:\n get_match(x, ref_labels=df_label_train['label-keywords'], threshold=0.75))\ndf_label_acronym_ver1['acronym_clean'] = df_label_acronym_ver1['acronym'].apply(clean_text)\ndf_label_acronym_ver1['match_train_acronym'] = df_label_acronym_ver1['acronym_clean'].apply(\n lambda x: x in df_label_train_acronym['acronym_clean'].tolist()\n)\ndf_label_acronym_ver1['keyword3'] = df_label_acronym_ver1['base'].apply(lambda x: detect_keywords(x.lower(), keywords3))\ndf_label_acronym_ver1['keyword4'] = df_label_acronym_ver1['base'].apply(lambda x: detect_keywords(x.lower(), keywords4))\ndf_label_acronym_ver1['black'] = df_label_acronym_ver1['acronym'].apply(lambda x: x.upper() in black_list)\ndf_label_acronym_ver1['white'] = df_label_acronym_ver1['acronym_clean'].apply(lambda x: x in white_list)\n\ndf_label_acronym_ver1.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:27.152073Z","iopub.execute_input":"2021-06-22T12:56:27.152497Z","iopub.status.idle":"2021-06-22T12:56:27.533964Z","shell.execute_reply.started":"2021-06-22T12:56:27.152456Z","shell.execute_reply":"2021-06-22T12:56:27.532873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_tmp = (df_label_acronym_ver1['len_label-keywords']>0)\nidx_tmp = idx_tmp & (df_label_acronym_ver1['match_train-keywords']=='no_match')\nidx_tmp = idx_tmp & (df_label_acronym_ver1['match_train_acronym']==False)\n# idx_tmp = idx_tmp & (df_label_acronym_ver1['black']==False)\nidx_tmp = idx_tmp & ((df_label_acronym_ver1['keyword3'])|(df_label_acronym_ver1['keyword4']==False))\nidx_tmp = idx_tmp | (df_label_acronym_ver1['white'])\nidx_tmp = idx_tmp & (df_label_acronym_ver1['cleaned_label'].duplicated()==False)\ndf_label_acronym_ver1_2 =ri(df_label_acronym_ver1[idx_tmp])\nprint(df_label_acronym_ver1.shape)\nprint(df_label_acronym_ver1_2.shape)\nprint(df_label_acronym_ver1_2[df_label_acronym_ver1_2['cleaned_label'].duplicated()==False].shape)\ndf_label_acronym_ver1_2.head(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:27.53568Z","iopub.execute_input":"2021-06-22T12:56:27.536093Z","iopub.status.idle":"2021-06-22T12:56:27.594094Z","shell.execute_reply.started":"2021-06-22T12:56:27.536052Z","shell.execute_reply":"2021-06-22T12:56:27.592739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_traintest['det_acronym'] = df_traintest['clean_text'].progress_apply(lambda x: \n detect_acronym(x, df_label_acronym_ver1_2['cleaned_label'], \n df_label_acronym_ver1_2['acronym_clean'], df_label_acronym_ver1_2['target'])\n)\ndf_traintest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:27.596239Z","iopub.execute_input":"2021-06-22T12:56:27.596907Z","iopub.status.idle":"2021-06-22T12:56:28.178428Z","shell.execute_reply.started":"2021-06-22T12:56:27.596853Z","shell.execute_reply":"2021-06-22T12:56:28.177458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = df_traintest['det_acronym'].values\ntmp = np.concatenate(tmp)\n\ndf_label_acronym_ver1_2['count'] = df_label_acronym_ver1_2['target'].apply(lambda x: np.sum(tmp==x))\ndf_label_acronym_ver1_2 = df_label_acronym_ver1_2.sort_values('count', ascending=False)\ndf_label_acronym_ver1_2","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:28.17981Z","iopub.execute_input":"2021-06-22T12:56:28.180188Z","iopub.status.idle":"2021-06-22T12:56:28.226953Z","shell.execute_reply.started":"2021-06-22T12:56:28.180146Z","shell.execute_reply":"2021-06-22T12:56:28.225845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if len(df_test)==4:\n TH_COUNT = 0 # 1 is best for private\nelse:\n TH_COUNT = 1\nidx_tmp = (df_label_acronym_ver1_2['count']>=TH_COUNT)\ndf_label_acronym_ver1_black =ri(df_label_acronym_ver1_2[idx_tmp])\nidx_tmp = (df_label_acronym_ver1_2['count']>=TH_COUNT)\nidx_tmp = idx_tmp & (df_label_acronym_ver1_2['black']==False)\ndf_label_acronym_ver1_3 =ri(df_label_acronym_ver1_2[idx_tmp])\nprint(df_label_acronym_ver1.shape)\nprint(df_label_acronym_ver1_2.shape)\nprint(df_label_acronym_ver1_black.shape)\nprint(df_label_acronym_ver1_3.shape)\ndf_label_acronym_ver1_3.head(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:28.228411Z","iopub.execute_input":"2021-06-22T12:56:28.228804Z","iopub.status.idle":"2021-06-22T12:56:28.282596Z","shell.execute_reply.started":"2021-06-22T12:56:28.228764Z","shell.execute_reply":"2021-06-22T12:56:28.281555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Detect Acronym ver1 self","metadata":{}},{"cell_type":"code","source":"TH_COUNT = 1 # 1 is best for private\ndf_label_acronym_ver1_self = df_label_acronym_ver1_3.drop('target', axis=1)\ndf_tmp = ri(df_label_acronym_ver1_self[df_label_acronym_ver1_self['acronym_clean'].duplicated()==False])\ndf_tmp['target'] = np.arange(len(df_tmp)) + df_label_acronym_ver1_3['target'].max()+1\ndf_label_acronym_ver1_self = pd.merge(\n df_label_acronym_ver1_self, df_tmp[['acronym_clean', 'target']], on='acronym_clean', how='left')\nprint(df_label_acronym_ver1.shape)\nprint(df_label_acronym_ver1_self.shape)\nprint(df_label_acronym_ver1_self[df_label_acronym_ver1_self['cleaned_label'].duplicated()==False].shape)\ndf_label_acronym_ver1_self.head(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:28.284365Z","iopub.execute_input":"2021-06-22T12:56:28.285149Z","iopub.status.idle":"2021-06-22T12:56:28.352395Z","shell.execute_reply.started":"2021-06-22T12:56:28.285104Z","shell.execute_reply":"2021-06-22T12:56:28.351417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Govt","metadata":{}},{"cell_type":"code","source":"df_govt = pd.read_csv(\"../input/coleridge-ext/datasets_govt.csv\")\nprint(df_govt.shape)\ndf_govt.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:28.35369Z","iopub.execute_input":"2021-06-22T12:56:28.354246Z","iopub.status.idle":"2021-06-22T12:56:28.585332Z","shell.execute_reply.started":"2021-06-22T12:56:28.354201Z","shell.execute_reply":"2021-06-22T12:56:28.584503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_govt['keyword'] = df_govt['title'].progress_apply(lambda x: detect_keywords(x, keywords))\nidx_tmp = df_govt['keyword']\ndf_govt2 = ri(df_govt[df_govt['keyword']])\ndf_govt2['cleaned_label'] = df_govt2['title'].progress_apply(clean_text)\ndf_govt2['match_train'] = df_govt2['cleaned_label'].progress_apply(\n get_match, ref_labels=df_label_train['cleaned_label_my'])\n\nidx_tmp = df_govt2['keyword']\nprint(df_govt2[idx_tmp].shape)\ndf_govt2[idx_tmp].head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:56:28.588063Z","iopub.execute_input":"2021-06-22T12:56:28.588341Z","iopub.status.idle":"2021-06-22T12:57:00.888377Z","shell.execute_reply.started":"2021-06-22T12:56:28.588312Z","shell.execute_reply":"2021-06-22T12:57:00.88752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_govt2['len'] = df_govt2['cleaned_label'].progress_apply(lambda x: len(x.split()))\nidx_tmp = df_govt2['match_train']=='no_match'\nidx_tmp = df_govt2['len']>=3\ndf_govt3 = ri(df_govt2[idx_tmp])\nprint(df_govt2[idx_tmp].shape)\ndf_govt2[idx_tmp].head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:00.889802Z","iopub.execute_input":"2021-06-22T12:57:00.890355Z","iopub.status.idle":"2021-06-22T12:57:01.036654Z","shell.execute_reply.started":"2021-06-22T12:57:00.890314Z","shell.execute_reply":"2021-06-22T12:57:01.035673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_govt3['match_ver1'] = df_govt2['cleaned_label'].progress_apply(\n get_match, ref_labels=df_label_acronym_ver1_3['cleaned_label'])\nidx_tmp = df_govt3['match_ver1']=='no_match'\nidx_tmp = idx_tmp & (df_govt3['match_train']=='no_match')\ndf_govt4 = ri(df_govt3[idx_tmp])\nprint(df_govt4.shape)\ndf_govt4.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:01.038138Z","iopub.execute_input":"2021-06-22T12:57:01.038517Z","iopub.status.idle":"2021-06-22T12:57:41.614345Z","shell.execute_reply.started":"2021-06-22T12:57:01.038476Z","shell.execute_reply":"2021-06-22T12:57:41.613297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_govt4['target'] = np.arange(len(df_govt4)) + df_label_acronym_ver1_self['target'].max()+1\ndf_govt4.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:41.615766Z","iopub.execute_input":"2021-06-22T12:57:41.616162Z","iopub.status.idle":"2021-06-22T12:57:41.63473Z","shell.execute_reply.started":"2021-06-22T12:57:41.616111Z","shell.execute_reply":"2021-06-22T12:57:41.633727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_tmp = df_govt4['len']<=10\ndf_govt5 = df_govt4[idx_tmp]\ndf_govt5 = ri(df_govt5.sort_values('len'))\nprint(df_govt5.shape)\ndf_govt5.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:41.636285Z","iopub.execute_input":"2021-06-22T12:57:41.636818Z","iopub.status.idle":"2021-06-22T12:57:41.665363Z","shell.execute_reply.started":"2021-06-22T12:57:41.636775Z","shell.execute_reply":"2021-06-22T12:57:41.664604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def func(x, ref):\n index_tmp = x['index']\n if index_tmp==0:\n return 'no_match'\n ref = ref[:index_tmp-1]\n for item in ref:\n if jaccard(x['cleaned_label'], item)>=0.5:\n return item\n return 'no_match'\nref_list = df_govt5['cleaned_label'].tolist()\ndf_govt5['index'] = np.arange(len(df_govt5))\nif len(df_test)==4:\n df_govt5 = df_govt5.iloc[:1000]\ndf_govt5['match_in_self'] = df_govt5.progress_apply(lambda x: func(x, ref_list), axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:41.666529Z","iopub.execute_input":"2021-06-22T12:57:41.666852Z","iopub.status.idle":"2021-06-22T12:57:42.865582Z","shell.execute_reply.started":"2021-06-22T12:57:41.666815Z","shell.execute_reply":"2021-06-22T12:57:42.864546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx_tmp = df_govt5['match_in_self']=='no_match'\ndf_govt6 = ri(df_govt5[idx_tmp])\nprint(df_govt6.shape)\ndf_govt6.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:42.867145Z","iopub.execute_input":"2021-06-22T12:57:42.867529Z","iopub.status.idle":"2021-06-22T12:57:42.887265Z","shell.execute_reply.started":"2021-06-22T12:57:42.867486Z","shell.execute_reply":"2021-06-22T12:57:42.886392Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_traintest['det_govt'] = df_traintest['clean_text'].progress_apply(lambda x: \n detect_label(x.lower(), ref_label=df_govt6['cleaned_label'].values, \n ref_target=df_govt6['target'].values))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:42.888547Z","iopub.execute_input":"2021-06-22T12:57:42.889043Z","iopub.status.idle":"2021-06-22T12:57:43.21955Z","shell.execute_reply.started":"2021-06-22T12:57:42.889002Z","shell.execute_reply":"2021-06-22T12:57:43.2184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp = df_traintest['det_govt'].values\ntmp = np.concatenate(tmp)\n\ndf_govt6['count'] = df_govt6['target'].apply(lambda x: np.sum(tmp==x))\ndf_govt6 = ri(df_govt6.sort_values('count', ascending=False))\ndf_govt6","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:43.220973Z","iopub.execute_input":"2021-06-22T12:57:43.221362Z","iopub.status.idle":"2021-06-22T12:57:43.250755Z","shell.execute_reply.started":"2021-06-22T12:57:43.221321Z","shell.execute_reply":"2021-06-22T12:57:43.249667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TH_COUNT = 2\nidx_tmp = df_govt6['count']>=TH_COUNT\nidx_tmp = idx_tmp & (df_govt6['match_train']=='no_match')\ndf_govt7 = ri(df_govt6[idx_tmp])\nprint(df_govt7.shape)\ndf_govt7.head(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:43.252476Z","iopub.execute_input":"2021-06-22T12:57:43.253184Z","iopub.status.idle":"2021-06-22T12:57:43.271758Z","shell.execute_reply.started":"2021-06-22T12:57:43.253141Z","shell.execute_reply":"2021-06-22T12:57:43.270713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# NER\ndet1, trainの表記ゆれ検出 \ntrainはdataset名でグルーピングする\nels, nelsは検出しない。","metadata":{}},{"cell_type":"code","source":"tokenizer = pickle_load(\"../input/coleridge-ext/tokenizer_210613_01.pkl\")","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:43.273193Z","iopub.execute_input":"2021-06-22T12:57:43.273566Z","iopub.status.idle":"2021-06-22T12:57:43.402589Z","shell.execute_reply.started":"2021-06-22T12:57:43.273527Z","shell.execute_reply":"2021-06-22T12:57:43.401547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label_ref = df_label_acronym_ver1_3[['cleaned_label', 'target']].copy()\ndf_tmp = df_label_train_ref[['cleaned_label_my', 'target_group']]\ndf_tmp.columns = ['cleaned_label', 'target']\ndf_label_ref = ri(pd.concat([df_label_ref, df_tmp], axis=0))\nprint(df_label_ref.shape)\ndf_label_ref.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:43.404126Z","iopub.execute_input":"2021-06-22T12:57:43.404709Z","iopub.status.idle":"2021-06-22T12:57:43.420972Z","shell.execute_reply.started":"2021-06-22T12:57:43.404665Z","shell.execute_reply":"2021-06-22T12:57:43.419843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_reduced['det_ref'] = df_test_reduced['clean_text'].progress_apply(lambda x: \n detect_label(x, df_label_ref['cleaned_label'], df_label_ref['target'].values)\n)\ndf_test_reduced['len_det'] = df_test_reduced['det_ref'].apply(len)\nprint(df_test_reduced[df_test_reduced['len_det']>0].shape) # 340,6\ndf_test_reduced[df_test_reduced['len_det']>0].head(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T12:57:43.422671Z","iopub.execute_input":"2021-06-22T12:57:43.423163Z","iopub.status.idle":"2021-06-22T12:57:52.639438Z","shell.execute_reply.started":"2021-06-22T12:57:43.423104Z","shell.execute_reply":"2021-06-22T12:57:52.63855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_include(x, ref):\n try:\n for item in ref:\n if item in x:\n return item\n return 'no_include'\n except:\n return 'no_include'\n \ndef get_include2(x, ref):\n try:\n for item in ref:\n if x in item:\n return item\n return 'no_include'\n except:\n return 'no_include'\n \nnels_els_list = [\n 'national education longitudinal study',\n 'education longitudinal study',\n]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:13:51.454684Z","iopub.execute_input":"2021-06-22T13:13:51.455037Z","iopub.status.idle":"2021-06-22T13:13:51.461138Z","shell.execute_reply.started":"2021-06-22T13:13:51.455005Z","shell.execute_reply":"2021-06-22T13:13:51.460193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ni = 41\nBATCH_SIZE = 64\ndef get_ner_pred(x):\n try:\n text = x['text']\n sentence_list = chunk_text(text, keywords=keywords) # keyword含む文のみのchunkを作成\n det_ref = x['det_ref']\n labels_det_ref = []\n for target in det_ref:\n labels_det_ref.append(df_label_ref['cleaned_label'][df_label_ref['target']==target].values[0])\n labels_det_ref = np.unique(labels_det_ref).tolist()\n\n num_batch = int(np.ceil(len(sentence_list)/BATCH_SIZE))\n naun_chunks = []\n for j in range(num_batch):\n batch = sentence_list[j*BATCH_SIZE:(j+1)*BATCH_SIZE]\n for k, sent in enumerate(batch):\n naun_chunks_k = list(finetuned_ner_nlp(sent).ents) # sentenceで区切ってバッチ化して処理される\n naun_chunks += naun_chunks_k\n naun_chunks = [str(chunk) for chunk in naun_chunks]\n naun_chunks2 = np.unique(naun_chunks)\n df_chunks = pd.DataFrame(naun_chunks2, columns=['chunk'])\n df_chunks['chunk-the'] = df_chunks['chunk'].apply(delete_the)\n df_chunks['cleaned_label'] = df_chunks['chunk-the'].apply(clean_text)\n df_chunks['len'] = df_chunks['cleaned_label'].apply(lambda x: len(x.split()))\n df_chunks['det_already'] = df_chunks['cleaned_label'].apply(lambda x: x in labels_det_ref)\n df_chunks['match_ref'] = df_chunks['cleaned_label'].apply(\n lambda x: get_match(x, ref_labels=df_label_ref['cleaned_label'], threshold=0.5)\n )\n df_chunks['match_nels_els'] = df_chunks['cleaned_label'].apply(\n lambda x: get_match(x, ref_labels=nels_els_list, threshold=0.5)\n )\n df_chunks['include_ref'] = df_chunks['cleaned_label'].apply(\n lambda x: get_include(x, ref=df_label_ref['cleaned_label'])\n )\n \n df_chunks['include_ref2'] = df_chunks['cleaned_label'].apply(\n lambda x: get_include2(x, ref=df_label_ref['cleaned_label'])\n )\n idx_tmp = df_chunks['cleaned_label'].duplicated()==False\n idx_tmp = idx_tmp & (df_chunks['len']>=3)\n idx_tmp = idx_tmp & (df_chunks['det_already']==False)\n idx_tmp = idx_tmp & (df_chunks['match_ref']!='no_match')\n idx_tmp = idx_tmp & (df_chunks['match_nels_els']=='no_match')\n idx_tmp = idx_tmp & (df_chunks['include_ref']=='no_include')\n idx_tmp = idx_tmp & (df_chunks['include_ref2']=='no_include')\n df_chunks2 = ri(df_chunks[idx_tmp])\n\n pred_ner = \"|\".join(df_chunks2['cleaned_label'].values.tolist())\n# pred_ner = 'A'\n return pred_ner\n except:\n return ''\n\ndf_test_reduced['pred_ner'] = \"\"\ndf_test_reduced['pred_ner'].iloc[:20] = df_test_reduced.iloc[:20].progress_apply(get_ner_pred, axis=1)\ndf_test_reduced.iloc[10:20]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:14:24.816381Z","iopub.execute_input":"2021-06-22T13:14:24.816773Z","iopub.status.idle":"2021-06-22T13:14:37.012377Z","shell.execute_reply.started":"2021-06-22T13:14:24.816741Z","shell.execute_reply":"2021-06-22T13:14:37.011448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\nver3\n10 joint center for political studies\n14 national study of postgraduate faculty\n\n\"\"\"\ndf_test_reduced[df_test_reduced['pred_ner']!='']","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:14:53.548646Z","iopub.execute_input":"2021-06-22T13:14:53.548994Z","iopub.status.idle":"2021-06-22T13:14:53.569424Z","shell.execute_reply.started":"2021-06-22T13:14:53.548963Z","shell.execute_reply":"2021-06-22T13:14:53.568561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_label_train_ref['cleaned_label'][df_label_train_ref['target_group']==3]","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:14:59.998866Z","iopub.execute_input":"2021-06-22T13:14:59.999226Z","iopub.status.idle":"2021-06-22T13:15:00.006523Z","shell.execute_reply.started":"2021-06-22T13:14:59.999194Z","shell.execute_reply":"2021-06-22T13:15:00.005481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_tmp = 'baltimore longitudinal study on aging'\nfor i in range(len(df_label_ref)):\n label_i = df_label_ref['cleaned_label'][i]\n if jaccard(label_i, label_tmp)>=0.5:\n print(label_i)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:02.596035Z","iopub.execute_input":"2021-06-22T13:15:02.596362Z","iopub.status.idle":"2021-06-22T13:15:02.609685Z","shell.execute_reply.started":"2021-06-22T13:15:02.59633Z","shell.execute_reply":"2021-06-22T13:15:02.60876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Test prediction","metadata":{}},{"cell_type":"code","source":"def pred_dup(x):\n df_tmp = df_train_reduced2[df_train_reduced2['text']==x]\n if len(df_tmp)>0:\n label_list = df_tmp['targets'].values[0]\n else:\n label_list = np.zeros(0, np.int64)\n return label_list\n\ndf_test['det_dup'] = df_test['text'].apply(lambda x: pred_dup(x))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:16.376505Z","iopub.execute_input":"2021-06-22T13:15:16.376972Z","iopub.status.idle":"2021-06-22T13:15:16.448827Z","shell.execute_reply.started":"2021-06-22T13:15:16.376931Z","shell.execute_reply":"2021-06-22T13:15:16.447795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['det_train'] = df_test['clean_text'].progress_apply(lambda x: \n detect_label(x.lower(), ref_label=df_label_train['label'].values, \n ref_target=df_label_train['target_relabeled'].values))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:16.621127Z","iopub.execute_input":"2021-06-22T13:15:16.621476Z","iopub.status.idle":"2021-06-22T13:15:16.665425Z","shell.execute_reply.started":"2021-06-22T13:15:16.621424Z","shell.execute_reply":"2021-06-22T13:15:16.664199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['det_train_acronym'] = df_test['clean_text'].progress_apply(lambda x: \n detect_acronym(x, df_label_train_acronym2['base'], \n df_label_train_acronym2['acronym_clean'], \n df_label_train_acronym2['target'], th=[1,2])\n)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:16.957957Z","iopub.execute_input":"2021-06-22T13:15:16.958292Z","iopub.status.idle":"2021-06-22T13:15:17.013352Z","shell.execute_reply.started":"2021-06-22T13:15:16.958259Z","shell.execute_reply":"2021-06-22T13:15:17.01232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['det_govt'] = df_test['clean_text'].progress_apply(lambda x: \n detect_label(x.lower(), ref_label=df_govt7['cleaned_label'].values, \n ref_target=df_govt7['target'].values))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:17.286107Z","iopub.execute_input":"2021-06-22T13:15:17.286428Z","iopub.status.idle":"2021-06-22T13:15:17.298361Z","shell.execute_reply.started":"2021-06-22T13:15:17.286396Z","shell.execute_reply":"2021-06-22T13:15:17.297101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['det_acronym'] = df_test['clean_text'].progress_apply(lambda x: \n detect_acronym(x, df_label_acronym_ver1_3['cleaned_label'], \n df_label_acronym_ver1_3['acronym_clean'], df_label_acronym_ver1_3['target'], th=[1,0])\n)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:17.546908Z","iopub.execute_input":"2021-06-22T13:15:17.547206Z","iopub.status.idle":"2021-06-22T13:15:17.625362Z","shell.execute_reply.started":"2021-06-22T13:15:17.547172Z","shell.execute_reply":"2021-06-22T13:15:17.624366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['det_acronym_self'] = df_test['clean_text'].progress_apply(lambda x: \n detect_acronym(x, df_label_acronym_ver1_self['cleaned_label'], \n df_label_acronym_ver1_self['acronym_clean'], df_label_acronym_ver1_self['target'], th=[1,1])\n)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:17.849864Z","iopub.execute_input":"2021-06-22T13:15:17.850291Z","iopub.status.idle":"2021-06-22T13:15:17.928921Z","shell.execute_reply.started":"2021-06-22T13:15:17.850242Z","shell.execute_reply":"2021-06-22T13:15:17.927887Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['det_ref'] = df_test['clean_text'].progress_apply(lambda x: \n detect_label(x, df_label_ref['cleaned_label'], df_label_ref['target'].values)\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:18.133534Z","iopub.execute_input":"2021-06-22T13:15:18.133848Z","iopub.status.idle":"2021-06-22T13:15:18.209964Z","shell.execute_reply.started":"2021-06-22T13:15:18.133818Z","shell.execute_reply":"2021-06-22T13:15:18.208927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['pred_ner'] = df_test.progress_apply(get_ner_pred, axis=1)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:18.422498Z","iopub.execute_input":"2021-06-22T13:15:18.422804Z","iopub.status.idle":"2021-06-22T13:15:23.981865Z","shell.execute_reply.started":"2021-06-22T13:15:18.422775Z","shell.execute_reply":"2021-06-22T13:15:23.981088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['pred_det'] = df_test.progress_apply(lambda x: \n np.sort(np.unique(np.concatenate([\n x['det_acronym_self'],\n x['det_acronym'],\n x['det_govt'],\n# x['det_dup'],\n# x['det_train'],\n# x['det_train_acronym'],\n ]))).astype(np.int64), axis=1\n)\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:23.983348Z","iopub.execute_input":"2021-06-22T13:15:23.983717Z","iopub.status.idle":"2021-06-22T13:15:24.023633Z","shell.execute_reply.started":"2021-06-22T13:15:23.98368Z","shell.execute_reply":"2021-06-22T13:15:24.022585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_tmp1 = df_label_acronym_ver1_self[['acronym_clean', 'target']]\ndf_tmp1.columns = ['label', 'target']\ndf_tmp2 = df_label_acronym_ver1_3[['cleaned_label', 'target']]\ndf_tmp2.columns = ['label', 'target']\ndf_tmp3 = df_govt7[['cleaned_label', 'target']]\ndf_tmp3.columns = ['label', 'target']\ndf_tmp4 = df_label_train[['cleaned_label_my', 'target']]\ndf_tmp4.columns = ['label', 'target']\ndf_tmp5 = df_label_train_acronym2[['acronym_clean', 'target']]\ndf_tmp5.columns = ['label', 'target']\ndf_label_all = pd.concat([df_tmp1, df_tmp2, df_tmp3, df_tmp4, df_tmp5]).reset_index(drop=True)\ndf_label_all = ri(df_label_all[df_label_all['target'].duplicated()==False])\ndf_label_all.tail()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:24.02549Z","iopub.execute_input":"2021-06-22T13:15:24.025844Z","iopub.status.idle":"2021-06-22T13:15:24.047884Z","shell.execute_reply.started":"2021-06-22T13:15:24.025807Z","shell.execute_reply":"2021-06-22T13:15:24.046757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['PredictionString'] = df_test['pred_det'].progress_apply(lambda x: \n get_label(x, df_label_all['label'].values, df_label_all['target'].values)\n)\ndf_test['PredictionString'] = df_test['PredictionString'] + '|' + df_test['pred_ner']\ndf_test['PredictionString'] = df_test['PredictionString'].apply(lambda x: x.strip('|'))\ndf_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:24.049784Z","iopub.execute_input":"2021-06-22T13:15:24.050286Z","iopub.status.idle":"2021-06-22T13:15:24.091565Z","shell.execute_reply.started":"2021-06-22T13:15:24.050248Z","shell.execute_reply":"2021-06-22T13:15:24.090679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sub = df_test[['Id', 'PredictionString']]\ndf_sub.to_csv(\"submission.csv\", index=None)\ndf_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:24.092833Z","iopub.execute_input":"2021-06-22T13:15:24.093375Z","iopub.status.idle":"2021-06-22T13:15:24.11006Z","shell.execute_reply.started":"2021-06-22T13:15:24.093335Z","shell.execute_reply":"2021-06-22T13:15:24.109255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(df_sub.iloc[:10])):\n print(df_sub['PredictionString'][i])","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:24.111388Z","iopub.execute_input":"2021-06-22T13:15:24.111779Z","iopub.status.idle":"2021-06-22T13:15:24.118575Z","shell.execute_reply.started":"2021-06-22T13:15:24.111741Z","shell.execute_reply":"2021-06-22T13:15:24.117611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(df_sub.iloc[:10])):\n pred_list = df_sub['PredictionString'][i].split('|')\n# print(pred_list, len(pred_list))\n if pred_list[0]!='':\n for j in range(len(pred_list)):\n print(\"{:4d}: {}\".format(i, pred_list[j]))","metadata":{"execution":{"iopub.status.busy":"2021-06-22T13:15:24.120119Z","iopub.execute_input":"2021-06-22T13:15:24.120563Z","iopub.status.idle":"2021-06-22T13:15:24.133942Z","shell.execute_reply.started":"2021-06-22T13:15:24.120519Z","shell.execute_reply":"2021-06-22T13:15:24.132827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}