{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n #   for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\nimport re\nimport collections\nimport json","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"execution":{"iopub.status.busy":"2021-06-23T09:37:18.728825Z","iopub.execute_input":"2021-06-23T09:37:18.729424Z","iopub.status.idle":"2021-06-23T09:37:18.741638Z","shell.execute_reply.started":"2021-06-23T09:37:18.729257Z","shell.execute_reply":"2021-06-23T09:37:18.740259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:18.743733Z","iopub.execute_input":"2021-06-23T09:37:18.744205Z","iopub.status.idle":"2021-06-23T09:37:18.887483Z","shell.execute_reply.started":"2021-06-23T09:37:18.744160Z","shell.execute_reply":"2021-06-23T09:37:18.885242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:18.891629Z","iopub.execute_input":"2021-06-23T09:37:18.892185Z","iopub.status.idle":"2021-06-23T09:37:18.927858Z","shell.execute_reply.started":"2021-06-23T09:37:18.892130Z","shell.execute_reply":"2021-06-23T09:37:18.926506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:18.930226Z","iopub.execute_input":"2021-06-23T09:37:18.930686Z","iopub.status.idle":"2021-06-23T09:37:19.038354Z","shell.execute_reply.started":"2021-06-23T09:37:18.930638Z","shell.execute_reply":"2021-06-23T09:37:19.037214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop_duplicates('cleaned_label')","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.040194Z","iopub.execute_input":"2021-06-23T09:37:19.040611Z","iopub.status.idle":"2021-06-23T09:37:19.074133Z","shell.execute_reply.started":"2021-06-23T09:37:19.040564Z","shell.execute_reply":"2021-06-23T09:37:19.072874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train.Id=='170113f9-399c-489e-ab53-2faf5c64c5bc']","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.075700Z","iopub.execute_input":"2021-06-23T09:37:19.076064Z","iopub.status.idle":"2021-06-23T09:37:19.101135Z","shell.execute_reply.started":"2021-06-23T09:37:19.076024Z","shell.execute_reply":"2021-06-23T09:37:19.099595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.103126Z","iopub.execute_input":"2021-06-23T09:37:19.103650Z","iopub.status.idle":"2021-06-23T09:37:19.119477Z","shell.execute_reply.started":"2021-06-23T09:37:19.103549Z","shell.execute_reply":"2021-06-23T09:37:19.117850Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.123664Z","iopub.execute_input":"2021-06-23T09:37:19.124140Z","iopub.status.idle":"2021-06-23T09:37:19.138256Z","shell.execute_reply.started":"2021-06-23T09:37:19.124095Z","shell.execute_reply":"2021-06-23T09:37:19.136770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.describe()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.141217Z","iopub.execute_input":"2021-06-23T09:37:19.141735Z","iopub.status.idle":"2021-06-23T09:37:19.167603Z","shell.execute_reply.started":"2021-06-23T09:37:19.141682Z","shell.execute_reply":"2021-06-23T09:37:19.166292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/coleridgeinitiative-show-us-the-data/train/93c27c16-9356-4b97-9785-9795ad9bb609.json') as json_file:\n    t = json.load(json_file)\n    text = []\n    for p in t:\n        text.append(p['text'])\n    text = ' '.join(text)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.169631Z","iopub.execute_input":"2021-06-23T09:37:19.170159Z","iopub.status.idle":"2021-06-23T09:37:19.192847Z","shell.execute_reply.started":"2021-06-23T09:37:19.170106Z","shell.execute_reply":"2021-06-23T09:37:19.191673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.194597Z","iopub.execute_input":"2021-06-23T09:37:19.195257Z","iopub.status.idle":"2021-06-23T09:37:19.205645Z","shell.execute_reply.started":"2021-06-23T09:37:19.195211Z","shell.execute_reply":"2021-06-23T09:37:19.202892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.208977Z","iopub.execute_input":"2021-06-23T09:37:19.209377Z","iopub.status.idle":"2021-06-23T09:37:19.220246Z","shell.execute_reply.started":"2021-06-23T09:37:19.209344Z","shell.execute_reply":"2021-06-23T09:37:19.219168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train.Id=='93c27c16-9356-4b97-9785-9795ad9bb609']","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.221840Z","iopub.execute_input":"2021-06-23T09:37:19.222272Z","iopub.status.idle":"2021-06-23T09:37:19.247329Z","shell.execute_reply.started":"2021-06-23T09:37:19.222228Z","shell.execute_reply":"2021-06-23T09:37:19.245905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntexts = {}\nfor i, id in enumerate(train.Id.unique()):\n    print(i, end=' ') if i%100==0 else next\n    with open('/kaggle/input/coleridgeinitiative-show-us-the-data/train/'+id+'.json') as json_file:\n        t = json.load(json_file)\n        text = []\n        for p in t:\n            text.append(p['text'])\n        text = ' '.join(text)\n        texts[id] = text","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:37:19.249147Z","iopub.execute_input":"2021-06-23T09:37:19.250396Z","iopub.status.idle":"2021-06-23T09:38:46.667210Z","shell.execute_reply.started":"2021-06-23T09:37:19.250345Z","shell.execute_reply":"2021-06-23T09:38:46.666385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_texts = ' '.join(list(texts.values()))","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:38:46.668343Z","iopub.execute_input":"2021-06-23T09:38:46.668777Z","iopub.status.idle":"2021-06-23T09:38:47.870804Z","shell.execute_reply.started":"2021-06-23T09:38:46.668745Z","shell.execute_reply":"2021-06-23T09:38:47.870078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntest_texts = {}\nfor i, id in enumerate(test.Id.unique()):\n    print(i, end=' ') if i%100==0 else next\n    with open('/kaggle/input/coleridgeinitiative-show-us-the-data/test/'+id+'.json') as json_file:\n        t = json.load(json_file)\n        test_text = []\n        for p in t:\n            test_text.append(p['text'])\n        test_text = ' '.join(test_text)\n        test_texts[id] = test_text","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:38:47.872174Z","iopub.execute_input":"2021-06-23T09:38:47.872476Z","iopub.status.idle":"2021-06-23T09:38:47.920793Z","shell.execute_reply.started":"2021-06-23T09:38:47.872446Z","shell.execute_reply":"2021-06-23T09:38:47.920088Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nc = collections.Counter()\nwords = ' '.join(train.dataset_title.unique().tolist()).split() + ' '.join(train.dataset_label.unique().tolist()).split() + ' '.join(train.cleaned_label.unique().tolist()).split()\ntotal_count = 0\nfor word in words:\n    c[word.lower()] += 1\n    total_count += 1","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:38:47.921937Z","iopub.execute_input":"2021-06-23T09:38:47.922444Z","iopub.status.idle":"2021-06-23T09:38:47.936832Z","shell.execute_reply.started":"2021-06-23T09:38:47.922401Z","shell.execute_reply":"2021-06-23T09:38:47.936054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(total_count)\nc.most_common(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:38:47.938352Z","iopub.execute_input":"2021-06-23T09:38:47.938865Z","iopub.status.idle":"2021-06-23T09:38:47.948335Z","shell.execute_reply.started":"2021-06-23T09:38:47.938807Z","shell.execute_reply":"2021-06-23T09:38:47.947352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nc_all = collections.Counter()\nwords = all_texts.split()\ntotal_count_all = 0\nfor word in words:\n    c_all[word.lower()] += 1\n    total_count_all += 1","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:38:47.950483Z","iopub.execute_input":"2021-06-23T09:38:47.950958Z","iopub.status.idle":"2021-06-23T09:40:05.025892Z","shell.execute_reply.started":"2021-06-23T09:38:47.950921Z","shell.execute_reply":"2021-06-23T09:40:05.024426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(total_count_all)\nc_all.most_common(30)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:05.028101Z","iopub.execute_input":"2021-06-23T09:40:05.028559Z","iopub.status.idle":"2021-06-23T09:40:05.316215Z","shell.execute_reply.started":"2021-06-23T09:40:05.028513Z","shell.execute_reply":"2021-06-23T09:40:05.315231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for word in c.most_common(30):\n    print(word[0], '\\t', round(word[1]/total_count,6), '\\t', round(dict(c_all)[word[0]]/total_count_all,6))","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:05.319567Z","iopub.execute_input":"2021-06-23T09:40:05.319890Z","iopub.status.idle":"2021-06-23T09:40:14.912759Z","shell.execute_reply.started":"2021-06-23T09:40:05.319857Z","shell.execute_reply":"2021-06-23T09:40:14.911668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['text'] = train['Id'].map(texts)\ntest['text'] = test['Id'].map(test_texts)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:14.914282Z","iopub.execute_input":"2021-06-23T09:40:14.914629Z","iopub.status.idle":"2021-06-23T09:40:14.940344Z","shell.execute_reply.started":"2021-06-23T09:40:14.914595Z","shell.execute_reply":"2021-06-23T09:40:14.938731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:14.942370Z","iopub.execute_input":"2021-06-23T09:40:14.942801Z","iopub.status.idle":"2021-06-23T09:40:14.959877Z","shell.execute_reply.started":"2021-06-23T09:40:14.942762Z","shell.execute_reply":"2021-06-23T09:40:14.958572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head(2)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:14.961738Z","iopub.execute_input":"2021-06-23T09:40:14.962218Z","iopub.status.idle":"2021-06-23T09:40:14.979454Z","shell.execute_reply.started":"2021-06-23T09:40:14.962167Z","shell.execute_reply":"2021-06-23T09:40:14.978387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain['cleaned_label_isin'] = train.apply(lambda x: 1 if x['text'].lower().find(x['cleaned_label'].lower()) else 0, axis=1)\ntrain['dataset_label_isin'] = train.apply(lambda x: 1 if x['text'].lower().find(x['dataset_label'].lower()) else 0, axis=1)\ntrain['dataset_title_isin'] = train.apply(lambda x: 1 if x['text'].lower().find(x['dataset_title'].lower()) else 0, axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:14.980730Z","iopub.execute_input":"2021-06-23T09:40:14.981241Z","iopub.status.idle":"2021-06-23T09:40:28.349120Z","shell.execute_reply.started":"2021-06-23T09:40:14.981202Z","shell.execute_reply":"2021-06-23T09:40:28.348078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:28.350451Z","iopub.execute_input":"2021-06-23T09:40:28.350758Z","iopub.status.idle":"2021-06-23T09:40:28.370471Z","shell.execute_reply.started":"2021-06-23T09:40:28.350726Z","shell.execute_reply":"2021-06-23T09:40:28.369100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[['cleaned_label_isin', 'dataset_label_isin', 'dataset_title_isin']].mean(axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:28.372318Z","iopub.execute_input":"2021-06-23T09:40:28.372775Z","iopub.status.idle":"2021-06-23T09:40:28.385365Z","shell.execute_reply.started":"2021-06-23T09:40:28.372724Z","shell.execute_reply":"2021-06-23T09:40:28.384150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"datawords = ['Survey', 'Data', 'Study', 'Database', 'Statistics']\nt = 3000\nprint(train.loc[t,'cleaned_label'])\nprint('='*50)\nfor dataword in datawords:\n    inds = [m.start() for m in re.finditer(dataword, train.loc[t, 'text'])]\n    for i in inds:\n        print(i, ':', train.loc[t, 'text'][i-40:i+40].replace('\\n',' '))","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:28.387459Z","iopub.execute_input":"2021-06-23T09:40:28.387959Z","iopub.status.idle":"2021-06-23T09:40:28.406740Z","shell.execute_reply.started":"2021-06-23T09:40:28.387903Z","shell.execute_reply":"2021-06-23T09:40:28.405350Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ds = list(set(train.dataset_title.unique().tolist() + train.dataset_label.unique().tolist() + train.cleaned_label.unique().tolist()))\nds = list(set([i.lower() for i in ds]))\nprint(len(ds))","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:28.408719Z","iopub.execute_input":"2021-06-23T09:40:28.409153Z","iopub.status.idle":"2021-06-23T09:40:28.424640Z","shell.execute_reply.started":"2021-06-23T09:40:28.409105Z","shell.execute_reply":"2021-06-23T09:40:28.423256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n[x for x in ds if x in train.loc[1, 'text'].lower()]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:28.426439Z","iopub.execute_input":"2021-06-23T09:40:28.426853Z","iopub.status.idle":"2021-06-23T09:40:28.445393Z","shell.execute_reply.started":"2021-06-23T09:40:28.426806Z","shell.execute_reply":"2021-06-23T09:40:28.444084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[d for d in ds if d.find('research')>-1]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:40:42.723699Z","iopub.execute_input":"2021-06-23T09:40:42.724399Z","iopub.status.idle":"2021-06-23T09:40:42.732629Z","shell.execute_reply.started":"2021-06-23T09:40:42.724337Z","shell.execute_reply":"2021-06-23T09:40:42.731766Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[d for d in ds if d.find('center')>-1]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:41:23.736293Z","iopub.execute_input":"2021-06-23T09:41:23.736739Z","iopub.status.idle":"2021-06-23T09:41:23.743402Z","shell.execute_reply.started":"2021-06-23T09:41:23.736697Z","shell.execute_reply":"2021-06-23T09:41:23.742275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[d for d in ds if d.find('dataset')>-1]","metadata":{"execution":{"iopub.status.busy":"2021-06-23T09:42:18.561558Z","iopub.execute_input":"2021-06-23T09:42:18.562140Z","iopub.status.idle":"2021-06-23T09:42:18.569657Z","shell.execute_reply.started":"2021-06-23T09:42:18.562099Z","shell.execute_reply":"2021-06-23T09:42:18.568576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_paper_sentence(s):\n    \"\"\"\n    This function is essentially clean_text without lowercasing.\n    \"\"\"\n    s = re.sub('[^A-Za-z0-9]+', ' ', str(s)).strip()\n    s = re.sub(' +', ' ', s)\n    return s","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:15:58.258825Z","iopub.execute_input":"2021-06-22T19:15:58.259251Z","iopub.status.idle":"2021-06-22T19:15:58.264619Z","shell.execute_reply.started":"2021-06-22T19:15:58.259215Z","shell.execute_reply":"2021-06-22T19:15:58.263515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndatawords = ['Survey', 'Data', 'Study', 'Database', 'Statistics', 'Dataset']\ntrain['good_candidates'] = ''\ntrain['bad_candidates'] = ''\nverbose = True\npreps = ['of', 'and', 'in', 'for']\n\nfor t in range(len(train)):\n    print('='*80) if verbose else next\n    print('='*80) if verbose else next\n    print(t, end=' ') #if t%100==0 and not verbose else print(t) if verbose else next\n    #print(t, end=' ') if t%100==0 else next\n    print(train.loc[t,'cleaned_label']) if verbose else next\n    print('='*50) if verbose else next\n    text = train.loc[t, 'text']\n    text = re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\", text) #remove text in brackets\n    text = text.replace('.', ' dot').replace(',',' comma') #replacing dots and commas\n    text = text.replace('/n', ' paragraph ')\n    text_words = re.sub(r\"[^a-zA-Z0-9] \",\" \",text).split(' ')\n    text_words = [w for w in text_words if w!='']\n    good_candidates = []\n    bad_candidates = []\n\n    for dataword in datawords:\n        inds = [i for i, w in enumerate(text_words) if w==dataword]\n        for i in inds:\n            print(i, ':', text_words[i-4:i+4]) if verbose else next\n            lower_limit=i\n            upper_limit=i\n\n            #for j in range(1,6):\n            #    if i-j>=0:\n            #        if text_words[i-j][0].isupper() or text_words[i-j] in preps:\n            #            lower_limit = i-j\n            #        else:\n            #            break\n            \n            for j in range(1,6):\n                if i-j >= 0:\n                    if text_words[i-j][0].isupper():\n                        lower_limit = i-j\n                    elif i-j-1 >= 0:\n                        if text_words[i-j] in preps and text_words[i-j-1][0].isupper():\n                            lower_limit = i-j-1\n                        else:\n                            break\n                    else:\n                        break\n                else:\n                    break\n\n            for j in range(1,6):\n                if i+j <len(text_words):\n                    if text_words[i+j][0].isupper():\n                        upper_limit = i+j\n                    elif i+j+1 <len(text_words):\n                        if text_words[i+j] in preps and text_words[i+j+1][0].isupper():\n                            upper_limit = i+j+1\n                        else:\n                            break\n                    else:\n                        break\n                else:\n                    break\n            \n            if text_words[lower_limit] == 'The':\n                lower_limit = lower_limit + 1\n                \n            #if text_words[lower_limit][-3:] =='ing':\n            #    lower_limit = lower_limit + 1\n                \n            if 'Center' in text_words[lower_limit:upper_limit+1] or 'Source' in text_words[lower_limit:upper_limit+1] or 'Note' in text_words[lower_limit:upper_limit+1]:\n                upper_limit=lower_limit\n                \n            if len([m.start() for m in re.finditer('Study', ' '.join(text_words[lower_limit:upper_limit+1]))])>1:\n                upper_limit=lower_limit\n            \n            \n            candidate = ' '.join(text_words[lower_limit:upper_limit+1])\n            if len(candidate.split(' '))>2:\n                good_candidates.append(candidate)\n            else:\n                bad_candidates.append(candidate)\n                \n            #if len(good_candidates) >=3:\n            #    good_candidates = sorted(good_candidates, key=len)[-3:]\n\n            print(candidate) if verbose else next\n            print('='*30) if verbose else next\n    \n    text = train.loc[t, 'text']\n    #text = re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\", text) #remove text in brackets\n    text = text.replace('.', ' dot').replace(',',' comma') #replacing dots and commas\n    text = text.replace('/n', ' paragraph ')\n    text = clean_paper_sentence(text)\n    \n    dict_cands = [x for x in ds if x in text.lower()]\n    for candidate in dict_cands:\n        print(candidate) if verbose else next\n        print('='*30) if verbose else next\n        #if len(candidate.split(' '))>2:\n        good_candidates.append(candidate)\n        #else:\n        #    bad_candidates.append(candidate)\n    \n    good_candidates = [i.lower() for i in good_candidates]\n    bad_candidates = [i.lower() for i in bad_candidates]\n    \n    good_candidates = '|'.join(list(set(good_candidates)))\n    bad_candidates = '|'.join(list(set(bad_candidates)))\n    print('Good candidates: ', good_candidates) if verbose else next\n    print('Bad candidates: ', bad_candidates) if verbose else next\n    train.loc[t, 'good_candidates'] = good_candidates\n    train.loc[t, 'bad_candidates'] = bad_candidates\n    \n    if t==1000:\n        break","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:16:14.171238Z","iopub.execute_input":"2021-06-22T19:16:14.1718Z","iopub.status.idle":"2021-06-22T19:16:43.743662Z","shell.execute_reply.started":"2021-06-22T19:16:14.171756Z","shell.execute_reply":"2021-06-22T19:16:43.742531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:16:43.745371Z","iopub.execute_input":"2021-06-22T19:16:43.745653Z","iopub.status.idle":"2021-06-22T19:16:43.762824Z","shell.execute_reply.started":"2021-06-22T19:16:43.745627Z","shell.execute_reply":"2021-06-22T19:16:43.761676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndatawords = ['Survey', 'Data', 'Study', 'Database', 'Statistics', 'Dataset']\ntest['good_candidates'] = ''\n#train['bad_candidates'] = ''\nverbose = False\npreps = ['of', 'and', 'in', 'for']\n\nfor t in range(len(test)):\n    print('='*80) if verbose else next\n    print('='*80) if verbose else next\n    #print(t, end=' ')# if t%100==0 and not verbose else print(t) if verbose else next\n    print(t, end=' ') if t%100==0 else next\n    #print(test.loc[t,'cleaned_label']) if verbose else next\n    print('='*50) if verbose else next\n    text = test.loc[t, 'text']\n    text = re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\", text) #remove text in brackets\n    text = text.replace('.', ' dot').replace(',',' comma') #replacing dots and commas\n    text = text.replace('/n', ' paragraph ')\n    text_words = re.sub(r\"[^a-zA-Z0-9] \",\" \",text).split(' ')\n    text_words = [w for w in text_words if w!='']\n    good_candidates = []\n    bad_candidates = []\n\n    for dataword in datawords:\n        inds = [i for i, w in enumerate(text_words) if w==dataword]\n        for i in inds:\n            print(i, ':', text_words[i-4:i+4]) if verbose else next\n            lower_limit=i\n            upper_limit=i\n\n            #for j in range(1,6):\n            #    if i-j>=0:\n            #        if text_words[i-j][0].isupper() or text_words[i-j] in preps:\n            #            lower_limit = i-j\n            #        else:\n            #            break\n            \n            for j in range(1,6):\n                if i-j >= 0:\n                    if text_words[i-j][0].isupper():\n                        lower_limit = i-j\n                    elif i-j-1 >= 0:\n                        if text_words[i-j] in preps and text_words[i-j-1][0].isupper():\n                            lower_limit = i-j-1\n                        else:\n                            break\n                    else:\n                        break\n                else:\n                    break\n\n            for j in range(1,6):\n                if i+j <len(text_words):\n                    if text_words[i+j][0].isupper():\n                        upper_limit = i+j\n                    elif i+j+1 <len(text_words):\n                        if text_words[i+j] in preps and text_words[i+j+1][0].isupper():\n                            upper_limit = i+j+1\n                        else:\n                            break\n                    else:\n                        break\n                else:\n                    break\n            \n            if text_words[lower_limit] == 'The':\n                lower_limit = lower_limit + 1\n                \n            #if text_words[lower_limit][-3:] =='ing':\n            #    lower_limit = lower_limit + 1\n                \n            if 'Center' in text_words[lower_limit:upper_limit+1] or 'Source' in text_words[lower_limit:upper_limit+1] or 'Note' in text_words[lower_limit:upper_limit+1]:\n                upper_limit=lower_limit\n                \n            if len([m.start() for m in re.finditer('Study', ' '.join(text_words[lower_limit:upper_limit+1]))])>1:\n                upper_limit=lower_limit\n            \n            \n            candidate = ' '.join(text_words[lower_limit:upper_limit+1])\n            if len(candidate.split(' '))>2:\n                good_candidates.append(candidate)\n            else:\n                bad_candidates.append(candidate)\n                \n            #if len(good_candidates) >=3:\n            #    good_candidates = sorted(good_candidates, key=len)[-3:]\n\n            print(candidate) if verbose else next\n            print('='*30) if verbose else next\n    \n    text = test.loc[t, 'text']\n    #text = re.sub(\"[\\(\\[].*?[\\)\\]]\", \"\", text) #remove text in brackets\n    text = text.replace('.', ' dot').replace(',',' comma') #replacing dots and commas\n    text = text.replace('/n', ' paragraph ')\n    text = clean_paper_sentence(text)\n    \n    dict_cands = [x for x in ds if x in text.lower()]\n    for candidate in dict_cands:\n        print(candidate) if verbose else next\n        print('='*30) if verbose else next\n        #if len(candidate.split(' '))>2:\n        good_candidates.append(candidate)\n        #else:\n        #    bad_candidates.append(candidate)\n    \n    good_candidates = [i.lower() for i in good_candidates]\n    #bad_candidates = [i.lower() for i in bad_candidates]\n    \n    good_candidates = '|'.join(list(set(good_candidates)))\n    #bad_candidates = '|'.join(list(set(bad_candidates)))\n    #print('Good candidates: ', good_candidates) if verbose else next\n    #print('Bad candidates: ', bad_candidates) if verbose else next\n    test.loc[t, 'PredictionString'] = good_candidates\n    #train.loc[t, 'bad_candidates'] = bad_candidates\n    \n    #if t==3000:\n    #    break","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:26:45.87144Z","iopub.execute_input":"2021-06-22T19:26:45.871831Z","iopub.status.idle":"2021-06-22T19:26:46.094644Z","shell.execute_reply.started":"2021-06-22T19:26:45.871787Z","shell.execute_reply":"2021-06-22T19:26:46.093598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[['Id', 'PredictionString']].to_csv('submission.csv', index=False)\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T19:26:47.490094Z","iopub.execute_input":"2021-06-22T19:26:47.49049Z","iopub.status.idle":"2021-06-22T19:26:47.507409Z","shell.execute_reply.started":"2021-06-22T19:26:47.490452Z","shell.execute_reply":"2021-06-22T19:26:47.506582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}