{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nimport os\nimport re\nimport json\nimport glob\nfrom copy import deepcopy\nfrom collections import defaultdict\nfrom textblob import TextBlob\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nfrom nltk import sent_tokenize\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport seaborn as sns\n\nimport unidecode\n\nfrom tqdm.notebook import tqdm\nimport string\n\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification\nimport torch\n\n%matplotlib inline\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')\n\ndef clean_text(txt):\n    return [re.sub('[^A-Za-z0-9]+', ' ', t.lower()) for t in txt]\n\ndevice='cuda' if torch.cuda.is_available() else 'cpu'","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","scrolled":true,"execution":{"iopub.status.busy":"2021-06-03T22:04:24.997495Z","iopub.execute_input":"2021-06-03T22:04:24.997872Z","iopub.status.idle":"2021-06-03T22:04:30.528664Z","shell.execute_reply.started":"2021-06-03T22:04:24.997842Z","shell.execute_reply":"2021-06-03T22:04:30.52786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"basepath='/kaggle/input/coleridgeinitiative-show-us-the-data/'\ntrain_df=pd.read_csv(basepath+'train.csv')\nsample_sub = pd.read_csv(basepath+'sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-03T22:04:30.529911Z","iopub.execute_input":"2021-06-03T22:04:30.530318Z","iopub.status.idle":"2021-06-03T22:04:30.696313Z","shell.execute_reply.started":"2021-06-03T22:04:30.53029Z","shell.execute_reply":"2021-06-03T22:04:30.695487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:38:28.822382Z","iopub.execute_input":"2021-05-31T14:38:28.822725Z","iopub.status.idle":"2021-05-31T14:38:28.845311Z","shell.execute_reply.started":"2021-05-31T14:38:28.822687Z","shell.execute_reply":"2021-05-31T14:38:28.844047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_path=basepath+'train/'\ntest_files_path=basepath+'test/'\ndef read_append_return(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.extend(sent_tokenize(unidecode.unidecode(data.get('section_title'))))\n            contents.extend(sent_tokenize(unidecode.unidecode(data.get('text'))))\n            combined.extend(sent_tokenize(unidecode.unidecode(data.get('section_title'))))\n            combined.extend(sent_tokenize(unidecode.unidecode(data.get('text'))))\n    \n    if output == 'text':\n        return contents\n    elif output == 'head':\n        return headings\n    else:\n        return combined","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:38:28.846945Z","iopub.execute_input":"2021-05-31T14:38:28.847286Z","iopub.status.idle":"2021-05-31T14:38:28.854213Z","shell.execute_reply.started":"2021-05-31T14:38:28.84725Z","shell.execute_reply":"2021-05-31T14:38:28.853078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()   #tqdm is used to show any code running with a progress bar. \ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:38:28.855845Z","iopub.execute_input":"2021-05-31T14:38:28.856271Z","iopub.status.idle":"2021-05-31T14:44:52.05609Z","shell.execute_reply.started":"2021-05-31T14:38:28.856231Z","shell.execute_reply":"2021-05-31T14:44:52.055117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:44:52.059816Z","iopub.execute_input":"2021-05-31T14:44:52.060161Z","iopub.status.idle":"2021-05-31T14:44:52.339222Z","shell.execute_reply.started":"2021-05-31T14:44:52.060124Z","shell.execute_reply":"2021-05-31T14:44:52.338276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['cleaned_text']=train_df.text.progress_apply(clean_text)\nsample_sub['cleaned_text']=sample_sub.text.progress_apply(clean_text)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:44:52.340691Z","iopub.execute_input":"2021-05-31T14:44:52.341028Z","iopub.status.idle":"2021-05-31T14:44:53.362455Z","shell.execute_reply.started":"2021-05-31T14:44:52.340991Z","shell.execute_reply":"2021-05-31T14:44:53.361671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels_list=train_df.cleaned_label","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:44:53.365068Z","iopub.execute_input":"2021-05-31T14:44:53.365538Z","iopub.status.idle":"2021-05-31T14:44:53.371976Z","shell.execute_reply.started":"2021-05-31T14:44:53.365499Z","shell.execute_reply":"2021-05-31T14:44:53.371111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels=[]\nsub_labels=[]\nfor text in tqdm(train_df.cleaned_text):\n    text=' '.join(text)\n    tmp=sorted([label for label in labels_list if label in text],key=lambda x: len(x))\n    result=[]\n    for i,label in enumerate(tmp):\n        try:\n            if sum([label in ref for ref in tmp[i+1:]])>0:\n                continue\n            else:\n                result.append(label)\n        except:\n            result.append(label)\n    labels.append(sorted(result))\n    \nfor text in tqdm(sample_sub.cleaned_text):\n    text=' '.join(text)\n    tmp=sorted([label for label in labels_list if label in text],key=lambda x: len(x))\n    sub_labels.append(tmp)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:44:53.373931Z","iopub.execute_input":"2021-05-31T14:44:53.374439Z","iopub.status.idle":"2021-05-31T14:45:54.689332Z","shell.execute_reply.started":"2021-05-31T14:44:53.374403Z","shell.execute_reply":"2021-05-31T14:45:54.688377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['complete_lower_labels']=labels","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:45:54.690752Z","iopub.execute_input":"2021-05-31T14:45:54.691093Z","iopub.status.idle":"2021-05-31T14:45:54.707099Z","shell.execute_reply.started":"2021-05-31T14:45:54.691055Z","shell.execute_reply":"2021-05-31T14:45:54.706058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_text_labels(row):\n    text_labels=[]\n    for i,txt in enumerate(row['cleaned_text']):\n        have_label=False\n        for existing in row['complete_lower_labels']:\n            tmp_span=[m.span() for m in re.finditer(existing,txt)]\n            if len(tmp_span) > 0:\n                have_label=True\n                break\n        text_labels.append(int(have_label))\n    return text_labels","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:45:54.708646Z","iopub.execute_input":"2021-05-31T14:45:54.709001Z","iopub.status.idle":"2021-05-31T14:45:54.71953Z","shell.execute_reply.started":"2021-05-31T14:45:54.708967Z","shell.execute_reply":"2021-05-31T14:45:54.718651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text_labels']=train_df.progress_apply(get_text_labels,axis=1)","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:45:54.72084Z","iopub.execute_input":"2021-05-31T14:45:54.721206Z","iopub.status.idle":"2021-05-31T14:46:01.456306Z","shell.execute_reply.started":"2021-05-31T14:45:54.721168Z","shell.execute_reply":"2021-05-31T14:46:01.455354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_texts=[t.strip() for txt in train_df.cleaned_text for t in txt]\ntraining_labels=[label for labels in train_df['text_labels'] for label in labels]\ntest_texts=[[t.strip() for t in txt] for txt in sample_sub.cleaned_text]","metadata":{"execution":{"iopub.status.busy":"2021-05-31T14:46:01.457642Z","iopub.execute_input":"2021-05-31T14:46:01.45799Z","iopub.status.idle":"2021-05-31T14:46:01.542939Z","shell.execute_reply.started":"2021-05-31T14:46:01.457954Z","shell.execute_reply":"2021-05-31T14:46:01.542155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"processed_train_df=pd.DataFrame(zip(training_texts,training_labels),columns=['text','label'])\nprocessed_test_df=pd.DataFrame(zip(test_texts,sample_sub.Id),columns=['text','Id'])\n\nprocessed_train_df.to_csv('processed_train_df.csv',index=False)\nprocessed_test_df.to_csv('processed_test_df.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_final_labels=[]\nfor i in sub_labels:\n    tmp=[]\n    for index,label in enumerate(i):\n        try:\n            if sum([label in ref for ref in i[index+1:]])>0:\n                continue\n            else:\n                tmp.append(label)\n        except:\n            tmp.append(label)\n    sub_final_labels.append('|'.join(sorted(tmp)))\n\nsubmissions=pd.DataFrame(zip(sample_sub.Id,sub_final_labels),columns=['Id','PredictionString'])\nsubmissions.to_csv('submission.csv',index=False)","metadata":{},"execution_count":null,"outputs":[]}]}