{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\nimport json\nfrom tqdm import tqdm\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\nfrom IPython.core.display import HTML\nfrom IPython.core import display\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n# for filename in filenames:\n# print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-12T10:07:33.111382Z","iopub.execute_input":"2021-06-12T10:07:33.111713Z","iopub.status.idle":"2021-06-12T10:07:33.116422Z","shell.execute_reply.started":"2021-06-12T10:07:33.111668Z","shell.execute_reply":"2021-06-12T10:07:33.115433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HTML(\"\"\"\n\n\n\n\"\"\")\n","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:07:33.118008Z","iopub.execute_input":"2021-06-12T10:07:33.118420Z","iopub.status.idle":"2021-06-12T10:07:33.131358Z","shell.execute_reply.started":"2021-06-12T10:07:33.118365Z","shell.execute_reply":"2021-06-12T10:07:33.130714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef text_cleaning(text):\n '''\n Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n text - Sentence that needs to be cleaned\n '''\n# text = ''.join([k for k in text if k not in string.punctuation])\n text = re.sub('[^A-Za-z0-9]+', ' ', text)\n# text = re.sub(\"/'+/g\", ' ', text)\n \n return text","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:07:33.132621Z","iopub.execute_input":"2021-06-12T10:07:33.132864Z","iopub.status.idle":"2021-06-12T10:07:33.140930Z","shell.execute_reply.started":"2021-06-12T10:07:33.132841Z","shell.execute_reply":"2021-06-12T10:07:33.139943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:07:33.142864Z","iopub.execute_input":"2021-06-12T10:07:33.143224Z","iopub.status.idle":"2021-06-12T10:07:33.292201Z","shell.execute_reply.started":"2021-06-12T10:07:33.143198Z","shell.execute_reply":"2021-06-12T10:07:33.291243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output=''):\n \"\"\"\n Function to read json file and then return the text data from them and append to the dataframe\n \"\"\"\n json_path = os.path.join(train_files_path, (filename+'.json'))\n headings = []\n contents = []\n combined = []\n with open(json_path, 'r') as f:\n json_decode = json.load(f)\n for data in json_decode:\n headings.append(data.get('section_title'))\n contents.append(data.get('text'))\n combined.append(data.get('section_title'))\n combined.append(' \\\\n ')\n combined.append(data.get('text'))\n \n all_headings = ' '.join(headings)\n all_contents = ' '.join(contents)\n all_data = '. '.join(combined)\n \n if output == 'text':\n return all_contents#text_cleaning(all_contents.lower())\n elif output == 'head':\n return all_headings\n else:\n return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:07:33.293487Z","iopub.execute_input":"2021-06-12T10:07:33.293763Z","iopub.status.idle":"2021-06-12T10:07:33.301426Z","shell.execute_reply.started":"2021-06-12T10:07:33.293739Z","shell.execute_reply":"2021-06-12T10:07:33.300556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nimport joblib\n\ndef load_data(df, func=read_append_return):\n pool = joblib.Parallel(4)\n mapper = joblib.delayed(func)\n tasks = [mapper(filename) for filename in df['Id']]\n res = pool(tqdm(tasks))\n return res\n\n\ntrain['text'] = load_data(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:07:33.302574Z","iopub.execute_input":"2021-06-12T10:07:33.302923Z","iopub.status.idle":"2021-06-12T10:07:52.358682Z","shell.execute_reply.started":"2021-06-12T10:07:33.302896Z","shell.execute_reply":"2021-06-12T10:07:52.357670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:07:52.359818Z","iopub.execute_input":"2021-06-12T10:07:52.360169Z","iopub.status.idle":"2021-06-12T10:07:52.380899Z","shell.execute_reply.started":"2021-06-12T10:07:52.360138Z","shell.execute_reply":"2021-06-12T10:07:52.380082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"idx = np.random.randint(0, len(train), 500)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:07:52.383141Z","iopub.execute_input":"2021-06-12T10:07:52.383434Z","iopub.status.idle":"2021-06-12T10:07:52.387503Z","shell.execute_reply.started":"2021-06-12T10:07:52.383408Z","shell.execute_reply":"2021-06-12T10:07:52.386435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_label(row, is_lower=False):\n title = row.pub_title\n txt = row.text\n label = row.dataset_label\n clean_label = row.cleaned_label\n if is_lower:\n txt = text_cleaning(txt)\n start_idx = txt.lower().index(clean_label)\n else:\n start_idx = txt.index(label)\n end_idx = start_idx + len(label)\n context_size = 100\n\n p1 = txt[max(0, start_idx-context_size):start_idx]\n p2 = txt[end_idx: min(end_idx+context_size, len(txt))]\n seg = f'

{title}

{p1}{label}{p2}

'\n return HTML(seg)","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:10:30.966178Z","iopub.execute_input":"2021-06-12T10:10:30.966741Z","iopub.status.idle":"2021-06-12T10:10:30.973258Z","shell.execute_reply.started":"2021-06-12T10:10:30.966709Z","shell.execute_reply":"2021-06-12T10:10:30.972298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in idx:\n row = train.iloc[i]\n print(i)\n try:\n a = show_label(row)\n except:\n print('find failed', i)\n a = show_label(row, True)\n \n display.display(a)\n print()","metadata":{"execution":{"iopub.status.busy":"2021-06-12T10:10:33.533571Z","iopub.execute_input":"2021-06-12T10:10:33.534019Z","iopub.status.idle":"2021-06-12T10:10:34.928503Z","shell.execute_reply.started":"2021-06-12T10:10:33.533992Z","shell.execute_reply":"2021-06-12T10:10:34.927592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}