{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%time\n!pip uninstall fastai en-core-web-sm en-core-web-lg spacy -y -q","metadata":{"execution":{"iopub.status.busy":"2021-06-18T15:23:51.562116Z","iopub.execute_input":"2021-06-18T15:23:51.562463Z","iopub.status.idle":"2021-06-18T15:24:00.305974Z","shell.execute_reply.started":"2021-06-18T15:23:51.562434Z","shell.execute_reply":"2021-06-18T15:24:00.304824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!pip install ../input/spacy3/catalogue-2.0.3-py3-none-any.whl ../input/spacy3/typer-0.3.2-py3-none-any.whl ../input/spacy3/srsly-2.4.1-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/pathy-0.5.2-py3-none-any.whl ../input/spacy3/smart_open-3.0.0-py3-none-any.whl ../input/spacy3/pydantic-1.7.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/thinc-8.0.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy-3.0.6-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy_legacy-3.0.5-py2.py3-none-any.whl -q","metadata":{"execution":{"iopub.status.busy":"2021-06-18T15:24:00.307718Z","iopub.execute_input":"2021-06-18T15:24:00.308031Z","iopub.status.idle":"2021-06-18T15:24:47.680992Z","shell.execute_reply.started":"2021-06-18T15:24:00.308000Z","shell.execute_reply":"2021-06-18T15:24:47.679835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!pip install ../input/spacy3/catalogue-2.0.3-py3-none-any.whl ../input/spacy3/typer-0.3.2-py3-none-any.whl ../input/spacy3/srsly-2.4.1-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/pathy-0.5.2-py3-none-any.whl ../input/spacy3/smart_open-3.0.0-py3-none-any.whl ../input/spacy3/pydantic-1.7.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/thinc-8.0.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy-3.0.6-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy_legacy-3.0.5-py2.py3-none-any.whl -q","metadata":{"execution":{"iopub.status.busy":"2021-06-18T15:24:47.685150Z","iopub.execute_input":"2021-06-18T15:24:47.685497Z","iopub.status.idle":"2021-06-18T15:25:14.816002Z","shell.execute_reply.started":"2021-06-18T15:24:47.685463Z","shell.execute_reply":"2021-06-18T15:25:14.814620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!pip install ../input/spacy3/en_core_web_lg-3.0.0-py3-none-any.whl ../input/spacy3/en_core_web_md-3.0.0-py3-none-any.whl ../input/spacy3/en_core_web_sm-3.0.0-py3-none-any.whl -q","metadata":{"execution":{"iopub.status.busy":"2021-06-18T15:25:14.818067Z","iopub.execute_input":"2021-06-18T15:25:14.818678Z","iopub.status.idle":"2021-06-18T15:25:59.737639Z","shell.execute_reply.started":"2021-06-18T15:25:14.818625Z","shell.execute_reply":"2021-06-18T15:25:59.735405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n!pip install ../input/spacy3/spacy_alignments-0.8.3-cp37-cp37m-manylinux2014_x86_64.whl ../input/spacy3/spacy_transformers-1.0.2-py2.py3-none-any.whl ../input/spacy3/en_core_web_trf-3.0.0-py3-none-any.whl -q","metadata":{"execution":{"iopub.status.busy":"2021-06-18T15:25:59.742598Z","iopub.execute_input":"2021-06-18T15:25:59.743031Z","iopub.status.idle":"2021-06-18T15:26:39.938847Z","shell.execute_reply.started":"2021-06-18T15:25:59.742984Z","shell.execute_reply":"2021-06-18T15:26:39.937761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import spacy\nassert spacy.__version__ == '3.0.6'","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:19.330855Z","iopub.execute_input":"2021-06-18T18:08:19.331686Z","iopub.status.idle":"2021-06-18T18:08:20.128344Z","shell.execute_reply.started":"2021-06-18T18:08:19.331562Z","shell.execute_reply":"2021-06-18T18:08:20.125173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport warnings\nfrom itertools import chain\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\nimport itertools\nimport os\nimport re\nimport json\nimport glob\nfrom pathlib import Path\nimport numpy as np\nimport nltk\nfrom tqdm.autonotebook import tqdm\nnlp = spacy.load('en_core_web_lg')\nnlp.max_length = 10000000\n%matplotlib inline","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.129781Z","iopub.status.idle":"2021-06-18T18:08:20.130240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_files_path =  '../input/coleridgeinitiative-show-us-the-data/train'\n# submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n# submission_df = submission_df.sample(n=10)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.131366Z","iopub.status.idle":"2021-06-18T18:08:20.131770Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_files_path = '../input/coleridgeinitiative-show-us-the-data/test'\nsubmission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.132885Z","iopub.status.idle":"2021-06-18T18:08:20.133431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, files_path = test_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.134432Z","iopub.status.idle":"2021-06-18T18:08:20.134836Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()   #tqdm is used to show any code running with a progress bar. \nsubmission_df['text'] = submission_df['Id'].progress_apply(read_append_return)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.135691Z","iopub.status.idle":"2021-06-18T18:08:20.136146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k if k not in '!\"#$%&\\'()*+,-/:;<=>?@[\\\\]^_`{|}~' else ' ' for k in text ])\n    text = re.sub('[^A-Za-z0-9.]+', ' ', str(text).lower()).strip()\n    \n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.137118Z","iopub.status.idle":"2021-06-18T18:08:20.137502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\nsubmission_df['text1'] = submission_df['text'].progress_apply(text_cleaning)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.138470Z","iopub.status.idle":"2021-06-18T18:08:20.138891Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"output_dir=Path('../input/coleridgemodel2/models3')\nprint(\"Loading from\",output_dir)\nnlp2 = spacy.load(output_dir)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.139756Z","iopub.status.idle":"2021-06-18T18:08:20.140329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_string_max_len(temp_df, grouped_indexes):\n    filtered_indexes = []\n    for _ in grouped_indexes:\n        temp = [len(temp_df.loc[i, 'PredictionString']) for i in _]\n        temp_dict = dict(zip(_,temp))\n        res = max(temp_dict.items(), key=operator.itemgetter(1))[0]\n        filtered_indexes.append(res)\n    return filtered_indexes\n\ndef reduce_similar_preds(pred_list):\n    temp_df = pd.DataFrame({'PredictionString': pred_list})\n    # create graph objects\n    def compare(tweet1, threshold=0.70):\n    # compare tweets using Levenshtein distance (or whatever string comparison metric) \n        matches = temp_df['PredictionString'].apply(lambda tweet2:(Levenshtein.ratio(tweet1, tweet2) >= threshold))\n\n        # get positive matches\n        matches = matches[matches].index.tolist()\n\n        # convert to list of tuples\n        return [*zip(iter(matches[:-1]), iter(matches[1:]))]\n\n    # create graph objects\n    nodes = temp_df.index.tolist()\n    edges = [*itertools.chain(*temp_df[\"PredictionString\"].apply(compare))]\n\n    # create graphs\n    G = nx.Graph()\n    G.add_nodes_from(nodes)\n    G.add_edges_from(edges)\n\n    # get connected component indexes\n    grouped_indexes = [*nx.connected_components(G)]\n\n    # get a random choice index from each group\n#     filtered_indexes = [random.choice([*_]) for _ in grouped_indexes]\n    filtered_indexes = get_string_max_len(temp_df, grouped_indexes)\n\n    return temp_df.loc[filtered_indexes, 'PredictionString'].to_list()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.141374Z","iopub.status.idle":"2021-06-18T18:08:20.141856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nimport Levenshtein\nimport networkx as nx\nimport operator\nall_pred_list = []\nall_Id_list = []\nn = 999999\n\nfor i in tqdm(range(submission_df.shape[0])): \n#     If length of the text is more than 999999 break it into multiple chunks of size 999999\n    if len(submission_df.loc[i]['text1']) > 999999:\n        chunks = [submission_df.loc[i]['text1'][j:j+n] for j in range(0, len(submission_df.loc[0]['text1']), n)]\n        pred_list = []\n        for chunk in chunks:\n            doc = nlp2(chunk)\n            pred_list.append(list(set([str(e) for e in list(doc.ents)])))\n        pred_list = list(set(chain.from_iterable(pred_list)))\n    else:\n        doc = nlp2(submission_df.loc[i]['text1'])\n        pred_list = list(set([str(e) for e in list(doc.ents)]))\n\n    if len(pred_list)>1:\n        pred_list = reduce_similar_preds(pred_list)\n        \n    pred = '|'.join(pred_list)\n    if not pred:\n        pred = ''\n    all_pred_list.append(pred)\n    all_Id_list.append(submission_df.loc[i]['Id'])\n\nout_df = pd.DataFrame({'Id': all_Id_list, 'PredictionString': all_pred_list})","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.142761Z","iopub.status.idle":"2021-06-18T18:08:20.143240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"out_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T18:08:20.144089Z","iopub.status.idle":"2021-06-18T18:08:20.144501Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}