{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# What's this?\nI will share how I load training texts and store as a `pandas.DataFrame`. This notebook creates a DataFrame as described below.  \nThe created DataFrame is saved as a pickle file.\n\n| Column name       | Description                                                                                                                   | \n| ----------------- | ----------------------------------------------------------------------------------------------------------------------------- | \n| section<br>title  | section title.<br>Loaded from given json files.                                                                               | \n| text              | text data.<br>Loaded from given json files.                                                                                   | \n| section<br>num    | The section numbers.                                                                                                          | \n| entities          | The position of detected datasets.<br>List of tuple(start_position, end_position, 'DATASET').<br>This is spaCy's data format. | \n| spans             | The position of detected datasets.<br>List of tuple(start_position, end_position).<br>Almost the same content as \"entities\".  | \n| matched<br>labels | The list of matched dataset labels.<br>Same order as \"entities\" and \"spans\".                                                  | \n| is_label<br>exist | Whether there are matches within the section. (Bool)                                                                          | \n| document<br>id    | The document_id of the section.                                                                                               | \n| true<br>labels    | The true_labels of the document.                                                                                              | \n\n# How to use this DataFrame?\nYou can use the created DataFrame just by loading this notebook in your notebook.\n\n- Click \"+ Add data\" at the top right of Notebook edit page.\n- Click \"Notebook output files\".\n- In the search box, type \"for-beginners-load-all-datas-as-dataframe\".\n- If this notebook is found, click \"Add data\". (If this notebook is not found, you can upvote this notebook and find this notebook clicking \"Your Favorites\", as a last resort.)\n\nThen, run this code.\n\n```python\nimport pandas as pd\ntrain_text_df = pd.read_pickle('../input/for-beginners-read-train-texts-as-dataframe/train_text_df.pkl')\n```\n\nNow you can use the created DataFrame as `train_text_df`. Of course, you can copy and paste the code below and use it. \nPlease note that it takes about 10 minutes to run this code at Kaggle environment.\n\nIf there are any problems, please let me know. Good luck! 😉","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport re\nimport pathlib\nimport pickle\nimport json\n\nfrom collections import defaultdict\nfrom tqdm import tqdm","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"input_path = pathlib.Path('../input/coleridgeinitiative-show-us-the-data')\ntrain_df = pd.read_csv(input_path / 'train.csv')\nsubmission_df = pd.read_csv(input_path / 'sample_submission.csv')\ntrain_files = input_path.glob('train/*.json')\ntest_files = input_path.glob('test/*.json')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text: str) -> str:\n    \"\"\"\n    https://www.kaggle.com/c/coleridgeinitiative-show-us-the-data/overview/evaluation\n    \"\"\"\n    \n    return re.sub('[^A-Za-z0-9]+', ' ', str(text).lower())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntmp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntmp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(tmp_1 + tmp_2 + tmp_3)\nmatching_text = '|'.join(existing_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# about 8 minutes loop\ntrain_text_dic = defaultdict(list)\ntest_text_dic = defaultdict(list)\ntrain_files = input_path.glob('train/*.json')\n\n# Load all training files.\nfor filepath in tqdm(train_files):\n    \n    with open(filepath, 'rb') as f:\n        datas = json.load(f)\n    \n    for i, data in enumerate(datas):\n        train_text_dic['section_title'].append(data['section_title'])\n        train_text_dic['text'].append(data['text'])\n        train_text_dic['section_num'].append(i)\n        matches = re.finditer(matching_text, data['text'].lower())\n        \n        # Are there any matches with true labels?\n        entities = []\n        spans = []\n        matched_labels = []\n        for match in matches:\n            entities.append((match.start(), match.end(), 'DATASET'))\n            spans.append((match.start(), match.end()))\n            matched_labels.append(data['text'][match.start():match.end()])\n        \n        train_text_dic['entities'].append(entities)\n        train_text_dic['spans'].append(spans)\n        train_text_dic['matched_labels'].append(matched_labels)\n        \n        train_text_dic['is_label_exist'].append(len(matched_labels) != 0)\n    \n    fileid = filepath.stem\n    train_text_dic['document_id'] += [fileid] * (i + 1)\n    \n    true_labels = list(train_df.query('Id == @fileid')['dataset_label'])\n    train_text_dic['true_labels'] += [true_labels] * (i + 1)\n    \n# # for test_files (without matching detection)\n# for filepath in test_files:\n#     with open(filepath, 'rb') as f:\n#         datas = json.load(f)\n    \n#     for i, data in enumerate(datas):\n#         test_text_dic['section_title'] += [data['section_title']]\n#         test_text_dic['text'] += [data['text']]\n#         test_text_dic['section_num'] += [i]\n    \n#     fileid = filepath.stem\n#     test_text_dic['document_id'] += [fileid] * (i + 1)\n\ntrain_text_df = pd.DataFrame(train_text_dic)\n# test_text_df = pd.DataFrame(test_text_dic)\n\ntrain_text_df.to_pickle('train_text_df.pkl')\n# test_text_df.to_pickle('test_text_df.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_text_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}