{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"---\n## 任意の単語数で分割する関数","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport gc\nimport json\nimport glob\nimport pickle\nimport string\nimport random\nimport itertools\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\n\nimport nltk\nfrom nltk.corpus import stopwords\n\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\nimport torch\nfrom torch import nn, optim\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.optim.lr_scheduler import CosineAnnealingLR\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer, seed_everything\nfrom pytorch_lightning.loggers import CometLogger\nimport transformers\nfrom transformers import BertModel, BertForTokenClassification\n\n%matplotlib inline\n\npd.set_option('display.max_columns', 500)\npd.set_option('display.max_rows', 300)\npd.set_option('display.max_colwidth', 300)\npd.options.display.float_format = '{:.5f}'.format","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-06-02T02:04:58.888303Z","iopub.execute_input":"2021-06-02T02:04:58.888722Z","iopub.status.idle":"2021-06-02T02:05:09.058225Z","shell.execute_reply.started":"2021-06-02T02:04:58.888634Z","shell.execute_reply":"2021-06-02T02:05:09.057027Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(data_dir):    \n    # Testデータの読み込み\n    test_files = glob.glob(data_dir + \"test/*.json\")\n\n    test = pd.DataFrame()\n\n    # jsonからDataFrameに\n    for tar in test_files:\n        file_data = pd.read_json(tar)\n        file_data.insert(0,'pub_id', tar.split('/')[-1].split('.')[0])\n        test = pd.concat([test, file_data])\n    \n    return test\n\n\ndef preprocess_text(text: str) -> str:\n    \"\"\"\n    テキストの前処理　クリーニング\n    \"\"\"\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    \n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.060286Z","iopub.execute_input":"2021-06-02T02:05:09.060759Z","iopub.status.idle":"2021-06-02T02:05:09.068951Z","shell.execute_reply.started":"2021-06-02T02:05:09.060710Z","shell.execute_reply":"2021-06-02T02:05:09.067952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/coleridgeinitiative-show-us-the-data/'\ntest = load_data(data_dir)\n\ntest['text'] = test['text'].apply(lambda x: preprocess_text(x) if isinstance(x, str) else x)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.070953Z","iopub.execute_input":"2021-06-02T02:05:09.071251Z","iopub.status.idle":"2021-06-02T02:05:09.191431Z","shell.execute_reply.started":"2021-06-02T02:05:09.071221Z","shell.execute_reply":"2021-06-02T02:05:09.190282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.193490Z","iopub.execute_input":"2021-06-02T02:05:09.193927Z","iopub.status.idle":"2021-06-02T02:05:09.216701Z","shell.execute_reply.started":"2021-06-02T02:05:09.193883Z","shell.execute_reply":"2021-06-02T02:05:09.215936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Expand Dataset","metadata":{}},{"cell_type":"code","source":"# 確認のため一つのpub_idに絞る\ntest_ids = test['pub_id'].unique()\ntar = test[test['pub_id'] == test_ids[0]]","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.217759Z","iopub.execute_input":"2021-06-02T02:05:09.218181Z","iopub.status.idle":"2021-06-02T02:05:09.225327Z","shell.execute_reply.started":"2021-06-02T02:05:09.218152Z","shell.execute_reply":"2021-06-02T02:05:09.224220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tar","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.226502Z","iopub.execute_input":"2021-06-02T02:05:09.227002Z","iopub.status.idle":"2021-06-02T02:05:09.249085Z","shell.execute_reply.started":"2021-06-02T02:05:09.226971Z","shell.execute_reply":"2021-06-02T02:05:09.247896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# それぞれのセクションのテキストの数を計算する\ntar['text_len'] = tar['text'].apply(lambda x: len(x.split(' ')) if isinstance(x, str) else 0)\ntar[['pub_id', 'section_title', 'text_len']]","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.250650Z","iopub.execute_input":"2021-06-02T02:05:09.251005Z","iopub.status.idle":"2021-06-02T02:05:09.269767Z","shell.execute_reply.started":"2021-06-02T02:05:09.250971Z","shell.execute_reply":"2021-06-02T02:05:09.268937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tar.drop('text_len', axis=1, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.272200Z","iopub.execute_input":"2021-06-02T02:05:09.272738Z","iopub.status.idle":"2021-06-02T02:05:09.292751Z","shell.execute_reply.started":"2021-06-02T02:05:09.272696Z","shell.execute_reply":"2021-06-02T02:05:09.291773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 指定したmax_lenより大きい場合は分割して行を分ける\n# 重複も許すように設計する\n\n# テキストの数\nmax_len = 32\n# 重複する単語数\noverride = 10\n# 結果格納用データフレーム\nres = pd.DataFrame()\n\nfor i in range(len(tar)):\n    row = tar.iloc[i]\n    tar_text = row['text'].split(' ')\n    text_len = len(tar_text)\n    \n    # 単語数がmax_lenより小さい場合はそのまま\n    if text_len < max_len:\n        res = pd.concat([res, pd.DataFrame(row).T], axis=0)\n        continue\n    \n    # 単語数がmax_lenより大きい場合は分割する\n    elif text_len > max_len:\n        # 分割する数を計算する\n        num_divide = int(np.ceil(text_len / (max_len - override)))\n        # 分割する分行を複製しておく（データフレーム化）\n        tmp_df = pd.DataFrame([row] * num_divide)\n        # 分割後のテキストを格納しておくリスト\n        divided_texts = []\n        \n        # max_lenごとのテキストに分割する\n        for i in range(len(tmp_df)):\n            div_text = tar_text[int(i * (max_len - override)) : int(i * (max_len - override) + max_len)]\n            # リストから文字列に直す\n            div_text = ' '.join(div_text)\n            # 結果を一旦リストにまとめておく\n            divided_texts.append(div_text)\n            \n        # 複製しておいたデータフレームに置換\n        tmp_df['text'] = divided_texts\n        # 全体のデータフレームに結合\n        res = pd.concat([res, tmp_df], axis=0)\n        \n    # 動作確認のため強制終了\n    break","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.294404Z","iopub.execute_input":"2021-06-02T02:05:09.294714Z","iopub.status.idle":"2021-06-02T02:05:09.308258Z","shell.execute_reply.started":"2021-06-02T02:05:09.294683Z","shell.execute_reply":"2021-06-02T02:05:09.307342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.309420Z","iopub.execute_input":"2021-06-02T02:05:09.309927Z","iopub.status.idle":"2021-06-02T02:05:09.329945Z","shell.execute_reply.started":"2021-06-02T02:05:09.309879Z","shell.execute_reply":"2021-06-02T02:05:09.328830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 単語数の確認\nres['text_len'] = res['text'].apply(lambda x: len(x.split(' ')))\nres","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.331186Z","iopub.execute_input":"2021-06-02T02:05:09.331464Z","iopub.status.idle":"2021-06-02T02:05:09.347152Z","shell.execute_reply.started":"2021-06-02T02:05:09.331436Z","shell.execute_reply":"2021-06-02T02:05:09.346432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 対象のテキストを一応表示しておく\ntar['text'].values[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.348209Z","iopub.execute_input":"2021-06-02T02:05:09.348677Z","iopub.status.idle":"2021-06-02T02:05:09.361008Z","shell.execute_reply.started":"2021-06-02T02:05:09.348639Z","shell.execute_reply":"2021-06-02T02:05:09.360009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 関数化\n\n上の処理を関数化しておく","metadata":{}},{"cell_type":"code","source":"def preprocess_text(text: str) -> str:\n    \"\"\"\n    テキストの前処理　クリーニング\n    \"\"\"\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    \n    return text\n\n\ndef expand_data(df, max_len, override=0) -> pd.DataFrame:\n    \"\"\"\n    指定したmax_lenを超えるテキストに対して分割を行う関数\n    \n    ---------------------------------------\n    Parameters\n    \n    df: pd.DataFrame\n        拡張対象のデータフレーム\n        pub_id, section_title, textが存在していること\n    max_len: int\n        分割する単語数\n    override: int\n        分割する際に重複する単語数\n        \n    ---------------------------------------\n    Returns\n    \n    res: pd.DataFrame\n        分割したテキストで構成されたデータフレーム\n    \n    \"\"\"\n    # 結果格納用データフレーム\n    res = pd.DataFrame()\n    \n    # テキストの前処理\n    df['text_clean'] = df['text'].apply(lambda x: preprocess_text(x) if isinstance(x, str) else x)\n    \n    ids = df['pub_id'].unique()\n    \n    for _id in ids:   \n        tar = df[df['pub_id'] == _id]\n\n        for i in range(len(tar)):\n            row = tar.iloc[i]\n            tar_text = row['text_clean'].split(' ')\n            text_len = len(tar_text)\n\n            # 単語数がmax_lenより小さい場合はそのまま\n            if text_len <= max_len:\n                res = pd.concat([res, pd.DataFrame(row).T], axis=0)   # Version 2で修正\n\n            # 単語数がmax_lenより大きい場合は分割する\n            elif text_len > max_len:\n                # 分割する数を計算する\n                num_divide = int(np.ceil(text_len / (max_len - override)))\n                # 分割する分行を複製しておく（データフレーム化）\n                tmp_df = pd.DataFrame([row] * num_divide)\n                # 分割後のテキストを格納しておくリスト\n                divided_texts = []\n\n                # max_lenごとのテキストに分割する\n                for i in range(len(tmp_df)):\n                    div_text = tar_text[int(i * (max_len - override)) : int(i * (max_len - override) + max_len)]\n                    # リストから文字列に直す\n                    div_text = ' '.join(div_text)\n                    # 結果を一旦リストにまとめておく\n                    divided_texts.append(div_text)\n\n                # 複製しておいたデータフレームに置換\n                tmp_df['text_clean'] = divided_texts\n                # 全体のデータフレームに結合\n                res = pd.concat([res, tmp_df], axis=0)\n                \n    # 余計な行を削除する\n    res = res.dropna()\n    res = res.reset_index(drop=True)\n    \n    return res","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.362571Z","iopub.execute_input":"2021-06-02T02:05:09.363020Z","iopub.status.idle":"2021-06-02T02:05:09.377829Z","shell.execute_reply.started":"2021-06-02T02:05:09.362978Z","shell.execute_reply":"2021-06-02T02:05:09.377021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nres = expand_data(test, max_len=128, override=3)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.379307Z","iopub.execute_input":"2021-06-02T02:05:09.380009Z","iopub.status.idle":"2021-06-02T02:05:09.645375Z","shell.execute_reply.started":"2021-06-02T02:05:09.379961Z","shell.execute_reply":"2021-06-02T02:05:09.644344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.646720Z","iopub.execute_input":"2021-06-02T02:05:09.647134Z","iopub.status.idle":"2021-06-02T02:05:09.665449Z","shell.execute_reply.started":"2021-06-02T02:05:09.647087Z","shell.execute_reply":"2021-06-02T02:05:09.664355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('拡張前')\nprint(test.shape)\nprint('拡張後')\nprint(res.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.666877Z","iopub.execute_input":"2021-06-02T02:05:09.667289Z","iopub.status.idle":"2021-06-02T02:05:09.678639Z","shell.execute_reply.started":"2021-06-02T02:05:09.667246Z","shell.execute_reply":"2021-06-02T02:05:09.677460Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = '../input/coleridgeinitiative-show-us-the-data/'\ntest = load_data(data_dir)\n\nres = expand_data(test, max_len=512, override=10)","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.680230Z","iopub.execute_input":"2021-06-02T02:05:09.680953Z","iopub.status.idle":"2021-06-02T02:05:09.948811Z","shell.execute_reply.started":"2021-06-02T02:05:09.680904Z","shell.execute_reply":"2021-06-02T02:05:09.947870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[test['pub_id'] == '8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60']","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.950611Z","iopub.execute_input":"2021-06-02T02:05:09.950966Z","iopub.status.idle":"2021-06-02T02:05:09.969491Z","shell.execute_reply.started":"2021-06-02T02:05:09.950932Z","shell.execute_reply":"2021-06-02T02:05:09.968066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"res[res['pub_id'] == '8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60']","metadata":{"execution":{"iopub.status.busy":"2021-06-02T02:05:09.971175Z","iopub.execute_input":"2021-06-02T02:05:09.971623Z","iopub.status.idle":"2021-06-02T02:05:09.995270Z","shell.execute_reply.started":"2021-06-02T02:05:09.971583Z","shell.execute_reply":"2021-06-02T02:05:09.994384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}