{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## literal matching","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n#jsonモジュールのインポート\nimport json\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport re\nimport string\nimport gc","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_text_from_json(dataInd,fileId):\n    filename = \"/kaggle/input/coleridgeinitiative-show-us-the-data/\" + dataInd + \"/\" + fileId + \".json\"\n    \n    fd = open(filename, mode='r')\n    data = json.load(fd)\n    fd.close()\n    json_text = ''\n    for sections in data:\n        json_text = json_text + ' ' + sections.get('text')\n    \n    return json_text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    text = ''.join([k if k not in string.punctuation else ' ' for k in text])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    \n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntrain_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_text = []\n\n# 学習データを取り込み\nfor Id in train_df[\"Id\"]:\n    training_text.append(create_text_from_json(\"train\", Id))\n\ntrain_df['text'] = training_text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'] = train_df['text'].apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_text = []\n# 学習データを取り込み\nfor Id in sample_submission_df[\"Id\"]:\n    submit_text.append(create_text_from_json(\"test\", Id))\n\nsample_submission_df['text'] = submit_text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\nexisting_labels = set(temp_1 + temp_2 + temp_3)\n\nid_list = []\nlables_list = []\nfor index, row in sample_submission_df.iterrows():\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 提出用データ作成\nmy_submission = pd.DataFrame()\nmy_submission['Id'] = id_list\nmy_submission['PredictionString1'] = lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del training_text\ndel submit_text\ndel id_list\ndel lables_list\ndel sample_text\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## simple transformers","metadata":{}},{"cell_type":"code","source":"!pip install '../input/simpletransformers0272/seqeval-0.0.12-py3-none-any.whl' -q\n!pip install '../input/tokenizers-070/tokenizers-0.7.0-cp37-cp37m-manylinux1_x86_64.whl' -q\n!pip install '../input/simpletransformers-0323-pypi/transformers-2.11.0-py3-none-any.whl' -q\n!pip install '../input/simpletransformers-0323-pypi/simpletransformers-0.32.3-py3-none-any.whl' -q","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from simpletransformers.classification import MultiLabelClassificationModel\nimport logging","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ログの設定\nlogging.basicConfig(level=logging.INFO)\ntransformers_logger = logging.getLogger(\"transformers\")\ntransformers_logger.setLevel(logging.WARNING)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"work_df = pd.get_dummies(train_df['cleaned_label']) \nlabel_list = list(work_df.columns)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del work_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distinct_train_df = pd.DataFrame()\ndistinct_train_df['Id'] = train_df['Id']\ndistinct_train_df['text'] = train_df['text']\ndistinct_train_df = distinct_train_df.drop_duplicates(subset=[\"Id\"])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"work_label_df = pd.DataFrame()\nfor label in label_list:\n    match_list = []\n    for index, row in distinct_train_df.iterrows():\n        match_list.append(1 if label in row['text'] else 0)\n    work_label_df[label] = match_list\n\ndistinct_train_df['label'] = work_label_df.values.tolist()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del match_list\ndel work_label_df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gensim.parsing.preprocessing import remove_stopwords","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopword_list = list(set([stopword for stopword in clean_text(' '.join(existing_labels)).split()]))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_preprocessing(json_text):\n    json_text = remove_stopwords(json_text)\n    \n    for label in label_list:\n        json_text.replace(label,'')\n\n    for stopword in stopword_list:\n        json_text.replace(stopword,'')\n\n    return json_text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distinct_train_df['text'] = distinct_train_df['text'].apply(text_preprocessing)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distinct_train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df['text'] = sample_submission_df['text'].apply(text_cleaning)\nsample_submission_df['text'] = sample_submission_df['text'].apply(text_preprocessing)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"distinct_train_df = distinct_train_df.sample(frac=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# モデルの作成\nmodel = MultiLabelClassificationModel('albert', '../input/pretrained-albert-pytorch/albert-base-v2', \n                                      num_labels=len(label_list),\n                                      use_cuda=False, \n                                      args={'reprocess_input_data': False, \n                                            'overwrite_output_dir': True, \n                                            'train_batch_size': 8, \n                                            'num_train_epochs': 1})\n\nmodel.train_model(distinct_train_df.drop('Id',axis=1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Prediction_list = []\nfor index, row in sample_submission_df.iterrows():\n    predictions, raw_outputs = model.predict([row['text']])\n    Prediction_list.append('|'.join(np.array(label_list)[predictions==1].tolist()))\n\nmy_submission['PredictionString2'] = Prediction_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#2つのモデルの結果をマージ\nmy_submission['PredictionString'] = np.where(my_submission['PredictionString1'] == '', \n                                             my_submission['PredictionString2'], \n                                             my_submission['PredictionString1'])\nmy_submission = my_submission.drop('PredictionString1', axis=1)\nmy_submission = my_submission.drop('PredictionString2', axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}