{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n#jsonモジュールのインポート\nimport json\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport requests\nimport gensim\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nimport re\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.over_sampling import RandomOverSampler","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"snowball = SnowballStemmer(language='english')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_wordlist_from_json(dataInd,fileId):\n    \n    filename = \"/kaggle/input/coleridgeinitiative-show-us-the-data/\" + dataInd + \"/\" + fileId + \".json\"\n    \n    fd = open(filename, mode='r')\n    data = json.load(fd)\n    fd.close()\n    json_text = ''\n    for sections in data:\n        json_text = json_text + ' ' + sections.get('text')\n    \n    json_text = json_text.replace('\\\\n',' ').replace('\\\\f',' ').replace('\\\\u','!!!').replace('\\\\b',' ').replace('\\\\t',' ').replace('\\\\',' ')\n    json_text = re.sub('!{3}[A-Za-z0-9]{4}',' ',json_text)\n    json_text= re.sub('r[^\\w\\s]',' ',json_text)\n    \n    textWordlist = nltk.word_tokenize(json_text)\n\n    #STOPWORDなし\n    #return TaggedDocument(words=textWordlist, tags=[fileId])\n\n    #STOPWORDあり\n    stopWords = stopwords.words('english') + \\\n    ['\"','{', '}', '[', ']', '(',')',  ',', ':', '``', \"''\", ';', '.', \n     '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '%']\n    \n    wordlist = [snowball.stem(word.lower()) for word in textWordlist if word.lower() not in stopWords]\n    return wordlist","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#SWEM-MAXの関数\ndef get_doc_swem_max_vector(words, model):\n    vector_size = len(model[0])\n    doc_vector = np.zeros((len(words), vector_size))\n    for i, word in enumerate(words):\n        try:\n            word_vector = model[word]\n        except KeyError:\n            word_vector = np.zeros(vector_size)\n        \n        doc_vector[i, :] = word_vector\n\n    doc_vector = np.max(doc_vector, axis=0)\n    return doc_vector","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntrain_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ラベルを修正\ntemp_df = train_df.drop('pub_title',axis=1).groupby(by=[\"dataset_title\",\"dataset_label\",\"cleaned_label\"]).count().reset_index()\ntemp_df = temp_df.sort_values(['dataset_title', 'Id'], ascending=[True, False])\nfix_label_df = pd.DataFrame(columns=['dataset_title','cleaned_label_fix'])\ntemp2_df = temp_df.drop_duplicates(subset='dataset_title')\nfor dataset_title in temp2_df['dataset_title']:\n    temp3_df = temp_df.query('dataset_title == \"' + dataset_title + '\"')\n    cleaned_label_fix = temp3_df['cleaned_label'].values[0]\n    if len(temp3_df) == 1:\n        fix_label_df = fix_label_df.append({'dataset_title': dataset_title, 'cleaned_label_fix': cleaned_label_fix}, \n                                           ignore_index=True)    \n    else:\n        fix_words = cleaned_label_fix.split()\n        second_words = temp3_df['cleaned_label'].values[1].split()\n        add_word = ' '.join([word for word in second_words if word not in fix_words])\n        cleaned_label_fix = cleaned_label_fix + ' ' + add_word\n        fix_label_df = fix_label_df.append({'dataset_title': dataset_title, 'cleaned_label_fix': cleaned_label_fix}, \n                                           ignore_index=True)\n\nnew_train_df = pd.merge(train_df, fix_label_df, on='dataset_title')\nnew_train_df = new_train_df.drop('cleaned_label', axis=1).rename(columns={'cleaned_label_fix': 'cleaned_label'})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = gensim.models.KeyedVectors.load_word2vec_format('../input/fasttext-pretrainedvectors-english-text/cc.en.300.vec', binary=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 空のリストを作成（学習データとなる各文書を格納）\ntraining_docs = []\n\ndistinct_train_df = new_train_df.drop_duplicates(subset=[\"Id\"])\ndistinct_train_df = distinct_train_df.sort_values('Id')\n\n# 学習データを取り込み\nfor Id in distinct_train_df[\"Id\"]:\n    training_docs.append(create_wordlist_from_json(\"train\", Id))\n\n# テストデータを取り込み\nfor Id in sample_submission_df[\"Id\"]:\n    training_docs.append(create_wordlist_from_json(\"test\", Id))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = np.zeros((len(distinct_train_df), len(model[0])))\nX_submit = np.zeros((len(training_docs) - len(distinct_train_df), len(model[0])))\n\nfor i, doc in enumerate(training_docs):\n    if i < len(distinct_train_df):\n        X[i, :] = get_doc_swem_max_vector(doc, model)\n    else :\n        X_submit[i - len(distinct_train_df), :] = get_doc_swem_max_vector(doc, model)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#ラベルを作成\nlabel_df = pd.DataFrame(new_train_df['Id'])\nwork_df = pd.get_dummies(new_train_df['cleaned_label']) \nlabel_list = list(work_df.columns)\nlabel_df = pd.concat([label_df, work_df], axis=1)\nlabel_df = label_df.groupby(by=['Id']).max()\nlabel_df = label_df.sort_values('Id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 提出用データ作成\nmy_submission = pd.DataFrame(sample_submission_df['Id'])\nmy_submission['PredictionString'] = ''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for label in label_list:\n    print(label)\n    temp_label_df = pd.DataFrame()\n    temp_label_df[label] = label_df[label]\n    \n    #オーバーサンプリング\n    positive_count_train = temp_label_df.sum()\n    ros = RandomOverSampler(sampling_strategy=0.5, random_state=71)\n    X_res, y_res = ros.fit_resample(pd.DataFrame(X), temp_label_df.reset_index().drop('Id', axis=1))\n    \n    \n    #訓練データと検証データに分割\n    train_X, val_X, train_y, val_y = train_test_split(X_res, y_res, test_size = 0.3, random_state=71)\n\n    # データセットを生成する\n    lgb_train = lgb.Dataset(train_X.values, train_y[label].values)\n    lgb_eval = lgb.Dataset(val_X.values, val_y[label].values, reference=lgb_train)\n\n    # LightGBM のハイパーパラメータ\n    params = {\n        # 二値分類問題\n        'objective': 'binary',\n        # AUC の最大化を目指す\n        'metric': 'auc',\n        # Fatal の場合出力\n        'verbosity': -1,\n    }\n\n    # 上記のパラメータでモデルを学習する\n    lgbModel = lgb.train(params, lgb_train, valid_sets=lgb_eval,\n                      verbose_eval=50,  # 50イテレーション毎に学習結果出力\n                      num_boost_round=1000,  # 最大イテレーション回数指定\n                      early_stopping_rounds=100\n                     )\n\n    # テストデータを予測する\n    y_pred = lgbModel.predict(val_X.values, num_iteration=lgbModel.best_iteration)\n\n    fpr, tpr, thresholds = metrics.roc_curve(val_y[label].values, y_pred)\n    auc = metrics.auc(fpr, tpr)\n    print(auc)\n\n    predicted = lgbModel.predict(X_submit, num_iteration=lgbModel.best_iteration)\n    predicted = np.round(predicted)\n    predicted_list = ['|' + label if i > 0 else '' for i in predicted]    \n    my_submission['tempString'] = predicted_list\n    my_submission['PredictionString'] = my_submission['PredictionString'] + my_submission['tempString']\n    my_submission = my_submission.drop('tempString', axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# you could use any filename. We choose submission here\nmy_submission['PredictionString'] = my_submission['PredictionString'].str[1:]\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}