{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Doc2Vec & LightGBM","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"### Import module","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n#jsonモジュールのインポート\nimport json\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport requests\nfrom gensim.models.doc2vec import Doc2Vec\nfrom gensim.models.doc2vec import TaggedDocument\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom tqdm import tqdm\nfrom collections import Counter\nimport re\n\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.over_sampling import RandomOverSampler\n\nimport random\nfrom sklearn.datasets import make_classification\nfrom sklearn.neighbors import NearestNeighbors","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### create TaggedDocument from Json File","metadata":{}},{"cell_type":"code","source":"def create_taggedDocument_from_json(dataInd,fileId):\n    \n    filename = \"/kaggle/input/coleridgeinitiative-show-us-the-data/\" + dataInd + \"/\" + fileId + \".json\"\n    \n    fd = open(filename, mode='r')\n    data = json.load(fd)\n    fd.close()\n    json_text = ''\n    for sections in data:\n        json_text = json_text + ' ' + sections.get('text')\n    \n    json_text = json_text.replace('\\\\n',' ').replace('\\\\f',' ').replace('\\\\u','!!!').replace('\\\\b',' ').replace('\\\\t',' ').replace('\\\\',' ')\n    json_text = re.sub('!{3}[A-Za-z0-9]{4}',' ',json_text)\n    json_text= re.sub('r[^\\w\\s]',' ',json_text)\n    \n    textWordlist = nltk.word_tokenize(json_text)\n\n    #STOPWORDなし\n    #return TaggedDocument(words=textWordlist, tags=[fileId])\n\n    #STOPWORDあり\n    stopWords = stopwords.words('english') + \\\n    ['\"','{', '}', '[', ']', '(',')',  ',', ':', '``', \"''\", ';', '.']\n    \n    wordlist = [snowball.stem(word.lower()) for word in textWordlist if word.lower() not in stopWords]\n    return TaggedDocument(words=wordlist, tags=[fileId])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### read CSV File","metadata":{}},{"cell_type":"code","source":"sample_submission_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntrain_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train Doc2Vec Model","metadata":{}},{"cell_type":"code","source":"snowball = SnowballStemmer(language='english')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 空のリストを作成（学習データとなる各文書を格納）\ntraining_docs = []\n\ndistinct_train_df = train_df.drop_duplicates(subset=[\"Id\"])\n\n# 学習データを取り込み\nfor Id in distinct_train_df[\"Id\"]:\n    training_docs.append(create_taggedDocument_from_json(\"train\", Id))\n\n# テストデータを取り込み\nfor Id in sample_submission_df[\"Id\"]:\n    training_docs.append(create_taggedDocument_from_json(\"test\", Id))\n\n# 学習実行（パラメータを調整可能）\n# documents:学習データ（TaggedDocumentのリスト）\n# min_count=1:最低1回出現した単語を学習に使用する\n# dm=0:学習モデル=DBOW（デフォルトはdm=1:学習モデル=DM）\nmodel = Doc2Vec(documents=training_docs, \n                vector_size=200, \n                alpha=0.0025, \n                min_alpha=0.000001, \n                window=15, \n                min_count=1, \n                dm=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Extract document vector","metadata":{}},{"cell_type":"code","source":"#Doc2Vecからベクトルを特徴量として抽出\ntrain_docvecs_df = pd.DataFrame()\nsubmit_docvecs_df = pd.DataFrame()\n\n\nfor Id in distinct_train_df[\"Id\"]:\n    train_docvecs_df[Id] = model.docvecs[Id]\nfor Id in sample_submission_df[\"Id\"]:\n    submit_docvecs_df[Id] = model.docvecs[Id]\n\ntrain_docvecs_df = train_docvecs_df.T\ntrain_docvecs_df = train_docvecs_df.rename_axis('Id').reset_index()\ntrain_docvecs_df = train_docvecs_df.sort_values('Id')\ntrain_docvecs_df = train_docvecs_df.drop(\"Id\", axis=1)\n\nsubmit_docvecs_df = submit_docvecs_df.T\nsubmit_docvecs_df = submit_docvecs_df.rename_axis('Id').reset_index()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create label","metadata":{}},{"cell_type":"code","source":"#ラベルを作成\nlabel_df = pd.DataFrame(train_df['Id'])\nwork_df = pd.get_dummies(train_df['cleaned_label']) \nlabel_list = list(work_df.columns)\nlabel_df = pd.concat([label_df, work_df], axis=1)\nlabel_df = label_df.groupby(by=['Id']).sum()\nlabel_df = label_df.sort_values('Id')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Initialize the data for submission","metadata":{}},{"cell_type":"code","source":"# 提出用データ作成\nmy_submission = pd.DataFrame(submit_docvecs_df['Id'])\nmy_submission['PredictionString'] = ''","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train LGB Model & Predict","metadata":{}},{"cell_type":"code","source":"for label in label_list:\n    print(label)\n    temp_label_df = pd.DataFrame()\n    temp_label_df[label] = label_df[label]\n    \n    #オーバーサンプリング\n    positive_count_train = temp_label_df.sum()\n    ros = RandomOverSampler(random_state=71)\n    X_res, y_res = ros.fit_resample(train_docvecs_df.reset_index().drop('index', axis=1), temp_label_df.reset_index().drop('Id', axis=1))\n    \n    \n    #訓練データと検証データに分割\n    train_X, val_X, train_y, val_y = train_test_split(X_res, y_res, test_size = 0.3, random_state=71)\n\n    # データセットを生成する\n    lgb_train = lgb.Dataset(train_X.values, train_y[label].values)\n    lgb_eval = lgb.Dataset(val_X.values, val_y[label].values, reference=lgb_train)\n\n    # LightGBM のハイパーパラメータ\n    params = {\n        # 二値分類問題\n        'objective': 'binary',\n        # AUC の最大化を目指す\n        'metric': 'auc',\n        # Fatal の場合出力\n        'verbosity': -1,\n    }\n\n    # 上記のパラメータでモデルを学習する\n    model = lgb.train(params, lgb_train, valid_sets=lgb_eval,\n                      verbose_eval=50,  # 50イテレーション毎に学習結果出力\n                      num_boost_round=1000,  # 最大イテレーション回数指定\n                      early_stopping_rounds=100\n                     )\n\n    # テストデータを予測する\n    y_pred = model.predict(val_X.values, num_iteration=model.best_iteration)\n\n    fpr, tpr, thresholds = metrics.roc_curve(val_y[label].values, y_pred)\n    auc = metrics.auc(fpr, tpr)\n    print(auc)\n\n    temp_df = submit_docvecs_df.drop('Id', axis=1)\n    predicted = model.predict(temp_df.values, num_iteration=model.best_iteration)\n    predicted = np.round(predicted)\n    predicted_list = ['|' + label if i > 0 else '' for i in predicted]    \n    my_submission['tempString'] = predicted_list\n    my_submission['PredictionString'] = my_submission['PredictionString'] + my_submission['tempString']\n    my_submission = my_submission.drop('tempString', axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Submit","metadata":{}},{"cell_type":"code","source":"# you could use any filename. We choose submission here\nmy_submission['PredictionString'] = my_submission['PredictionString'].str[1:]\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}