{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n#jsonモジュールのインポート\nimport json\nimport matplotlib.pyplot as plt\nimport requests\nfrom gensim.models.doc2vec import Doc2Vec\nfrom gensim.models.doc2vec import TaggedDocument\nfrom gensim.parsing.preprocessing import remove_stopwords\nimport nltk\nfrom nltk.stem import SnowballStemmer\nimport string\nimport re","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"snowball = SnowballStemmer(language='english')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_taggedDocument_from_json(dataInd,fileId):\n    \n    filename = \"/kaggle/input/coleridgeinitiative-show-us-the-data/\" + dataInd + \"/\" + fileId + \".json\"\n    \n    fd = open(filename, mode='r')\n    data = json.load(fd)\n    fd.close()\n    json_text = ''\n    for sections in data:\n        json_text = json_text + ' ' + sections.get('text')\n    \n    json_text = ''.join([k for k in json_text if k not in string.punctuation])\n    json_text = re.sub('[^A-Za-z0-9]+', ' ', str(json_text).lower()).strip()\n    json_text = json_text.lower()\n    json_text = remove_stopwords(json_text)\n    \n    textWordlist = nltk.word_tokenize(json_text)\n\n    #STOPWORDあり\n    wordlist = [snowball.stem(word) for word in textWordlist]\n    return TaggedDocument(words=wordlist, tags=[fileId])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntrain_df = pd.read_csv(\"/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 空のリストを作成（学習データとなる各文書を格納）\ntraining_docs = []\n\ndistinct_train_df = train_df.drop_duplicates(subset=[\"Id\"])\n\n# 学習データを取り込み\nfor Id in distinct_train_df[\"Id\"]:\n    training_docs.append(create_taggedDocument_from_json(\"train\", Id))\n\n# テストデータを取り込み\nfor Id in sample_submission_df[\"Id\"]:\n    training_docs.append(create_taggedDocument_from_json(\"test\", Id))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def countWords(text):\n    return len(text.split())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 学習実行（パラメータを調整可能）\n# documents:学習データ（TaggedDocumentのリスト）\n# min_count=1:最低1回出現した単語を学習に使用する\n# dm=0:学習モデル=DBOW（デフォルトはdm=1:学習モデル=DM）\nmodel = Doc2Vec(documents=training_docs, \n                vector_size=250, \n                epochs=50, \n                alpha=0.0025, \n                min_alpha=0.000001, \n                sample=0.001, \n                min_count=5, \n                window=train_df['cleaned_label'].apply(countWords).max(), \n                negative=5,\n                ns_exponent=0.75, \n                dbow_words=0, \n                dm=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df = pd.DataFrame([], columns = ['test_Id', 'cleaned_label'])\n# テストデータの類似文書の情報を格納\nfor Id in sample_submission_df[\"Id\"]:\n    result_df = pd.DataFrame([], columns = ['test_Id' , 'train_Id', 'train_cos', 'cleaned_label'])\n    doccnt = 0\n    loc = 0\n    for doc in model.dv.most_similar(Id, topn=1):\n        queryString = 'Id == \"' + doc[0] + '\"'\n        temp_df = train_df.query(queryString)\n        if len(temp_df) > 0:\n            doccnt = doccnt + 1\n            for index, row in temp_df.iterrows():\n                result_df.loc[loc] = [Id, doc[0], doc[1], row['cleaned_label']]\n                loc = loc + 1\n    \n    result_df = result_df.groupby(['test_Id', 'cleaned_label'], as_index=False).count()\n    result_df = result_df.sort_values(['test_Id', 'train_Id'], ascending=[True, False])\n    result_df = result_df.groupby('test_Id')['cleaned_label'].apply('|'.join).reset_index()\n    predict_df = predict_df.append(result_df)\n\nsubmit_df = pd.merge(sample_submission_df, predict_df, how='left', left_on='Id', right_on='test_Id')\nsubmit_df = submit_df.fillna(\"\")\nsubmit_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission = pd.DataFrame({'Id': submit_df.Id, 'PredictionString': submit_df.cleaned_label})\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}