{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n#json\nimport json\nimport matplotlib.pyplot as plt\nimport requests\nfrom gensim.models.doc2vec import Doc2Vec\nfrom gensim.models.doc2vec import TaggedDocument\nimport nltk\nfrom nltk.corpus import stopwords\nfrom tqdm import tqdm\nfrom collections import Counter\nimport re","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_taggedDocument_from_json(dataInd,fileId):\n    \n    filename = \"../input/coleridgeinitiative-show-us-the-data/\" + dataInd + \"/\" + fileId + \".json\"\n    \n    fd = open(filename, mode='r')\n    data = json.load(fd)\n    fd.close()\n    json_text = json.dumps(data).replace('\\\\n',' ').replace('\\\\f',' ').replace('\\\\u','!!!').replace('\\\\b',' ').replace('\\\\t',' ').replace('\\\\',' ')\n    json_text = re.sub('!{3}[A-Za-z0-9]{4}',' ',json_text)\n    textWordlist = nltk.word_tokenize(json_text)\n\n    #STOPWORD\n    #return TaggedDocument(words=textWordlist, tags=[fileId])\n\n    #STOPWORD\n    stopWords = stopwords.words('english') + \\\n   ['\"','}', 'then', 'result', 'been', '%', 'to', 'due', 'important',\n    'one', 'However', 'other', 'first', 'be', '``', 'for', 'with', ']', 'This', 'not', 'These', '{', \n    'because', 'The', 'at', 'did', 'that', 'research', 'high', 'of', 'than', 'related', 'into', 'when', \n     ':', '5', 'were', 'may', 'those', ')', 'As', 'an', 'data', 'all', 'over', 'these', 'the', 'this', \n     'level', 'will', 'based', 'less', 'can', 'text', 'a', 'some', 'different', 'large', 'included',\n     'available', 'who', '(', ',', 'within', 'two', \"'s\", 'differences', 'and', 'similar', 'have', '1', \n     'it', 'through', 'each', 'three', 'analysis', 'as', 'total', 'significant', 'about', 'while', 'is', \n     'on', 'but', 'if', '[', 'In', 'reported', 'Table', 'in', 'among', 'likely', 'which', 'studies', \n     'section_title', 'by', 'both', 'between', \"''\", 'across', 'using', 'more', 'information', 'A', \n     'also', 'addition', 'study', ';', '.', 'or', 'any', 'are', 'has', 'had', 'For', 'results', 'used', \n     'from', 'was', 'time', 'associated', '20', 'could', 'after', 'number', 'found', 'such', '9', '3', \n     'lower', '4', 'would', 'many', '8', 'further', '7', '6', 'years', 'higher', 'shown', 'et', '2', \n     'several', 'compared', 'they', 'their', 'use', 'To', 'most', 'during', 'same', 'including', \n     'model', 'following', 'no', 'small', 'do', 'only', 'we', 'should', 'well', '12', 'include', \n     'possible', '10', 'We', 'where', 'there', 'increase', 'example', 'It', 'provide', 'our', \n     'potential', 'its']\n    \n    wordlist = [word for word in textWordlist if word.lower() not in stopWords]\n    return TaggedDocument(words=wordlist, tags=[fileId])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\ntrain_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"training_docs = []\n\ndistinct_train_df = train_df.drop_duplicates(subset=[\"Id\"])\n\nfor Id in distinct_train_df[\"Id\"]:\n    training_docs.append(create_taggedDocument_from_json(\"train\", Id))\n\nfor Id in sample_submission_df[\"Id\"]:\n    training_docs.append(create_taggedDocument_from_json(\"test\", Id))\n\nmodel = Doc2Vec(documents=training_docs, min_count=1, dm=0)\n \n    # print(model.docvecs['d1'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predict_df = pd.DataFrame([], columns = ['test_Id', 'cleaned_label'])\n\nfor Id in sample_submission_df[\"Id\"]:\n    result_df = pd.DataFrame([], columns = ['test_Id' , 'train_Id', 'train_cos', 'cleaned_label'])\n    doccnt = 0\n    loc = 0\n    for doc in model.docvecs.most_similar(Id, topn=1):\n        queryString = 'Id == \"' + doc[0] + '\"'\n        temp_df = train_df.query(queryString)\n        if len(temp_df) > 0:\n            doccnt = doccnt + 1\n            for index, row in temp_df.iterrows():\n                result_df.loc[loc] = [Id, doc[0], doc[1], row['cleaned_label']]\n                loc = loc + 1\n    \n    result_df = result_df.groupby(['test_Id', 'cleaned_label'], as_index=False).count()\n    result_df = result_df.sort_values(['test_Id', 'train_Id'], ascending=[True, False])\n    result_df = result_df.groupby('test_Id')['cleaned_label'].apply('|'.join).reset_index()\n    predict_df = predict_df.append(result_df)\n\nsubmit_df = pd.merge(sample_submission_df, predict_df, how='left', left_on='Id', right_on='test_Id')\nsubmit_df = submit_df.fillna(\"\")\nsubmit_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:7: DeprecationWarning: Call to deprecated `docvecs` (The `docvecs` property has been renamed `dv`.).\nimport sys","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"my_submission = pd.DataFrame({'Id': submit_df.Id, 'PredictionString': submit_df.cleaned_label})\n# you could use any filename. We choose submission here\nmy_submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}