{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfrom tqdm import tqdm\nimport json\n\nimport re\nfrom gensim.parsing.preprocessing import remove_stopwords\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom nltk.stem.porter import PorterStemmer\n\nfrom collections import Counter\nimport matplotlib.pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = \"../input/coleridgeinitiative-show-us-the-data/test\"\nall_test_path = [os.path.join(test_path,s) + \".json\" for s in sample_df[\"Id\"]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltexts = []\n\nfor json_path in tqdm(all_test_path):\n\n    with open(json_path, 'r') as f:\n            json_decode = json.load(f)\n    jsontest = pd.DataFrame(json_decode)\n\n    texts = \"\"\n\n    for a in jsontest.values:\n        texts += a[0] + \" \" + a[1] + \" \"\n        \n    alltexts.append(texts)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df[\"text\"] = alltexts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = \"../input/coleridgeinitiative-show-us-the-data/train\"\nall_train_path = [os.path.join(train_path,s) + \".json\" for s in train_df[\"Id\"]]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alltexts = []\n\nfor json_path in tqdm(all_train_path):\n\n    with open(json_path, 'r') as f:\n            json_decode = json.load(f)\n    jsontest = pd.DataFrame(json_decode)\n\n    texts = \"\"\n\n    for a in jsontest.values:\n        texts += a[0] + \" \" + a[1] + \" \"\n        \n    alltexts.append(texts)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"text\"] = alltexts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Figure Train Text","metadata":{}},{"cell_type":"code","source":"cleaned = list(train_df['cleaned_label'])\n\ncleans = []\n\nfor i in cleaned:\n    i = list(i.split(\" \"))\n    for j in i:\n        cleans.append(j.replace(\" \", \"\"))\n\ncln = Counter(cleans)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clns = list(set(cln))\nplt.rcParams['figure.figsize'] = (15,15)\nwordcloud = WordCloud(max_font_size = 50, max_words = 100, background_color = 'white').generate(str(clns))\n\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.title('Cleaned Label Distribution', size = 23, color = 'darkblue')\nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\n\ncleaned = list(train_df['dataset_label'])\n\ncleans = []\n\nfor i in cleaned:\n    i = list(i.split(\" \"))\n    for j in i:\n        cleans.append(j.replace(\" \", \"\"))\n\ncln = Counter(cleans)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n\nclns = list(set(cln))\nplt.rcParams['figure.figsize'] = (15,15)\nwordcloud = WordCloud(max_font_size = 50, max_words = 100, background_color = 'white').generate(str(clns))\n\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.title('Dataset Label Distribution', size = 23, color = 'darkblue')\nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\n\ncleaned = list(train_df['dataset_title'])\n\ncleans = []\n\nfor i in cleaned:\n    i = list(i.split(\" \"))\n    for j in i:\n        cleans.append(j.replace(\" \", \"\"))\n\ncln = Counter(cleans)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n\nclns = list(set(cln))\nplt.rcParams['figure.figsize'] = (15,15)\nwordcloud = WordCloud(max_font_size = 50, max_words = 100, background_color = 'white').generate(str(clns))\n\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.title('Dataset Title Distribution', size = 23, color = 'darkblue')\nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from collections import Counter\n\ncleaned = list(train_df['pub_title'])\n\ncleans = []\n\nfor i in cleaned:\n    i = list(i.split(\" \"))\n    for j in i:\n        cleans.append(j.replace(\" \", \"\"))\n\ncln = Counter(cleans)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n\nclns = list(set(cln))\nplt.rcParams['figure.figsize'] = (15,15)\nwordcloud = WordCloud(max_font_size = 50, max_words = 100, background_color = 'white').generate(str(clns))\n\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis('off')\nplt.title('Pub Title Distribution', size = 23, color = 'darkblue')\nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Combine Datasets","metadata":{}},{"cell_type":"code","source":"porter = PorterStemmer()\ndef lower_case(input_str):\n    input_str = input_str.lower()\n    return input_str\ndef stemSentence(sentence):\n    token_words=word_tokenize(sentence)\n    token_words\n    stem_sentence=[]\n    for word in token_words:\n        stem_sentence.append(porter.stem(word))\n        stem_sentence.append(\" \")\n    return \"\".join(stem_sentence)\n\ndef clean_text(text):\n    text= re.sub('r[^\\w\\s]',' ',text)\n    #text= re.sub('[^a-zA-z0-9\\s]','',text)\n    text=lower_case(text)\n    text=remove_stopwords(text)\n    text=stemSentence(text)    \n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\ntemp_4 = [x.lower() for x in train_df['pub_title'].unique()]\ntemp_5 = [x.lower() for x in train_df['text'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3 + temp_4  + temp_5)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_list = []\nlables_list = []\nfor index, row in tqdm(sample_df.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n\n    temp_df = train_df[train_df['text'] == clean_text(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append(' | '.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Making submission","metadata":{}},{"cell_type":"code","source":"sample_df[\"PredictionString\"] = lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del sample_df['text']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df.to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}