{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport re\nimport json\nimport glob\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in train.columns:\n    print(f\"{col}: {len(train[col].unique())}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['dataset_title'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Wordcloud of the Articles Titles"},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n\nwords_in_titles = list(train.pub_title.str.split(expand=True).stack())\n\nwordcloud = WordCloud(stopwords = STOPWORDS,\n                      background_color = \"white\",\n                      width = 3000,\n                      height = 2000\n                     ).generate(' '.join(words_in_titles))\nplt.figure(1, figsize = (18, 12))\nplt.imshow(wordcloud)\nplt.axis('off')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Wordclouds of Article Titles by Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\n\nwords_in_titles_by_dataset = defaultdict(list)\n\n# Separating out positive and negative words (i.e., words appearing in negative and positive tweets),\n# in order to visualize each set of words independently\nfor _, row in train.iterrows():\n    words_in_titles_by_dataset[row['dataset_title']].extend(row['pub_title'].split())\n\n# Defining our word cloud drawing function\ndef wordcloud_draw(data, color = 'white'):\n    wordcloud = WordCloud(stopwords = STOPWORDS,\n                          background_color = color,\n                          width = 3000,\n                          height = 2000\n                         ).generate(' '.join(data))\n    plt.figure(1, figsize = (12, 8))\n    plt.imshow(wordcloud)\n    plt.axis('off')\n    plt.show()\n\nfor dataset_title in train['dataset_title'].unique():\n    print(\"Wordcloud for\", dataset_title, \":\")\n    wordcloud_draw(words_in_titles_by_dataset[dataset_title])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading JSON Contents into a Pandas DataFrame"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Gathering the files paths\ntrain_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/train/*.json\")\ntest_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*.json\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the training publications dataframe\ndf_train_publications = pd.DataFrame()\n\nfor train_file in train_files:\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id', train_file.split('/')[-1].split('.')[0])\n    df_train_publications = pd.concat([df_train_publications, file_data])\n\ndf_train_publications.to_csv(\"df_train_publications.csv\",index=False)\n\ndf_train_publications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the testing publications dataframe\ndf_test_publications = pd.DataFrame()\n\nfor test_file in test_files:\n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0])\n    df_test_publications = pd.concat([df_test_publications, file_data])\n\ndf_test_publications.to_csv(\"df_test_publications.csv\",index=False)\n\ndf_test_publications","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Naïve Dataset Title Matching Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)\ndatasets_titles = [x.lower() for x in train['dataset_title'].unique()]\n\nlabels = []\nfor index in submission_df.index:\n    publication_text = df_test_publications[df_test_publications['pub_id'] == index].text.str.cat(sep='\\n').lower()\n    label = []\n    for dataset_title in datasets_titles:\n        if dataset_title in publication_text:\n            label.append(clean_text(dataset_title))\n    labels.append('|'.join(label))\n\nsubmission_df['PredictionString'] = labels\n\nsubmission_df.to_csv('submission.csv')\n\nsubmission_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}