{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport re\nimport json\nimport glob\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n%matplotlib inline\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are provided with 4 main pieces of data:\n\ntrain.csv: The CSV file containing all the metadata of the publications, such as their title and the dataset they utilize.\ntrain: The directory containing the actual publications that are referenced in train.csv in JSON format.\ntest: The directory containing the actual publications that will be used for testing purposes (thus, with no ground truth CSV file available).\nsample_submission.csv: The CSV file containing all the publications IDs in the test set, for which we'll have to populate the prediction column."},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in train.columns:\n    print(f\"{col}: {len(train[col].unique())}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Printing below the dataset titles that have multiple dataset labels associated with them:\\n\")\ndatasets_titles_unique = train[\"dataset_title\"].unique()\nfor dataset_title in datasets_titles_unique:\n    if len(train[train[\"dataset_title\"] == dataset_title][\"dataset_label\"].unique()) > 1:\n        print(f\"'{dataset_title}':\", list(train[train[\"dataset_title\"] == dataset_title][\"dataset_label\"].unique()), \"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_titles_counts = train['dataset_title'].value_counts()\n\nfig = go.Figure(data=[go.Table(\n  columnwidth = [0.25, 2, 0.5],\n  header=dict(\n    values=[\"<b>Rank</b>\", \"<b>Dataset Title</b>\", \"<b>Mentions</b>\"],\n    line_color='darkslategray',\n    fill_color=\"royalblue\",\n    align='center',\n    font=dict(color='white', size=12)\n  ),\n  cells=dict(\n    values=np.array([np.array((str(i+1), \"<i>\" + x + \"</i>\", \"<b>\" + str(y) + \"</b>\", )) for i, (x, y) in enumerate(zip(dataset_titles_counts.index, dataset_titles_counts.values))]).T,\n    line_color='darkslategray',\n    # 2-D list of colors for alternating rows\n    fill_color = [[\"white\",\"lavender\"]*25],\n    align = 'center',\n    font = dict(color = 'darkslategray', size = 11)\n    ))\n])\n\nfig.update_layout(\n    title={\"text\": \"<b>Datasets Titles Mentions Counts</b>\",\n           \"x\": 0.5,\n           \"xanchor\":\"center\",\n           \"font_size\": 22},\n    margin={\"r\":20, \"l\":20})\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multi_mentions = [] # list of tuples (Id, num_unique_mentions) \n\nfor Id in train.Id.unique():\n    num_unique_mentions = len(train[train['Id'] == Id]['dataset_title'].unique())\n    if num_unique_mentions > 1:\n        multi_mentions.append((Id, num_unique_mentions))\n\nprint(f\"There are {len(multi_mentions)} publications in the training set that mention more than 1 dataset. That is {len(multi_mentions) / len(train.Id.unique()) * 100:.2f}% of the publications.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"co_mentions = defaultdict(int)\n\nfor Id, num_mentions in multi_mentions:\n    co_mentions[tuple(sorted(train[train['Id'] == Id][\"dataset_title\"].unique()))] += 1\n\nprint(f\"There are {len(co_mentions)} unique sets of co-occurences of dataset mentions.\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"co_mentions = dict(sorted(co_mentions.items(), key=lambda item: item[1], reverse=True))\n\nfig = go.Figure(data=[go.Table(\n  columnwidth = [0.25, 2, 0.5],\n  header=dict(\n    values=[\"<b>Rank</b>\", \"<b>Datasets Co-Mentions Sets</b>\", \"<b>Occurences</b>\"],\n    line_color='darkslategray',\n    fill_color=\"royalblue\",\n    align='center',\n    font=dict(color='white', size=12)\n  ),\n  cells=dict(\n    values=np.array([np.array((str(i+1), \"<i>\" + str(k) + \"</i>\", \"<b>\" + str(v) + \"</b>\", )) for i, (k, v) in enumerate(co_mentions.items())]).T,\n    line_color='darkslategray',\n    fill_color = [[\"white\",\"lavender\"]*60],\n    align = 'center',\n    font = dict(color = 'darkslategray', size = 11)\n    ))\n])\n\nfig.update_layout(\n    title={\"text\": \"<b>Datasets Mentioned in the same Publications</b>\",\n           \"x\": 0.5,\n           \"xanchor\":\"center\",\n           \"font_size\": 22},\n    margin={\"r\":20, \"l\":20})\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n\nwords_in_titles = list(train.pub_title.str.split(expand=True).stack())\n\nwordcloud = WordCloud(stopwords = STOPWORDS,\n                      background_color = \"white\",\n                      width = 3000,\n                      height = 2000\n                     ).generate(' '.join(words_in_titles))\nplt.figure(1, figsize = (18, 12))\nplt.imshow(wordcloud)\nplt.axis('off')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"words_in_titles_by_dataset = defaultdict(list)\n\n# Separating out positive and negative words (i.e., words appearing in negative and positive tweets),\n# in order to visualize each set of words independently\nfor _, row in train.iterrows():\n    words_in_titles_by_dataset[row['dataset_title']].extend(row['pub_title'].split())\n\n# Defining our word cloud drawing function\ndef wordcloud_draw(data, color = 'white'):\n    wordcloud = WordCloud(stopwords = STOPWORDS,\n                          background_color = color,\n                          width = 3000,\n                          height = 2000\n                         ).generate(' '.join(data))\n    plt.figure(1, figsize = (12, 8))\n    plt.imshow(wordcloud)\n    plt.axis('off')\n    plt.show()\n\nfor dataset_title in train['dataset_title'].unique():\n    print(\"Wordcloud for publications mentioning\", dataset_title, \":\")\n    wordcloud_draw(words_in_titles_by_dataset[dataset_title])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Gathering the files paths\ntrain_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/train/*.json\")\ntest_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*.json\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the training publications dataframe\ndf_train_publications = pd.DataFrame()\n\nfor train_file in train_files:\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id', train_file.split('/')[-1].split('.')[0])\n    df_train_publications = pd.concat([df_train_publications, file_data])\n\ndf_train_publications.to_csv(\"df_train_publications.csv\",index=False)\n\ndf_train_publications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Generate the testing publications dataframe\ndf_test_publications = pd.DataFrame()\n\nfor test_file in test_files:\n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0])\n    df_test_publications = pd.concat([df_test_publications, file_data])\n\ndf_test_publications.to_csv(\"df_test_publications.csv\",index=False)\n\ndf_test_publications","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)submission_df = pd.read_csv('../input/cosubmission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)leridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)\ndatasets_titles = [x.lower() for x in train['dataset_title'].unique()]\n\nlabels = []\nfor index in submission_df.index:\n    publication_text = df_test_publications[df_test_publications['pub_id'] == index].text.str.cat(sep='\\n').lower()\n    label = []\n    for dataset_title in datasets_titles:\n        if dataset_title in publication_text:\n            label.append(clean_text(dataset_title))\n    labels.append('|'.join(label))\n\nsubmission_df['PredictionString'] = labels\n\nsubmission_df.to_csv('submission.csv')\n\nsubmission_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}