{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Asthetics\nimport warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nimport pandas as pd\npd.set_option('display.max_columns', None)\nimport numpy as np\nimport json\nimport os\nimport random\nfrom tqdm.autonotebook import tqdm\nimport string\nimport re\nimport glob\nfrom functools import partial\nfrom collections import defaultdict\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm.autonotebook import tqdm\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\n\n%matplotlib inline\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'\ntrain_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/train/*.json\")\ntest_files = glob.glob(\"../input/coleridgeinitiative-show-us-the-data/test/*.json\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def basic_eda(df, row_limit=5, list_elements_limit=10):\n    ### rows and columns\n    print('Info : There are {} columns in the dataset'.format(df.shape[1]))\n    print('Info : There are {} rows in the dataset'.format(df.shape[0]))\n    \n    print(\"==================================================\")\n    \n    ## data types\n    print(\"\\nData type information of different columns\")\n    dtypes_df = pd.DataFrame(df.dtypes).reset_index().rename(columns={0:'dtype', 'index':'column_name'})\n    cat_df = dtypes_df[dtypes_df['dtype']=='object']\n    num_df = dtypes_df[dtypes_df['dtype']!='object']\n    print('Info : There are {} categorical columns'.format(len(cat_df)))\n    print('Info : There are {} numerical columns'.format(len(dtypes_df)-len(cat_df)))\n    \n    if list_elements_limit >= len(cat_df):\n        print(\"Categorical columns : \", list(cat_df['column_name']))\n    else:\n        print(\"Categorical columns : \", list(cat_df['column_name'])[:list_elements_limit])\n        \n    if list_elements_limit >= len(num_df):\n        print(\"Numerical columns : \", list(num_df['column_name']))\n    else:\n        print(\"Numerical columns : \", list(num_df['column_name'])[:list_elements_limit])\n    \n    #dtypes_df['dtype'].value_counts().plot.bar()\n    display(dtypes_df.head(row_limit))\n    \n#     print(\"==================================================\")\n#     print(\"\\nDescription of numerical variables\")\n    \n#     #### Describibg numerical columns\n#     desc_df_num = df[list(num_df['column_name'])].describe().T.reset_index().rename(columns={'index':'column_name'})\n#     display(desc_df_num.head(row_limit))\n    \n    print(\"==================================================\")\n    print(\"\\nDescription of categorical variables\")\n    \n    desc_df_cat = df[list(cat_df['column_name'])].describe().T.reset_index().rename(columns={'index':'column_name'})\n    display(desc_df_cat.head(row_limit))\n    \n    return","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It looks like there are only 14,316 unique IDs in the dataset, meaning that some publications include a multitude of datasets.   \nnotice that the pub_title unique count is slightly smaller than the Id unique counts.   \nThis points to the precense of several occurences of having 2 separate publications, eahc with a unique ID, but sharing the exact same title.\n\nAlso, there are a total of 45 unique dataset_title and 130 unique dataset_label. Meaning that a single dataset could have multible labels throughout different publications.","metadata":{}},{"cell_type":"markdown","source":"## Multiple public titles with same Ids","metadata":{}},{"cell_type":"code","source":"print(\"Printing below the public titles that have multiple Ids associated with them:\\n\")\npublic_titles_unique = train[\"pub_title\"].unique()\nfor pub_title in public_titles_unique:\n    if len(train[train[\"pub_title\"] == pub_title][\"Id\"].unique()) > 1:\n        print(f\"'{pub_title}':\", list(train[train[\"pub_title\"] == pub_title][\"Id\"].unique()), \"\\n\")","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"basic_eda(train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1) There are duplicate id's meaning that there are some pulications that are using mutiple datasets. That's why that id is repeating  \n2) Same is the case with pub_title. A single publication is using mutiple datasets.  \n3) There is NO one to one mapping of id and pub_title. Meaning that there are cases when two different publications (from two different authors) have same title. Well, interesting.      \n4) There 45 dataset titles but 130 dataet labels. Meaning that there are some datasets that has multiple labels. We'll look into how these two are related.","metadata":{}},{"cell_type":"markdown","source":"## 1.1. Dataset Title VS Dataset Label \n","metadata":{}},{"cell_type":"code","source":"print(\"Printing below the dataset titles that have multiple dataset labels associated with them:\\n\")\ndatasets_titles_unique = train[\"dataset_title\"].unique()\nfor dataset_title in datasets_titles_unique:\n    if len(train[train[\"dataset_title\"] == dataset_title][\"dataset_label\"].unique()) > 1:\n        print(f\"'{dataset_title}':\", list(train[train[\"dataset_title\"] == dataset_title][\"dataset_label\"].unique()), \"\\n\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_titles = train['dataset_title'].unique()\ndup_title = []\ncount = []\ndup_list = []\nfor ut in unique_titles:\n    title_df = train[train['dataset_title'] == ut]\n    tdf = title_df[['Id', 'dataset_title', 'dataset_label']].drop_duplicates('dataset_label')\n    if tdf.shape[0] > 1:\n        #print(ut)\n        dup_title.append(ut)\n        count.append(tdf.shape[0])\n        dup_list.append(list(tdf['dataset_label']))\n        \ndup_df = pd.DataFrame({'dataset_title':dup_title, 'label_count':count, 'label_list':dup_list})","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dup_df.set_index('dataset_title')['label_count'].sort_values(ascending=False).plot.barh(figsize=(12,18))\nplt.title(\"No of labels that a dataset have\")\nplt.xlabel('labels_count')\nplt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.2. Datasets Popularity","metadata":{}},{"cell_type":"code","source":"dataset_titles_counts = train['dataset_title'].value_counts()\nfig = go.Figure(data=[go.Table(\n  columnwidth = [0.25, 2, 0.5],\n  header=dict(\n    values=[\"<b>Rank</b>\", \"<b>Dataset Title</b>\", \"<b>Mentions</b>\"],\n    line_color='darkslategray',\n    fill_color=\"royalblue\",\n    align='center',\n    font=dict(color='white', size=12)\n  ),\n  cells=dict(\n    values=np.array([np.array((str(i+1), \"<i>\" + x + \"</i>\", \"<b>\" + str(y) + \"</b>\", )) for i, (x, y) in enumerate(zip(dataset_titles_counts.index, dataset_titles_counts.values))]).T,\n    line_color='darkslategray',\n    # 2-D list of colors for alternating rows\n    fill_color = [[\"white\",\"lavender\"]*25],\n    align = 'center',\n    font = dict(color = 'darkslategray', size = 11)\n    ))\n])\n\nfig.update_layout(\n    title={\"text\": \"<b>Datasets Titles Mentions Counts</b>\",\n           \"x\": 0.5,\n           \"xanchor\":\"center\",\n           \"font_size\": 22},\n    margin={\"r\":20, \"l\":20})\n\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1.3. Datasets Occurence Together","metadata":{}},{"cell_type":"code","source":"multi_mentions = [] # list of tuples (Id, num_unique_mentions) \n\nfor Id in train.Id.unique():\n    num_unique_mentions = len(train[train['Id'] == Id]['dataset_title'].unique())\n    if num_unique_mentions > 1:\n        multi_mentions.append((Id, num_unique_mentions))\n\nprint(f\"There are {len(multi_mentions)} publications in the training set that mention more than 1 dataset. That is {len(multi_mentions) / len(train.Id.unique()) * 100:.2f}% of the publications.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"co_mentions = defaultdict(int)\n\nfor Id, num_mentions in multi_mentions:\n    co_mentions[tuple(sorted(train[train['Id'] == Id][\"dataset_title\"].unique()))] += 1\n\nprint(f\"There are {len(co_mentions)} unique sets of co-occurences of dataset mentions.\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"co_mentions = dict(sorted(co_mentions.items(), key=lambda item: item[1], reverse=True))\n\nfig = go.Figure(data=[go.Table(\n  columnwidth = [0.25, 2, 0.5],\n  header=dict(\n    values=[\"<b>Rank</b>\", \"<b>Datasets Co-Mentions Sets</b>\", \"<b>Occurences</b>\"],\n    line_color='darkslategray',\n    fill_color=\"royalblue\",\n    align='center',\n    font=dict(color='white', size=12)\n  ),\n  cells=dict(\n    values=np.array([np.array((str(i+1), \"<i>\" + str(k) + \"</i>\", \"<b>\" + str(v) + \"</b>\", )) for i, (k, v) in enumerate(co_mentions.items())]).T,\n    line_color='darkslategray',\n    fill_color = [[\"white\",\"lavender\"]*60],\n    align = 'center',\n    font = dict(color = 'darkslategray', size = 11)\n    ))\n])\n\nfig.update_layout(\n    title={\"text\": \"<b>Datasets Mentioned in the same Publications</b>\",\n           \"x\": 0.5,\n           \"xanchor\":\"center\",\n           \"font_size\": 22},\n    margin={\"r\":20, \"l\":20})\n\nfig.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train Json","metadata":{}},{"cell_type":"code","source":"print(\"Files in train directory : \\n\")\nprint(len(os.listdir('../input/coleridgeinitiative-show-us-the-data/train')))\nprint(os.listdir('../input/coleridgeinitiative-show-us-the-data/train')[:5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"\\nFiles in test directory : \\n\")\nprint(len(os.listdir('../input/coleridgeinitiative-show-us-the-data/test')))\nprint(os.listdir('../input/coleridgeinitiative-show-us-the-data/test')[:5])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('../input/coleridgeinitiative-show-us-the-data/train/d0fa7568-7d8e-4db9-870f-f9c6f668c17b.json') as f:\n    sample = json.load(f)\nsample","metadata":{"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train.Id=='d0fa7568-7d8e-4db9-870f-f9c6f668c17b']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Set preparation","metadata":{}},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(read_append_return)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.to_csv('train_with_text.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\n# Generate the training publications dataframe\ndf_train_publications = pd.DataFrame()\n\nfor train_file in train_files:\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id', train_file.split('/')[-1].split('.')[0])\n    df_train_publications = pd.concat([df_train_publications, file_data])\ndf_train_publications.to_csv(\"df_train_publications.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\n# Generate the training publications dataframe\ndf_test_publications = pd.DataFrame()\n\nfor test_file in test_files:\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0])\n    df_test_publications = pd.concat([df_test_publications, file_data])\ndf_test_publications.to_csv(\"df_test_publications.csv\",index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}