{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json\nfrom collections import Counter\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check json file structure in train/test directory","metadata":{}},{"cell_type":"code","source":"# Check data type\n# All data in train/test directory are json files\ntrain_path_lists = []\ntest_path_lists = []\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    if dirname == \"/kaggle/input/coleridgeinitiative-show-us-the-data/train\":\n        for filename in filenames:\n            train_path_lists.append(os.path.join(dirname, filename))\n        print(f\"length of all train data: {len(filenames)}\")\n        print(f\"number of json file in train data : {len([i for i in filenames if i[-4:] == 'json'])}\")\n        \n    if dirname == \"/kaggle/input/coleridgeinitiative-show-us-the-data/test\":\n        for filename in filenames:\n            test_path_lists.append(os.path.join(dirname, filename))\n        print(f\"length of all test data: {len(filenames)}\")\n        print(f\"number of json file in test data : {len([i for i in filenames if i[-4:] == 'json'])}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Pick up a json file sample\n# Index0 stores abstract, index1 stores introduction\njson_open = open('../input/coleridgeinitiative-show-us-the-data/train/0007f880-0a9b-492d-9a58-76eb0b0e0bd7.json', 'r')\njson_df = json.load(json_open)\nprint(f\"index0 keys: {json_df[0].keys()}\")\nprint()\nprint(json_df[0])\nprint()\nprint(f\"index1 keys: {json_df[1].keys()}\")\nprint()\nprint(json_df[1])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# All json file are same structure?\n# ⇨ No, some file have more indexes. \n# Train directory\nindex_num = []\nsection_title = []\nsection_title_value = []\n\nfor path in train_path_lists:\n    with open(path, 'r') as json_open:\n        json_df = json.load(json_open)\n        index_num.append(len(json_df))\n        for i in range(len(json_df)):\n            for key,value in json_df[i].items():\n                section_title.append(key)\n                if key == \"section_title\":\n                    section_title_value.append(value)\nprint(f\"index_num: {Counter(index_num)}\")\nprint()\nprint(f\"section_title: {Counter(section_title)}\")\nprint()\nprint(f\"section_title_value: {Counter(section_title_value)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Test directory\nindex_num = []\nsection_title = []\nsection_title_value = []\n\nfor path in test_path_lists:\n    with open(path, 'r') as json_open:\n        json_df = json.load(json_open)\n        index_num.append(len(json_df))\n        for i in range(len(json_df)):\n            for key,value in json_df[i].items():\n                section_title.append(key)\n                if key == \"section_title\":\n                    section_title_value.append(value)\nprint(f\"index_num: {Counter(index_num)}\")\nprint()\nprint(f\"section_title: {Counter(section_title)}\")\nprint()\nprint(f\"section_title_value: {Counter(section_title_value)}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Check other json.file\nwith open(\"../input/coleridgeinitiative-show-us-the-data/train/0008656f-0ba2-4632-8602-3017b44c2e90.json\", 'r') as json_open:\n    json_df = json.load(json_open)\njson_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Check train.csv","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain_df","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.describe()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Thre is no null in all columns\ntrain_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Conbine train.csv and json file which have same id\n# Why there are 2 columns with same Id?\ntrain_df[train_df[\"Id\"]==\"d0fa7568-7d8e-4db9-870f-f9c6f668c17b\"].value_counts","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"section_titles = []\nwith open(\"../input/coleridgeinitiative-show-us-the-data/train/d0fa7568-7d8e-4db9-870f-f9c6f668c17b.json\", 'r') as json_open:\n    json_df = json.load(json_open)\nprint(json_df[0])\nprint()\nprint(json_df[1])\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_json(\"../input/coleridgeinitiative-show-us-the-data/train/d0fa7568-7d8e-4db9-870f-f9c6f668c17b.json\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[value for df in json_df for key, value in df.items() if key == 'section_title']","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Transform json to csv","metadata":{}},{"cell_type":"code","source":"# Generate the training publications dataframe\ndf_train_publications = pd.DataFrame()\n\nfor train_file in train_path_lists:\n    file_data = pd.read_json(train_file)\n    file_data.insert(0,'pub_id', train_file.split('/')[-1].split('.')[0])\n    df_train_publications = pd.concat([df_train_publications, file_data])\n\ndf_train_publications.to_csv(\"df_train_publications.csv\",index=False)\n\ndf_train_publications","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generate the testing publications dataframe\ndf_test_publications = pd.DataFrame()\n\nfor test_file in test_path_lists:\n    file_data = pd.read_json(test_file)\n    file_data.insert(0,'pub_id', test_file.split('/')[-1].split('.')[0])\n    df_test_publications = pd.concat([df_test_publications, file_data])\n\ndf_test_publications.to_csv(\"df_test_publications.csv\",index=False)\n\ndf_test_publications","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test_publications[df_test_publications['pub_id'] == \"2100032a-7c33-4bff-97ef-690822c43466\"].text.str.cat(sep='\\n').lower()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# If dataset_title includes the text of the id, appended the dataset_title\nsubmission_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv', index_col=0)\ndatasets_titles = [x.lower() for x in set(train_df['dataset_title'].unique()).union(set(train_df['dataset_label'].unique()))]\n\nlabels = []\nfor index in submission_df.index:\n    publication_text = df_test_publications[df_test_publications['pub_id'] == index].text.str.cat(sep='\\n').lower()\n    label = []\n    for dataset_title in datasets_titles:\n        if dataset_title in publication_text:\n            label.append(clean_text(dataset_title))\n    labels.append('|'.join(label))\n\nsubmission_df['PredictionString'] = labels\n\nsubmission_df.to_csv('submission.csv')\n\nsubmission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}