{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# # This Python 3 environment comes with many helpful analytics libraries installed\n# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# # For example, here's several helpful packages to load\n\n# import numpy as np # linear algebra\n# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# # Input data files are available in the read-only \"../input/\" directory\n# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-11T09:27:47.491838Z","iopub.execute_input":"2021-06-11T09:27:47.492222Z","iopub.status.idle":"2021-06-11T09:27:47.497664Z","shell.execute_reply.started":"2021-06-11T09:27:47.492192Z","shell.execute_reply":"2021-06-11T09:27:47.496556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Competition Objective\nOne liner - We are required to build an algorithm that can find our what are the datasets that a publications uses.\n\nDescription - In this competition, we need to develop an algorithm to automate the discovery of how scientific data are referenced in publications. We have with us the full text of scientific publications from numerous research areas, we'll identify data sets that the publications' authors used in their work.\n\nWe have a labelled dataset (train set) that we'll use to develop our algorithm. The unlabelled dataset (test set) will be used for evaluation of the algorithm.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.499139Z","iopub.execute_input":"2021-06-11T09:27:47.499473Z","iopub.status.idle":"2021-06-11T09:27:47.522408Z","shell.execute_reply.started":"2021-06-11T09:27:47.499415Z","shell.execute_reply":"2021-06-11T09:27:47.521611Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data provided to us in this competition comprises of following 4 items:\n\n* train.csv - This file comprises of publication-id and the associated data title and labels along with the cleansed label.\n* train folder - This folder provides json file for each of the id's present in the above csv file, with each of the json file describing section-details of the publication and the associated text with it.\n* sample_submission.csv - This file comprises of id, for which we must predict the data-set title.\n* test folder - This folder provides json file for each of the id's present in test.csv file and the json file describes the section details of the publication and the associated text.","metadata":{}},{"cell_type":"code","source":"import json\nwith open('../input/coleridgeinitiative-show-us-the-data/train/f8b03c87-9d1a-4f20-b76b-cb6c69d447b2.json') as f:\n    sample = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.524338Z","iopub.execute_input":"2021-06-11T09:27:47.524779Z","iopub.status.idle":"2021-06-11T09:27:47.537871Z","shell.execute_reply.started":"2021-06-11T09:27:47.524741Z","shell.execute_reply":"2021-06-11T09:27:47.536852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[:2]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.539796Z","iopub.execute_input":"2021-06-11T09:27:47.540273Z","iopub.status.idle":"2021-06-11T09:27:47.551682Z","shell.execute_reply.started":"2021-06-11T09:27:47.540239Z","shell.execute_reply":"2021-06-11T09:27:47.550381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for s in sample:\n    print(s['section_title'])","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.553403Z","iopub.execute_input":"2021-06-11T09:27:47.554054Z","iopub.status.idle":"2021-06-11T09:27:47.574933Z","shell.execute_reply.started":"2021-06-11T09:27:47.554003Z","shell.execute_reply":"2021-06-11T09:27:47.573300Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\ndf_train=pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.576398Z","iopub.execute_input":"2021-06-11T09:27:47.576850Z","iopub.status.idle":"2021-06-11T09:27:47.666341Z","shell.execute_reply.started":"2021-06-11T09:27:47.576805Z","shell.execute_reply":"2021-06-11T09:27:47.665290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.667745Z","iopub.execute_input":"2021-06-11T09:27:47.668081Z","iopub.status.idle":"2021-06-11T09:27:47.675925Z","shell.execute_reply.started":"2021-06-11T09:27:47.668047Z","shell.execute_reply":"2021-06-11T09:27:47.674656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in df_train.columns:\n    print(f\"{col}: {len(df_train[col].unique())}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.677705Z","iopub.execute_input":"2021-06-11T09:27:47.678501Z","iopub.status.idle":"2021-06-11T09:27:47.710127Z","shell.execute_reply.started":"2021-06-11T09:27:47.678452Z","shell.execute_reply":"2021-06-11T09:27:47.709042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test=pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ndf_test","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.712796Z","iopub.execute_input":"2021-06-11T09:27:47.713134Z","iopub.status.idle":"2021-06-11T09:27:47.729079Z","shell.execute_reply.started":"2021-06-11T09:27:47.713100Z","shell.execute_reply":"2021-06-11T09:27:47.727567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input = pd.DataFrame(columns=['id','section_title','text','data_label'])\nids=df_train['Id'].values\nimport warnings\nwarnings.filterwarnings(\"ignore\", 'This pattern has match groups')\nlen(ids)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.731327Z","iopub.execute_input":"2021-06-11T09:27:47.731709Z","iopub.status.idle":"2021-06-11T09:27:47.745310Z","shell.execute_reply.started":"2021-06-11T09:27:47.731672Z","shell.execute_reply":"2021-06-11T09:27:47.743800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.747095Z","iopub.execute_input":"2021-06-11T09:27:47.747594Z","iopub.status.idle":"2021-06-11T09:27:47.771373Z","shell.execute_reply.started":"2021-06-11T09:27:47.747543Z","shell.execute_reply":"2021-06-11T09:27:47.770110Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.773115Z","iopub.execute_input":"2021-06-11T09:27:47.773542Z","iopub.status.idle":"2021-06-11T09:27:47.791273Z","shell.execute_reply.started":"2021-06-11T09:27:47.773498Z","shell.execute_reply":"2021-06-11T09:27:47.790077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_input.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.792539Z","iopub.execute_input":"2021-06-11T09:27:47.792987Z","iopub.status.idle":"2021-06-11T09:27:47.812877Z","shell.execute_reply.started":"2021-06-11T09:27:47.792942Z","shell.execute_reply":"2021-06-11T09:27:47.811344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from wordcloud import WordCloud, ImageColorGenerator\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport nltk\nimport pickle\nimport pyLDAvis.sklearn\nfrom collections import Counter\nfrom textblob import TextBlob\nfrom nltk.tokenize import word_tokenize\nfrom nltk.probability import FreqDist\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import LatentDirichletAllocation, NMF\nfrom wordcloud import WordCloud, ImageColorGenerator\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.814694Z","iopub.execute_input":"2021-06-11T09:27:47.815078Z","iopub.status.idle":"2021-06-11T09:27:47.828680Z","shell.execute_reply.started":"2021-06-11T09:27:47.814997Z","shell.execute_reply":"2021-06-11T09:27:47.827419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words =list( df_input['data_label'].values)\nwords=[word.split() for word in words]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.829908Z","iopub.execute_input":"2021-06-11T09:27:47.830201Z","iopub.status.idle":"2021-06-11T09:27:47.848235Z","shell.execute_reply.started":"2021-06-11T09:27:47.830172Z","shell.execute_reply":"2021-06-11T09:27:47.846934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"allwords = []\nfor wordlist in words:\n    allwords += wordlist\n#print(allwords)\n\n\n\n\n","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.850450Z","iopub.execute_input":"2021-06-11T09:27:47.850793Z","iopub.status.idle":"2021-06-11T09:27:47.859779Z","shell.execute_reply.started":"2021-06-11T09:27:47.850758Z","shell.execute_reply":"2021-06-11T09:27:47.858838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles_list = list(df_train['dataset_label'])\ncommontitles =FreqDist(titles_list).most_common(25)\nprint(commontitles)\n\nx, y = zip(*commontitles)\nplt.figure(figsize=(60,40))\n#plt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Dataset titles', fontsize=50)\nplt.ylabel('Frequency of Dataset Labels', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40, ha='right')\nplt.title('Freq of 25 Most Common Dataset labels', fontsize=50)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:47.860973Z","iopub.execute_input":"2021-06-11T09:27:47.861287Z","iopub.status.idle":"2021-06-11T09:27:49.291348Z","shell.execute_reply.started":"2021-06-11T09:27:47.861256Z","shell.execute_reply":"2021-06-11T09:27:49.290231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles_list = list(df_train['dataset_title'])\ncommontitles =FreqDist(titles_list).most_common(45)\nprint(commontitles)\n\nx, y = zip(*commontitles)\nplt.figure(figsize=(70,40))\n#plt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Dataset titles', fontsize=50)\nplt.ylabel('Frequency of Dataset Titles', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40, ha='right')\nplt.title('Freq of 25 Most Common Dataset titles', fontsize=50)\nplt.savefig(\"test.png\")\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:27:49.292969Z","iopub.execute_input":"2021-06-11T09:27:49.293296Z","iopub.status.idle":"2021-06-11T09:27:53.607711Z","shell.execute_reply.started":"2021-06-11T09:27:49.293264Z","shell.execute_reply":"2021-06-11T09:27:53.606554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Finding the dataset labels per title to filter. ","metadata":{}},{"cell_type":"code","source":"print('unique dataset titles: ' ,df_train['dataset_title'].nunique())\nprint('unique dataset labels: ' ,df_train['dataset_label'].nunique())","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:28:27.112384Z","iopub.execute_input":"2021-06-11T09:28:27.112813Z","iopub.status.idle":"2021-06-11T09:28:27.130958Z","shell.execute_reply.started":"2021-06-11T09:28:27.112781Z","shell.execute_reply":"2021-06-11T09:28:27.129754Z"},"trusted":true},"execution_count":null,"outputs":[]}]}