{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Building an algorithm that can discern what datasets have been used by publications","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set(style=\"whitegrid\")\nfrom wordcloud import WordCloud, STOPWORDS\nimport plotly.graph_objects as go\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:48:21.553238Z","iopub.execute_input":"2021-06-18T05:48:21.553637Z","iopub.status.idle":"2021-06-18T05:48:21.976736Z","shell.execute_reply.started":"2021-06-18T05:48:21.553606Z","shell.execute_reply":"2021-06-18T05:48:21.975676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"path = '../input/coleridgeinitiative-show-us-the-data/train/'\nfor f in os.listdir(path):\n    f = open(path+f)\n    \n    sample = json.load(f)\n    break\n    \nsample[0]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:48:21.978550Z","iopub.execute_input":"2021-06-18T05:48:21.978839Z","iopub.status.idle":"2021-06-18T05:48:21.999684Z","shell.execute_reply.started":"2021-06-18T05:48:21.978811Z","shell.execute_reply":"2021-06-18T05:48:21.998791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i, s in enumerate(sample):\n    print(s['section_title'])\n    \n    if (i>5):\n        break","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:48:22.001137Z","iopub.execute_input":"2021-06-18T05:48:22.001473Z","iopub.status.idle":"2021-06-18T05:48:22.008160Z","shell.execute_reply.started":"2021-06-18T05:48:22.001444Z","shell.execute_reply":"2021-06-18T05:48:22.006840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsubmit = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntest=submit[['Id']]\n\ntrain.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:48:22.009593Z","iopub.execute_input":"2021-06-18T05:48:22.009880Z","iopub.status.idle":"2021-06-18T05:48:22.102046Z","shell.execute_reply.started":"2021-06-18T05:48:22.009853Z","shell.execute_reply":"2021-06-18T05:48:22.101184Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:48:22.106148Z","iopub.execute_input":"2021-06-18T05:48:22.106471Z","iopub.status.idle":"2021-06-18T05:48:22.123801Z","shell.execute_reply.started":"2021-06-18T05:48:22.106441Z","shell.execute_reply":"2021-06-18T05:48:22.122776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef get_train_text(filename):\n    path = '../input/coleridgeinitiative-show-us-the-data/train/' + filename + '.json'\n    df = pd.read_json(path)\n    \n    text = ' '.join(df['text'])\n    return text\n\ndef get_test_text(filename):\n    path = '../input/coleridgeinitiative-show-us-the-data/test/' + filename + '.json'\n    df = pd.read_json(path)\n    \n    text = ' '.join(df['text'])\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:48:22.126026Z","iopub.execute_input":"2021-06-18T05:48:22.126471Z","iopub.status.idle":"2021-06-18T05:48:22.133932Z","shell.execute_reply.started":"2021-06-18T05:48:22.126428Z","shell.execute_reply":"2021-06-18T05:48:22.132571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in tqdm(range(len(train))):\n    filename = train.loc[i,'Id']\n    train.loc[i,'text'] = clean_text(get_train_text(filename))\n    \nfor i in range(len(test)):\n    filename = test.loc[i,'Id']\n    test.loc[i,'text'] = get_test_text(filename)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:48:22.135988Z","iopub.execute_input":"2021-06-18T05:48:22.136551Z","iopub.status.idle":"2021-06-18T05:53:11.417113Z","shell.execute_reply.started":"2021-06-18T05:48:22.136507Z","shell.execute_reply":"2021-06-18T05:53:11.415892Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(train)):\n    label = train.loc[i,'cleaned_label']\n    text = train.loc[i,'text']\n    \n    if (label in text):\n        train.loc[i,'is_present']=1\n    else:\n        train.loc[i,'is_present']=0","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:53:11.419218Z","iopub.execute_input":"2021-06-18T05:53:11.419675Z","iopub.status.idle":"2021-06-18T05:53:19.343015Z","shell.execute_reply.started":"2021-06-18T05:53:11.419628Z","shell.execute_reply":"2021-06-18T05:53:19.342241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['is_present'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:53:19.344096Z","iopub.execute_input":"2021-06-18T05:53:19.344559Z","iopub.status.idle":"2021-06-18T05:53:19.353666Z","shell.execute_reply.started":"2021-06-18T05:53:19.344529Z","shell.execute_reply":"2021-06-18T05:53:19.352636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_labels = train['cleaned_label'].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:53:19.420226Z","iopub.execute_input":"2021-06-18T05:53:19.420543Z","iopub.status.idle":"2021-06-18T05:53:19.428345Z","shell.execute_reply.started":"2021-06-18T05:53:19.420514Z","shell.execute_reply":"2021-06-18T05:53:19.427597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(len(test)):\n    text = clean_text(test.loc[i,'text'])\n    \n    prediction_labels=[]\n    for label in dataset_labels:\n        if (label in text):\n            prediction_labels.append(label)\n            \n    test.loc[i,'PredictionString'] = '|'.join(prediction_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:53:19.429615Z","iopub.execute_input":"2021-06-18T05:53:19.430112Z","iopub.status.idle":"2021-06-18T05:53:19.496651Z","shell.execute_reply.started":"2021-06-18T05:53:19.430082Z","shell.execute_reply":"2021-06-18T05:53:19.495843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:53:19.497836Z","iopub.execute_input":"2021-06-18T05:53:19.498439Z","iopub.status.idle":"2021-06-18T05:53:19.512397Z","shell.execute_reply.started":"2021-06-18T05:53:19.498396Z","shell.execute_reply":"2021-06-18T05:53:19.511565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit['PredictionString'] = test['PredictionString']\nsubmit","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:53:19.513839Z","iopub.execute_input":"2021-06-18T05:53:19.514137Z","iopub.status.idle":"2021-06-18T05:53:19.530304Z","shell.execute_reply.started":"2021-06-18T05:53:19.514107Z","shell.execute_reply":"2021-06-18T05:53:19.529552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T05:53:19.531337Z","iopub.execute_input":"2021-06-18T05:53:19.531771Z","iopub.status.idle":"2021-06-18T05:53:19.544069Z","shell.execute_reply.started":"2021-06-18T05:53:19.531730Z","shell.execute_reply":"2021-06-18T05:53:19.543189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}