{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**All necessary libraries**","metadata":{}},{"cell_type":"code","source":"import json\nimport os\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport glob\nimport matplotlib.pyplot as plt\nfrom nltk.corpus import stopwords\nimport re\nimport string\nfrom wordcloud import WordCloud, STOPWORDS\nimport seaborn as sns\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:33.174977Z","iopub.execute_input":"2021-05-29T06:19:33.175409Z","iopub.status.idle":"2021-05-29T06:19:34.823531Z","shell.execute_reply.started":"2021-05-29T06:19:33.175308Z","shell.execute_reply":"2021-05-29T06:19:34.822462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lets understand the data #\n\n1. The objective of the competition is to identify the mention of datasets within scientific publications.\n2. The predictions will be short excerpts from the publications that appear to note a dataset.\n\n**Files present in our dataset**\n\n* train - the full text of the training set's publications in JSON format, broken into sections with section titles\n* test - the full text of the test set's publications in JSON format, broken into sections with section titles\n* train.csv - labels and metadata for the training set\n* sample_submission.csv - a sample submission file in the correct format\n","metadata":{}},{"cell_type":"code","source":"all_dir = os.listdir(\"../input/coleridgeinitiative-show-us-the-data\")\nprint(all_dir)\ntrain_path = \"../input/coleridgeinitiative-show-us-the-data/train\"\ntest_path = \"../input/coleridgeinitiative-show-us-the-data/train\"\nsub_path = \"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\"\ntrain_path = \"../input/coleridgeinitiative-show-us-the-data/train.csv\"","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:34.825217Z","iopub.execute_input":"2021-05-29T06:19:34.825688Z","iopub.status.idle":"2021-05-29T06:19:34.837632Z","shell.execute_reply.started":"2021-05-29T06:19:34.825632Z","shell.execute_reply":"2021-05-29T06:19:34.836306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**columns in csv files**\n\n* id - publication id - note that there are multiple rows for some training documents, indicating multiple mentioned datasets\n* pub_title - title of the publication (a small number of publications have the same title)\n* dataset_title - the title of the dataset that is mentioned within the publication\n* dataset_label - a portion of the text that indicates the dataset\n* cleaned_label - the dataset_label, as passed through the clean_text function from the Evaluation page\n","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(train_path)  # reading csv file\ntrain_df.head(5) # get the first 5 rows","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:34.841765Z","iopub.execute_input":"2021-05-29T06:19:34.842077Z","iopub.status.idle":"2021-05-29T06:19:35.082685Z","shell.execute_reply.started":"2021-05-29T06:19:34.842047Z","shell.execute_reply":"2021-05-29T06:19:35.081586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(\"../input/coleridgeinitiative-show-us-the-data/train/0007f880-0a9b-492d-9a58-76eb0b0e0bd7.json\") as f:\n    sample = json.load(f)\nsample[:2]","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:35.084701Z","iopub.execute_input":"2021-05-29T06:19:35.085153Z","iopub.status.idle":"2021-05-29T06:19:35.103525Z","shell.execute_reply.started":"2021-05-29T06:19:35.085109Z","shell.execute_reply":"2021-05-29T06:19:35.102486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**** main aim of the competition is to get the dataset label for the particular publication which is provied in json format and use the same dataset labels to get prediction for the test dataset ****","metadata":{}},{"cell_type":"code","source":"duplicate_df = train_df[train_df['Id'] == \"170113f9-399c-489e-ab53-2faf5c64c5bc\"].drop_duplicates(\"dataset_title\")\nduplicate_df","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:35.105163Z","iopub.execute_input":"2021-05-29T06:19:35.10562Z","iopub.status.idle":"2021-05-29T06:19:35.132598Z","shell.execute_reply.started":"2021-05-29T06:19:35.105577Z","shell.execute_reply":"2021-05-29T06:19:35.131536Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**from above it is clear that each id has more than one dataset labels which may also belong to different Id**","metadata":{}},{"cell_type":"code","source":"train_df.describe()  #get detail information about id, publication title, dataset title, dataset label.","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:35.134271Z","iopub.execute_input":"2021-05-29T06:19:35.134897Z","iopub.status.idle":"2021-05-29T06:19:35.226311Z","shell.execute_reply.started":"2021-05-29T06:19:35.134863Z","shell.execute_reply":"2021-05-29T06:19:35.225241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **let us check the Id having different dataset labels**","metadata":{}},{"cell_type":"code","source":"dataset_title = train_df.groupby('Id').count()[['dataset_title']].sort_values(by = \"dataset_title\", ascending = False)\nid_pub_title = dataset_title[dataset_title['dataset_title'] >1][['dataset_title']].reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:35.227922Z","iopub.execute_input":"2021-05-29T06:19:35.228405Z","iopub.status.idle":"2021-05-29T06:19:35.271654Z","shell.execute_reply.started":"2021-05-29T06:19:35.228359Z","shell.execute_reply":"2021-05-29T06:19:35.270578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (13,13))\nsns.barplot(id_pub_title['dataset_title'].iloc[:20], id_pub_title['Id'].iloc[:20])\nplt.title(\"dataset titles vs publication\")\nplt.xticks(fontsize=13)\nplt.yticks(fontsize=13)\nplt.ylabel(\"\")\nplt.xlabel(\"Count\", fontsize=14)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:35.275578Z","iopub.execute_input":"2021-05-29T06:19:35.275922Z","iopub.status.idle":"2021-05-29T06:19:35.770511Z","shell.execute_reply.started":"2021-05-29T06:19:35.275891Z","shell.execute_reply":"2021-05-29T06:19:35.769216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Similarly we can get for the publication title vs dataset title**","metadata":{}},{"cell_type":"code","source":"pub_title = train_df.groupby('pub_title').count()[['dataset_title']].sort_values(by = ['dataset_title'], ascending = False)\nid_title = pub_title.reset_index()\nid_title","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:35.772959Z","iopub.execute_input":"2021-05-29T06:19:35.773423Z","iopub.status.idle":"2021-05-29T06:19:35.828066Z","shell.execute_reply.started":"2021-05-29T06:19:35.773363Z","shell.execute_reply":"2021-05-29T06:19:35.826701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize = (13,13))\nsns.barplot(id_title['dataset_title'].iloc[:20], id_pub_title['Id'].iloc[:20])\nplt.title(\"dataset titles vs publication titles\")\nplt.xticks(fontsize=13)\nplt.yticks(fontsize=13)\nplt.ylabel(\"\")\nplt.xlabel(\"Count\", fontsize=14)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:35.829981Z","iopub.execute_input":"2021-05-29T06:19:35.830513Z","iopub.status.idle":"2021-05-29T06:19:36.301532Z","shell.execute_reply.started":"2021-05-29T06:19:35.830466Z","shell.execute_reply":"2021-05-29T06:19:36.30046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Now check how many different dataset title are present with different label**","metadata":{}},{"cell_type":"code","source":"data_title = train_df.groupby('dataset_title').count()[['dataset_label']].sort_values(by = ['dataset_title'], ascending = False)\nid_title = pub_title.reset_index()\nid_title","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:36.303219Z","iopub.execute_input":"2021-05-29T06:19:36.303753Z","iopub.status.idle":"2021-05-29T06:19:36.336119Z","shell.execute_reply.started":"2021-05-29T06:19:36.303706Z","shell.execute_reply":"2021-05-29T06:19:36.334761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nplt.figure(figsize = (13,13))\nsns.barplot(id_title['dataset_title'].iloc[:20], id_pub_title['Id'].iloc[:20])\nplt.title(\"dataset titles vs dataset label\")\nplt.xticks(fontsize=13)\nplt.yticks(fontsize=13)\nplt.ylabel(\"\")\nplt.xlabel(\"Count\", fontsize=14)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:36.338012Z","iopub.execute_input":"2021-05-29T06:19:36.338554Z","iopub.status.idle":"2021-05-29T06:19:36.809003Z","shell.execute_reply.started":"2021-05-29T06:19:36.338506Z","shell.execute_reply":"2021-05-29T06:19:36.807914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.sample(5) # some  of the random samples","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:36.810697Z","iopub.execute_input":"2021-05-29T06:19:36.811162Z","iopub.status.idle":"2021-05-29T06:19:36.828303Z","shell.execute_reply.started":"2021-05-29T06:19:36.811117Z","shell.execute_reply":"2021-05-29T06:19:36.826789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Word cloud representation","metadata":{}},{"cell_type":"code","source":"stopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color='white',\n                      stopwords=stopwords,\n                      max_words=100,\n                      max_font_size=30,\n                      scale=3,\n                      random_state=1)\n   \nwordcloud=wordcloud.generate(str(train_df['dataset_title'].unique()))\nfig = plt.figure(1, figsize=(12, 12))\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:36.830654Z","iopub.execute_input":"2021-05-29T06:19:36.83139Z","iopub.status.idle":"2021-05-29T06:19:37.366911Z","shell.execute_reply.started":"2021-05-29T06:19:36.831327Z","shell.execute_reply":"2021-05-29T06:19:37.365647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**word with larger size are repeated the most in the dataset title\nsimilary you can plot the sane wordcloud for the text data**","metadata":{}},{"cell_type":"markdown","source":"**Now lets concat the text in json file with our train csv file**","metadata":{}},{"cell_type":"code","source":"def get_text(filename, test=False):\n    if test:\n        df = pd.read_json('../input/coleridgeinitiative-show-us-the-data/test/{}.json'.format(filename))\n    else:\n        df = pd.read_json('../input/coleridgeinitiative-show-us-the-data/train/{}.json'.format(filename))\n    text = \" \".join(list(df['text']))\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:37.368698Z","iopub.execute_input":"2021-05-29T06:19:37.369118Z","iopub.status.idle":"2021-05-29T06:19:37.376253Z","shell.execute_reply.started":"2021-05-29T06:19:37.369079Z","shell.execute_reply":"2021-05-29T06:19:37.375057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['text'] = train_df['Id'].apply(get_text)\ntrain_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:19:37.37808Z","iopub.execute_input":"2021-05-29T06:19:37.37892Z","iopub.status.idle":"2021-05-29T06:22:59.898016Z","shell.execute_reply.started":"2021-05-29T06:19:37.378849Z","shell.execute_reply":"2021-05-29T06:22:59.896918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Text Preprocessing(optional)","metadata":{}},{"cell_type":"markdown","source":"***lower casing the text***","metadata":{}},{"cell_type":"code","source":"train_df['lower'] = train_df['text'].str.lower()","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:22:59.899709Z","iopub.execute_input":"2021-05-29T06:22:59.900173Z","iopub.status.idle":"2021-05-29T06:23:04.627236Z","shell.execute_reply.started":"2021-05-29T06:22:59.90013Z","shell.execute_reply":"2021-05-29T06:23:04.626059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:23:04.62905Z","iopub.execute_input":"2021-05-29T06:23:04.629513Z","iopub.status.idle":"2021-05-29T06:23:04.655316Z","shell.execute_reply.started":"2021-05-29T06:23:04.629463Z","shell.execute_reply":"2021-05-29T06:23:04.65414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***removing punctuation***\n\n\nwe also need to remove the punctuation symbols from the text.\n\npunctuation in python contains the following punctuation symbols\n\n!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_{|}~`\n\nWe can add or remove more punctuations as per our need.","metadata":{}},{"cell_type":"code","source":"PUNCT_TO_REMOVE = '!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~\\n'\ndef remove_punctuation(text):\n    \"\"\"custom function to remove the punctuation\"\"\"\n    return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))\n\ntrain_df[\"text_wo_punct\"] = train_df[\"lower\"].apply(lambda text: remove_punctuation(text))\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:23:04.656977Z","iopub.execute_input":"2021-05-29T06:23:04.657801Z","iopub.status.idle":"2021-05-29T06:24:37.976616Z","shell.execute_reply.started":"2021-05-29T06:23:04.657756Z","shell.execute_reply":"2021-05-29T06:24:37.975573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('train_df.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2021-05-29T06:24:37.978297Z","iopub.execute_input":"2021-05-29T06:24:37.978811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***removing stopwords***","metadata":{}},{"cell_type":"code","source":"#Stopwords = list(stopwords.words('english'))\nfrom nltk.corpus import stopwords\nStopwords = list(stopwords.words('english'))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Stopwords = list(stopwords.words('english'))\ndef remove_stopwords(text):\n    return \" \".join([word for word in str(text).split() if word not in Stopwords])\n\ntrain_df[\"text_wo_stop\"] = train_df[\"text_wo_punct\"].apply(lambda text: remove_stopwords(text))\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"str1 = train_df['text_wo_stop'][0]\nstr2 = train_df['text'][0]\n#results\nprint(str1[:250])\nprint(str2[:250])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# String Matching\n\nNow we have to get all the dataset titles from the csv file and use the same dataset titles for getting predictions over test files","metadata":{}},{"cell_type":"code","source":"test_files = os.listdir('../input/coleridgeinitiative-show-us-the-data/test')\ntest = pd.DataFrame({'Id':test_files})\ntest['Id'] = test['Id'].apply(lambda x : x.split('.')[0])\ntest['text'] = test['Id'].apply(get_text, test=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles = [x.lower() for x in set(train_df['dataset_title'].unique()).union(set(train_df['dataset_label'].unique()))]\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to clean the text data\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#matching the string\nlabels = []\nfor index in submission_df['Id']:\n    pub_text = test[test['Id'] == index].text.str.cat(sep = '\\n').lower()\n    #print(pub_text)\n    label = []\n    for data_title in titles:\n        if data_title in pub_text:\n            label.append(clean_text(data_title))\n            \n            \n    labels.append(\"|\".join(label))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df['PredictionString'] = labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Thankyou for having patience and reading my notebook\nplease upvote if you understood**\n\n**Credits:**\nhttps://www.kaggle.com/anthokalel/coleridge-complete-eda","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}