{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nimport os\nimport re\nimport json\nimport glob\nfrom collections import defaultdict\nfrom textblob import TextBlob\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport seaborn as sns\n\nimport nltk\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\nfrom nltk.probability import FreqDist\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm.autonotebook import tqdm\nimport string\n\n%matplotlib inline\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We are provided with 4 main pieces of data:\n\n* `train.csv:` The CSV file containing all the metadata of the publications, such as their title and the dataset they utilize.\n* `train:` The directory containing the actual publications that are referenced in train.csvin JSON format.\n* `test:` The directory containing the actual publications that will be used for testing purposes (thus, with no ground truth CSV file available).\n* `sample_submission.csv:` The CSV file containing all the publications IDs in the test set, for which we'll have to populate the prediction column."},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# reading csv files and train & test file paths\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.head(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[print(f\"{col}:{len(train_df[col].unique())}\") for col in train_df.columns]   #finding unique values in each column","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntqdm.pandas()   #tqdm is used to show any code running with a progress bar. \ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.parsing.preprocessing import remove_stopwords\nfrom nltk.tokenize import word_tokenize, sent_tokenize\nfrom nltk.stem.porter import PorterStemmer\nfrom gensim.summarization import summarize\nfrom gensim.summarization import keywords\nporter = PorterStemmer()\ndef lower_case(input_str):\n    input_str = input_str.lower()\n    return input_str\ndef stemSentence(sentence):\n    token_words=word_tokenize(sentence)\n    token_words\n    stem_sentence=[]\n    for word in token_words:\n        stem_sentence.append(porter.stem(word))\n        stem_sentence.append(\" \")\n    return \"\".join(stem_sentence)\n\ndef text_preprocessing(text):\n    text= re.sub('r[^\\w\\s]',' ',text)\n    #text= re.sub('[^a-zA-z0-9\\s]','',text)\n    text=lower_case(text)\n    text=remove_stopwords(text)\n    text=stemSentence(text)    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntqdm.pandas()\n#train_df['text'] = train_df['text'].progress_apply(text_cleaning)\ntrain_df['text'] = train_df['text'].progress_apply(text_preprocessing)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It takes time!......................"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n    #return re.sub('r[^\\w\\s]', ' ', str(txt).lower()).strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n\n    temp_df = train_df[train_df['text'] == text_preprocessing(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append(' | '.join(cleaned_labels))\n    id_list.append(row_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = lables_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hurray! We are done with the submission and model. Hope you like this kernel. If so, don't forget to upvote and leave your valuable comment. Thank you😊"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}