{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nimport os\nimport re\nimport json\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport nltk\n\n# Can we use BERT and Spacy ?\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\n\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm.autonotebook import tqdm\nimport string\n\n# os.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We are provided with 4 main pieces of data:\n\n* `train.csv:` The CSV file containing all the metadata of the publications, such as their title and the dataset they utilize.\n* `train:` The directory containing the actual publications that are referenced in train.csvin JSON format.\n* `test:` The directory containing the actual publications that will be used for testing purposes (thus, with no ground truth CSV file available).\n* `sample_submission.csv:` The CSV file containing all the publications IDs in the test set, for which we'll have to populate the prediction column.","metadata":{}},{"cell_type":"code","source":"# reading csv files and train & test file paths\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(6)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"[print(f\"{col}:{len(train_df[col].unique())}\") for col in train_df.columns]   #finding unique values in each column","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()   #tqdm is used to show any code running with a progress bar. \ntrain_df['text'] = train_df['Id'].progress_apply(read_append_return)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('r[^\\w\\s]', ' ', str(text).lower()).strip()\n    lem = nltk.stem.wordnet.WordNetLemmatizer()\n    text = lem.lemmatize(text)\n#     text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n#     text = re.sub(\"/'+/g\", ' ', text)\n    \n    return text","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\ntrain_df['text'] = train_df['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words = list( train_df['cleaned_label'].values)\nstopwords=['ourselves', 'hers','the','of','and','in', 'between', 'yourself', 'but', 'again','of', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']\nsplit_words=[]\nfor word in words:\n    lo_w=[]\n    list_of_words=str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = train_df[train_df['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = lables_list","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.set_option(\"display.max_rows\", None, \"display.max_columns\", None)\nsubmission.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}