{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport re\nimport json\nimport pickle\nfrom collections import defaultdict, Counter\nimport gc\n\nimport pandas as pd\nimport numpy as np\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score, plot_confusion_matrix\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom tqdm.notebook import tqdm\ntqdm.pandas()\n\n%matplotlib inline\nfrom IPython.core.display import display, HTML\ndisplay(HTML(\"<style>.container { width:100% !important; }</style>\")) # full screen width of Jupyter notebook\npd.options.display.max_rows, pd.options.display.max_columns = 500, 100\n\n# NLP imports\nimport nltk\n\n# Neural network imports\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nprint( 'tf version:', tf.__version__)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\"\"\" Loading data\"\"\"\ndata_path = '../input/coleridgeinitiative-show-us-the-data/'\n\ndef read_json_from_folder(folder_name):\n    json_dict = {}\n    for filename in os.listdir(folder_name):\n        with open(os.path.join(folder_name, filename)) as f:\n            json_dict[filename[:-5]] = json.load(f)\n    return json_dict\n\n# train_dict = read_json_from_folder(os.path.join(data_path, 'train'))\n# test_dict = read_json_from_folder(os.path.join(data_path, 'test'))\ntrain_df = pd.read_csv(os.path.join(data_path, 'train.csv'))\nsample_sub = pd.read_csv(os.path.join(data_path,'sample_submission.csv'))\n    \n# len(train_dict), len(test_dict), \ntrain_df.shape, sample_sub.shape","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ext dataset 1\nadnl_govt_labels = pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\nadnl_govt_labels.sample(5)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_training_text(txt):\n    \"\"\"\n    similar to the default clean_text function but without lowercasing.\n    \"\"\"\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\ndef read_json_pub(filename, train_data_path=paper_train_folder, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['ext_cleaned_label'] = ''\n\nto_append = []\nfor index, row in tqdm(train_df.iterrows(), total = train_df.shape[0]):\n    to_append = [row['Id'], []]\n    large_string = str(read_json_pub(row['Id']))\n    clean_string = text_cleaning(large_string)\n    for index, row2 in adnl_govt_labels.iterrows():\n        query_string = str(row2['title'])\n        if query_string in clean_string:\n            to_append[1].append(clean_text(query_string))\n    \n#     literal_preds.append(to_append[1])\n    train_df.loc[train_df['Id']==row['Id'], 'ext_cleaned_label'] = '|'.join(np.unique(to_append[1]))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.to_csv('./train_df.csv', index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.read_csv('./train_df.csv').head()","metadata":{},"execution_count":null,"outputs":[]}]}