{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\n\nimport os\nimport re\nimport json\nimport glob\nfrom collections import defaultdict, Counter\nfrom textblob import TextBlob\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport seaborn as sns\n\nimport nltk\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\nfrom nltk.probability import FreqDist\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm.autonotebook import tqdm\nimport string\n\n%matplotlib inline\n\nos.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-06T06:40:53.673678Z","iopub.execute_input":"2021-06-06T06:40:53.674327Z","iopub.status.idle":"2021-06-06T06:41:08.261239Z","shell.execute_reply.started":"2021-06-06T06:40:53.674201Z","shell.execute_reply":"2021-06-06T06:41:08.260145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. Load Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/show-us-the-datanew/new_train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T06:59:43.84865Z","iopub.execute_input":"2021-06-06T06:59:43.849379Z","iopub.status.idle":"2021-06-06T07:00:05.145121Z","shell.execute_reply.started":"2021-06-06T06:59:43.849331Z","shell.execute_reply":"2021-06-06T07:00:05.143775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:05.147224Z","iopub.execute_input":"2021-06-06T07:00:05.147656Z","iopub.status.idle":"2021-06-06T07:00:05.159305Z","shell.execute_reply.started":"2021-06-06T07:00:05.147611Z","shell.execute_reply":"2021-06-06T07:00:05.157546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. How many kinds of cleaned_labels?\n\n130 cleaned_labels","metadata":{}},{"cell_type":"code","source":"existing_set = Counter(list(train_df['cleaned_label']))\nsorted_set = sorted(existing_set.items(), key=lambda x: x[1], reverse=True)\ntotal_set = 0\nfor name, num in sorted_set:\n#     print(\"{}: {}\".format(name, num))\n    total_set += num\nprint(total_set)\nmostcommon_set = sorted_set[:10]\nx, y = zip(*mostcommon_set)\nplt.figure(figsize=(50,30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Datasets', fontsize=50)\nplt.ylabel('Frequency of Datasets', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.tight_layout(pad=0)\nplt.title('Freq of 10 Most Common Datasets in cleaned_label', fontsize=60)\nplt.show()\n# existing_set = set(train_df['cleaned_label'])\n# for item in existing_set:\n#     print(\"the {} has found {}\".format(item, list(train_df['cleaned_label']).count(item)))\nexisting_labels = train_df['cleaned_label'].unique()\nprint(len(existing_labels))\n# print(existing_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:14.372716Z","iopub.execute_input":"2021-06-06T07:00:14.373183Z","iopub.status.idle":"2021-06-06T07:00:15.354015Z","shell.execute_reply.started":"2021-06-06T07:00:14.373142Z","shell.execute_reply":"2021-06-06T07:00:15.35257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt, nlp=nlp):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:20.123685Z","iopub.execute_input":"2021-06-06T07:00:20.124147Z","iopub.status.idle":"2021-06-06T07:00:20.130112Z","shell.execute_reply.started":"2021-06-06T07:00:20.124108Z","shell.execute_reply":"2021-06-06T07:00:20.128873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text_advanced(txt, nlp=nlp):\n    target_size = 5\n    curr_size = 0\n    lemma_sentence = []\n    cleaned_txt = re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n    word_list = list(set(cleaned_txt.split()))\n    word_set = Counter(word_list)\n    sorted_set = sorted(word_set.items(), key=lambda x: x[1], reverse=True)\n    for name, num in sorted_set:\n        name = nlp(name)\n        if name[0].is_stop or name[0].is_digit:\n            continue\n        else:\n            lemma_sentence.append(name[0].lemma_)\n            curr_size += 1\n        if curr_size == target_size:\n            return ' '.join(lemma_sentence)\n    return ' '.join(lemma_sentence)\n#     word_dict = dict()\n    # print(word_set)\n#     for name, num in word_set.items():\n#         name = nlp(name)\n#         if not name[0].is_stop:\n#             name = name[0].lemma_\n#         else:\n#             continue\n#         if word_dict.get(name) is None:\n#             word_dict[name] = num\n#         else:\n#             word_dict[name] += num\n#     sorted_set = sorted(word_dict.items(), key=lambda x: x[1], reverse=True)\n#     return sorted_set","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:23.543959Z","iopub.execute_input":"2021-06-06T07:00:23.54437Z","iopub.status.idle":"2021-06-06T07:00:23.554458Z","shell.execute_reply.started":"2021-06-06T07:00:23.544337Z","shell.execute_reply":"2021-06-06T07:00:23.553087Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Example of clean_text effect","metadata":{}},{"cell_type":"code","source":"# print(clean_text_advanced(\"I am a boy. You are a Girl dataset datasets has have.\"))\n# print(clean_text(\"I am a boy. You are a Girl dataset.\"))","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:30.758544Z","iopub.execute_input":"2021-06-06T07:00:30.758937Z","iopub.status.idle":"2021-06-06T07:00:30.76346Z","shell.execute_reply.started":"2021-06-06T07:00:30.758905Z","shell.execute_reply":"2021-06-06T07:00:30.762373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = [ ]\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:32.789839Z","iopub.execute_input":"2021-06-06T07:00:32.79027Z","iopub.status.idle":"2021-06-06T07:00:32.799243Z","shell.execute_reply.started":"2021-06-06T07:00:32.790233Z","shell.execute_reply":"2021-06-06T07:00:32.798101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %%time\n# tqdm.pandas()\n# train_df['text'] = train_df['Id'].progress_apply(read_append_return)\n# train_df['most_common'] = train_df['text'].progress_apply(clean_text_advanced)\n\n# train_df.to_csv('new_train.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:39.19095Z","iopub.execute_input":"2021-06-06T07:00:39.191357Z","iopub.status.idle":"2021-06-06T07:00:39.196355Z","shell.execute_reply.started":"2021-06-06T07:00:39.191326Z","shell.execute_reply":"2021-06-06T07:00:39.194938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:39.593189Z","iopub.execute_input":"2021-06-06T07:00:39.593524Z","iopub.status.idle":"2021-06-06T07:00:39.608588Z","shell.execute_reply.started":"2021-06-06T07:00:39.593485Z","shell.execute_reply":"2021-06-06T07:00:39.607597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(read_append_return, train_files_path=test_files_path))\nsample_sub['most_common'] = sample_sub['text'].progress_apply(clean_text_advanced)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:43.420523Z","iopub.execute_input":"2021-06-06T07:00:43.420919Z","iopub.status.idle":"2021-06-06T07:00:43.702479Z","shell.execute_reply.started":"2021-06-06T07:00:43.420884Z","shell.execute_reply":"2021-06-06T07:00:43.701084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:51.104502Z","iopub.execute_input":"2021-06-06T07:00:51.104876Z","iopub.status.idle":"2021-06-06T07:00:51.1203Z","shell.execute_reply.started":"2021-06-06T07:00:51.104843Z","shell.execute_reply":"2021-06-06T07:00:51.119019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:00:53.02854Z","iopub.execute_input":"2021-06-06T07:00:53.028912Z","iopub.status.idle":"2021-06-06T07:00:53.035308Z","shell.execute_reply.started":"2021-06-06T07:00:53.028882Z","shell.execute_reply":"2021-06-06T07:00:53.034398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nprint(len(existing_labels))\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample_sub.iterrows()):\n    sample_text = row['text']\n    sample_most_common = row['most_common']\n    row_id = row['Id']\n    # print(train_df['text'] == text_cleaning(sample_text))\n    sample_text = clean_text(sample_text)\n    cleaned_labels = []\n    temp_df = train_df[train_df['most_common'].progress_apply(partial(jaccard, str2=sample_most_common)) > 0.1]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    print(len(set(cleaned_labels)))\n    print(set(cleaned_labels))\n#     for known_label in existing_labels:\n#         if known_label in sample_text.lower():\n#             print(\"matching: {}\".format(known_label))\n#             cleaned_labels.append(clean_text(known_label))\n#     print(set(cleaned_labels))\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:01:38.656218Z","iopub.execute_input":"2021-06-06T07:01:38.656588Z","iopub.status.idle":"2021-06-06T07:01:39.465579Z","shell.execute_reply.started":"2021-06-06T07:01:38.656553Z","shell.execute_reply":"2021-06-06T07:01:39.464607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = lables_list","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:01:44.588151Z","iopub.execute_input":"2021-06-06T07:01:44.588585Z","iopub.status.idle":"2021-06-06T07:01:44.596835Z","shell.execute_reply.started":"2021-06-06T07:01:44.588548Z","shell.execute_reply":"2021-06-06T07:01:44.595663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.set_option(\"display.max_rows\", None, \"display.max_columns\", None)\nsubmission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:01:47.987921Z","iopub.execute_input":"2021-06-06T07:01:47.988317Z","iopub.status.idle":"2021-06-06T07:01:47.998628Z","shell.execute_reply.started":"2021-06-06T07:01:47.988281Z","shell.execute_reply":"2021-06-06T07:01:47.997624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-06T07:01:54.313705Z","iopub.execute_input":"2021-06-06T07:01:54.314108Z","iopub.status.idle":"2021-06-06T07:01:54.320073Z","shell.execute_reply.started":"2021-06-06T07:01:54.314075Z","shell.execute_reply":"2021-06-06T07:01:54.318932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}