{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport os\nimport json\nimport re\nimport warnings\nwarnings.filterwarnings(\"ignore\", 'This pattern has match groups')\n\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt\n\ndef text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n\npath = '../input/coleridgeinitiative-show-us-the-data'\n\nstopwords = ['ourselves', 'hers','the', 'between', 'yourself', 'but', 'again','of', 'there', 'about',\n             'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some',\n             'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is',\n             's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below',\n             'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more',\n             'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to',\n             'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and',\n             'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what',\n             'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has',\n             'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom',\n             't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', \n             'was', 'here', 'than']","metadata":{"execution":{"iopub.status.busy":"2021-06-12T09:14:22.664278Z","iopub.execute_input":"2021-06-12T09:14:22.664846Z","iopub.status.idle":"2021-06-12T09:14:22.743549Z","shell.execute_reply.started":"2021-06-12T09:14:22.664725Z","shell.execute_reply":"2021-06-12T09:14:22.742211Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = pd.read_csv(path + '/train.csv')  # shape (19661, 5)\nfor col in df_train.columns:\n    print(f\"{col}: {len(df_train[col].unique())}\")\n\n# Id: 14316\n# pub_title: 14271\n# dataset_title: 45\n# dataset_label: 130\n# cleaned_label: 130\n\n\ndf_input = pd.DataFrame(columns=['id', 'section_title', 'text', 'data_label'])\nfor ID in df_train['Id'].unique():\n    df = pd.read_json(path + '/train/{}.json'.format(ID))\n\n    for data_label in df_train[df_train['Id'] == ID]['dataset_label'].values:\n        new_df = df[df['text'].str.contains(data_label)].copy(deep=True)\n        new_df['data_label'] = data_label\n        new_df['id'] = ID\n        new_df.reset_index(inplace=True, drop=True)\n        df_input = pd.concat([df_input, new_df], ignore_index=True, sort=False)\n        df_input.reset_index(inplace=True, drop=True)\n\n\n# words = df_input['data_label'].values  # numpy.ndarray of String\n\n\ndf_test = pd.read_csv(path + '/sample_submission.csv')\ndf_test_input = pd.DataFrame(columns=['id', 'section_title', 'text'])\nfor ID in df_test['Id'].values:\n    df = pd.read_json(path + '/test/{}.json'.format(ID))\n    \n    df['id'] = ID\n    df.reset_index(inplace=True, drop=True)\n    df_test_input = pd.concat([df_test_input, df], ignore_index=True, sort=False)\n    df_test_input.reset_index(inplace=True,drop=True)\n\ndf_test_input['length'] = df_test_input.text.str.len()\ndf_test_input = df_test_input[df_test_input.length > 0]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:21:25.176855Z","iopub.execute_input":"2021-06-11T09:21:25.177213Z","iopub.status.idle":"2021-06-11T09:26:05.032754Z","shell.execute_reply.started":"2021-06-11T09:21:25.177182Z","shell.execute_reply":"2021-06-11T09:26:05.031633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(s1, s2):\n    l1 = s1.split('|')\n    l2 = s2.split('|')    \n    intersection = len(set(l1).intersection(l2))\n    union = len(l1) + len(l2) - intersection\n    return float(intersection) / union","metadata":{"execution":{"iopub.status.busy":"2021-06-11T09:28:06.431629Z","iopub.execute_input":"2021-06-11T09:28:06.432017Z","iopub.status.idle":"2021-06-11T09:28:07.122944Z","shell.execute_reply.started":"2021-06-11T09:28:06.431988Z","shell.execute_reply":"2021-06-11T09:28:07.122212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_labels = set()\n\nfor label_1, label_2, label_3 in df_train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n\nlen(all_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:47:04.160224Z","iopub.execute_input":"2021-06-11T07:47:04.160699Z","iopub.status.idle":"2021-06-11T07:47:04.20428Z","shell.execute_reply.started":"2021-06-11T07:47:04.160669Z","shell.execute_reply":"2021-06-11T07:47:04.203065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# datasets_titles = [str(x).lower() for x in df_input['data_label'].unique()]\n\nlabels = []\nfor index in df_test['Id']:\n    tmp_df = df_test_input[df_test_input['id'] == index]\n    match_text = tmp_df.text.str.cat(sep='\\n').lower() + tmp_df.section_title.str.cat(sep='\\n').lower()\n\n    label = [text_cleaning(dataset_label) for dataset_label in all_labels if dataset_label in match_text]\n    labels.append('|'.join(label))\n\nsubmission_df = pd.read_csv(path + '/sample_submission.csv', index_col=0)\nsubmission_df['PredictionString'] = labels\nsubmission_df.to_csv('./submission.csv')\nprint('submission complete')","metadata":{"execution":{"iopub.status.busy":"2021-06-11T07:55:51.413527Z","iopub.execute_input":"2021-06-11T07:55:51.414043Z","iopub.status.idle":"2021-06-11T07:55:51.467324Z","shell.execute_reply.started":"2021-06-11T07:55:51.414012Z","shell.execute_reply":"2021-06-11T07:55:51.466687Z"},"trusted":true},"execution_count":null,"outputs":[]}]}