{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport pandas as pd\nimport numpy as np\nimport os\nimport re\nimport json\nimport string\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom tqdm.autonotebook import tqdm\nfrom functools import partial\nfrom wordcloud import WordCloud, STOPWORDS\nimport nltk\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\nfrom nltk.probability import FreqDist\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom nltk.corpus import stopwords\nfrom unidecode import unidecode\n\nSTOPWORDS = set(stopwords.words('english'))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train.columns:\n    print(col + \":\" + str(len(train[col].unique())))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample_sub.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def json_to_text(filename, train_files_path=train_files_path, output='text'):\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain['text'] = train['Id'].progress_apply(json_to_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\nsample_sub['text'] = sample_sub['Id'].progress_apply(partial(json_to_text, train_files_path=test_files_path))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub.head(10)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tqdm.pandas()\ntrain['text'] = train['text'].progress_apply(text_cleaning)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Evaluate it using the metric that they use in this dataset\ndef jaccard(str1, str2): \n    a = set(str1.lower().split()) \n    b = set(str2.lower().split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import itertools    \nimport collections\n\nmax=1500\n\nL_word=[]\nfor i in range(max):\n    L_word+=list(train.text[i].split())\nprint(len(L_word))\n\nL_new=[]\nfor x in L_word:\n    if x not in STOPWORDS:\n        if len(x)>2:\n            L_new.append(x)\nprint(len(L_new))\n\nwords=[L_new]\nall_words = list(itertools.chain(*words))\n\ncounts_words = collections.Counter(all_words)\n\ncounts_words.most_common(15)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len=19661\nL_labels=[]\nfor i in range(max_len):\n    L_labels+=list(train.dataset_label[i].split())\nprint(len(L_labels))\n\nL_new_lab=[]\nfor x in L_labels:\n    if x not in STOPWORDS:\n        if len(x)>2:\n            L_new_lab.append(x)\nprint(len(L_new_lab))\n\nlabels=[L_new_lab]\n\ncounts_labels = collections.Counter(list(itertools.chain(*labels)))\n\nlen(counts_labels.most_common())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def extract(L):\n    most=[]\n    for x in L:\n        most.append(x[0])\n    return most\n\nmaxi=2000\n\nmost_frequent=extract(counts_labels.most_common(maxi))+extract(counts_words.most_common(maxi))\nprint(len(counts_labels.most_common(maxi)))\nprint(len(counts_words.most_common(maxi)))\nprint(len(most_frequent))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\n\nL_sub=[]\nfor i in range(len(sample_sub.text)):\n    L_sub+=list((sample_sub.text[i].split()))\nprint(len(L_sub))\n\nL_new_sub=[]\nfor x in L_sub:\n    if x not in STOPWORDS:\n        if len(x)>2:\n            L_new_sub.append(clean_text(x))\nprint(len(L_new_sub))\n\nwordd=[L_new_sub]\nall_sub = list(itertools.chain(*wordd))\n\ncounts_sub = collections.Counter(all_sub)\n\nmost=extract(counts_sub.most_common(maxi))\n\ncounts_sub.most_common(15)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\nexisting_labels = set(temp_1 + temp_2 + temp_3)\n\ndef predict(sample_sub):\n    id_list = []\n    lables_list = []\n    for index, row in tqdm(sample_sub.iterrows()):\n        sample_text = row['text']\n        row_id = row['Id']\n        temp_df = train[train['text'] == text_cleaning(sample_text)]\n        cleaned_labels = temp_df['cleaned_label'].to_list()\n        for known_label in existing_labels:\n            if known_label in sample_text.lower():\n                if known_label not in STOPWORDS:\n                    if len(known_label)>1:\n                        if known_label in most:\n                            cleaned_labels.append(clean_text(known_label))\n        cleaned_labels = [clean_text(x) for x in cleaned_labels]\n        cleaned_labels = set(cleaned_labels)\n        lables_list.append('|'.join(cleaned_labels))\n        id_list.append(row_id)\n    return (id_list,lables_list)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Z=predict(sample_sub)\nsubmission = pd.DataFrame()\nsubmission['Id'] = Z[0]\nsubmission['PredictionString'] = Z[1]\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X=predict(train.head(1000))[1]\nscore=[]\n\nfor i in range(len(X)):\n    score.append(jaccard(X[i], train.dataset_label[i]))\nprint(f'Score is : {np.mean(score)}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(r'submission.csv',index=None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}