{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"<img src= \"https://overdeck.org/wp-content/uploads/CI_logo.jpg\" alt =\"Titanic\" style='width: 700px;'>\n","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n#import os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport glob\nfrom collections import defaultdict\nfrom textblob import TextBlob\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport matplotlib.pyplot as plt\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport seaborn as sns\n\nimport nltk\nimport spacy\nnlp = spacy.load('en_core_web_lg', disable=['parser', 'ner'])\nnlp.max_length = 4000000\nfrom nltk.probability import FreqDist\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom tqdm.autonotebook import tqdm\nimport string\n\n%matplotlib inline\n#os.listdir('/kaggle/input/coleridgeinitiative-show-us-the-data/')","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:52:00.422188Z","iopub.execute_input":"2021-06-17T09:52:00.422648Z","iopub.status.idle":"2021-06-17T09:52:09.062278Z","shell.execute_reply.started":"2021-06-17T09:52:00.422601Z","shell.execute_reply":"2021-06-17T09:52:09.061326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Data Paths\ntrain = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:52:23.458459Z","iopub.execute_input":"2021-06-17T09:52:23.458823Z","iopub.status.idle":"2021-06-17T09:52:23.625867Z","shell.execute_reply.started":"2021-06-17T09:52:23.458789Z","shell.execute_reply":"2021-06-17T09:52:23.62484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:52:24.911162Z","iopub.execute_input":"2021-06-17T09:52:24.911501Z","iopub.status.idle":"2021-06-17T09:52:24.936216Z","shell.execute_reply.started":"2021-06-17T09:52:24.911472Z","shell.execute_reply":"2021-06-17T09:52:24.934995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:55:46.418615Z","iopub.execute_input":"2021-06-17T09:55:46.418974Z","iopub.status.idle":"2021-06-17T09:55:46.455182Z","shell.execute_reply.started":"2021-06-17T09:55:46.418945Z","shell.execute_reply":"2021-06-17T09:55:46.454175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Common Values\n[print(f\"{col}:{len(train[col].unique())}\") for col in train.columns]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:52:35.439692Z","iopub.execute_input":"2021-06-17T09:52:35.440106Z","iopub.status.idle":"2021-06-17T09:52:35.469828Z","shell.execute_reply.started":"2021-06-17T09:52:35.440064Z","shell.execute_reply":"2021-06-17T09:52:35.468928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Function to get data from JSON FILES\ndef read_append_return(filename, train_path=train_path, output='text'):\n    json_path = os.path.join(train_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:53:18.754236Z","iopub.execute_input":"2021-06-17T09:53:18.754595Z","iopub.status.idle":"2021-06-17T09:53:18.763187Z","shell.execute_reply.started":"2021-06-17T09:53:18.754562Z","shell.execute_reply":"2021-06-17T09:53:18.762041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas() \ntrain['text'] = train['Id'].progress_apply(read_append_return)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:53:59.065999Z","iopub.execute_input":"2021-06-17T09:53:59.066506Z","iopub.status.idle":"2021-06-17T09:54:57.094257Z","shell.execute_reply.started":"2021-06-17T09:53:59.066473Z","shell.execute_reply":"2021-06-17T09:54:57.09303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:54:59.908605Z","iopub.execute_input":"2021-06-17T09:54:59.908959Z","iopub.status.idle":"2021-06-17T09:54:59.923756Z","shell.execute_reply.started":"2021-06-17T09:54:59.90893Z","shell.execute_reply":"2021-06-17T09:54:59.922629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\nsample['text'] = sample['Id'].progress_apply(partial(read_append_return, train_path=test_path))","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:56:32.874112Z","iopub.execute_input":"2021-06-17T09:56:32.874443Z","iopub.status.idle":"2021-06-17T09:56:32.94865Z","shell.execute_reply.started":"2021-06-17T09:56:32.874412Z","shell.execute_reply":"2021-06-17T09:56:32.947617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:56:36.049912Z","iopub.execute_input":"2021-06-17T09:56:36.050266Z","iopub.status.idle":"2021-06-17T09:56:36.065465Z","shell.execute_reply.started":"2021-06-17T09:56:36.050238Z","shell.execute_reply":"2021-06-17T09:56:36.06437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cleaning Function for data\ndef text_cleaning(text):\n    text = ''.join([k for k in text if k not in string.punctuation])\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    \n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:56:54.851229Z","iopub.execute_input":"2021-06-17T09:56:54.851568Z","iopub.status.idle":"2021-06-17T09:56:54.857644Z","shell.execute_reply.started":"2021-06-17T09:56:54.851539Z","shell.execute_reply":"2021-06-17T09:56:54.856359Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntqdm.pandas()\ntrain['text'] = train['text'].progress_apply(text_cleaning)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T09:56:56.786027Z","iopub.execute_input":"2021-06-17T09:56:56.78637Z","iopub.status.idle":"2021-06-17T10:00:57.574961Z","shell.execute_reply.started":"2021-06-17T09:56:56.78634Z","shell.execute_reply":"2021-06-17T10:00:57.573951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:00:57.57705Z","iopub.execute_input":"2021-06-17T10:00:57.577468Z","iopub.status.idle":"2021-06-17T10:00:57.595195Z","shell.execute_reply.started":"2021-06-17T10:00:57.57742Z","shell.execute_reply":"2021-06-17T10:00:57.594167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"words =list( train['cleaned_label'].values)\nimport numpy as np\nwordsnp = np.array(words)\nnp.unique(wordsnp)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:01:14.297003Z","iopub.execute_input":"2021-06-17T10:01:14.297361Z","iopub.status.idle":"2021-06-17T10:01:14.340856Z","shell.execute_reply.started":"2021-06-17T10:01:14.297329Z","shell.execute_reply":"2021-06-17T10:01:14.339932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stopwords=['ourselves', 'hers','the','of','and','in', 'between', 'yourself', 'but', 'again','of', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'through', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']\nsplit_words=[]\nfor word in words:\n    lo_w=[]\n    list_of_words=str(word).split()\n    for w in list_of_words:\n        if w not in stopwords:\n            lo_w.append(w)\n    split_words.append(lo_w)\nallwords = []\nfor wordlist in split_words:\n    allwords += wordlist\n\nprint(allwords[1:10])","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:01:33.594405Z","iopub.execute_input":"2021-06-17T10:01:33.594794Z","iopub.status.idle":"2021-06-17T10:01:33.920124Z","shell.execute_reply.started":"2021-06-17T10:01:33.594755Z","shell.execute_reply":"2021-06-17T10:01:33.919026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostcommon = FreqDist(allwords).most_common(100)\nwordcloud = WordCloud(width=1500, height=750, background_color='white', stopwords=STOPWORDS).generate(str(mostcommon))\nfig = plt.figure(figsize=(30,10), facecolor='white')\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis('off')\nplt.title('100 Common Words', fontsize=40)\nplt.tight_layout(pad=0)\nplt.show()\n\nmostcommon_small = FreqDist(allwords).most_common(25)\nx, y = zip(*mostcommon_small)\nplt.figure(figsize=(50,30))\nplt.margins(0.02)\nplt.bar(x, y)\nplt.xlabel('Words', fontsize=50)\nplt.ylabel('Frequency', fontsize=50)\nplt.yticks(fontsize=40)\nplt.xticks(rotation=60, fontsize=40)\nplt.tight_layout(pad=0)\nplt.title('25 Common Words', fontsize=60)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:01:41.176038Z","iopub.execute_input":"2021-06-17T10:01:41.176518Z","iopub.status.idle":"2021-06-17T10:01:44.610335Z","shell.execute_reply.started":"2021-06-17T10:01:41.176478Z","shell.execute_reply":"2021-06-17T10:01:44.609274Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def prepare_text(text, nlp=nlp):\n    doc = nlp(text)\n    lemma_list = [token.lemma_ for token in doc if not token.is_stop]\n    lemmatized_sentence = ' '.join(lemma_list)\n    \n    return lemmatized_sentence","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:03:19.453488Z","iopub.execute_input":"2021-06-17T10:03:19.454485Z","iopub.status.idle":"2021-06-17T10:03:19.463435Z","shell.execute_reply.started":"2021-06-17T10:03:19.454417Z","shell.execute_reply":"2021-06-17T10:03:19.462292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:03:22.685468Z","iopub.execute_input":"2021-06-17T10:03:22.685842Z","iopub.status.idle":"2021-06-17T10:03:22.690895Z","shell.execute_reply.started":"2021-06-17T10:03:22.68581Z","shell.execute_reply":"2021-06-17T10:03:22.689597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_1 = [x.lower() for x in train['dataset_label'].unique()]\ntemp_2 = [x.lower() for x in train['dataset_title'].unique()]\ntemp_3 = [x.lower() for x in train['cleaned_label'].unique()]\n\nexisting_labels = set(temp_1 + temp_2 + temp_3)\nid_list = []\nlables_list = []\nfor index, row in tqdm(sample.iterrows()):\n    sample_text = row['text']\n    row_id = row['Id']\n    temp_df = train[train['text'] == text_cleaning(sample_text)]\n    cleaned_labels = temp_df['cleaned_label'].to_list()\n    for known_label in existing_labels:\n        if known_label in sample_text.lower():\n            cleaned_labels.append(clean_text(known_label))\n    cleaned_labels = [clean_text(x) for x in cleaned_labels]\n    cleaned_labels = set(cleaned_labels)\n    lables_list.append('|'.join(cleaned_labels))\n    id_list.append(row_id)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:06:00.41012Z","iopub.execute_input":"2021-06-17T10:06:00.41055Z","iopub.status.idle":"2021-06-17T10:06:00.853914Z","shell.execute_reply.started":"2021-06-17T10:06:00.410472Z","shell.execute_reply":"2021-06-17T10:06:00.852927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame()\nsubmission['Id'] = id_list\nsubmission['PredictionString'] = lables_list","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:06:00.926464Z","iopub.execute_input":"2021-06-17T10:06:00.92684Z","iopub.status.idle":"2021-06-17T10:06:00.936373Z","shell.execute_reply.started":"2021-06-17T10:06:00.926809Z","shell.execute_reply":"2021-06-17T10:06:00.935124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['PredictionString'][3]","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:07:18.465802Z","iopub.execute_input":"2021-06-17T10:07:18.466308Z","iopub.status.idle":"2021-06-17T10:07:18.471144Z","shell.execute_reply.started":"2021-06-17T10:07:18.466276Z","shell.execute_reply":"2021-06-17T10:07:18.470291Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-17T10:08:40.814833Z","iopub.execute_input":"2021-06-17T10:08:40.815415Z","iopub.status.idle":"2021-06-17T10:08:40.827181Z","shell.execute_reply.started":"2021-06-17T10:08:40.81538Z","shell.execute_reply":"2021-06-17T10:08:40.826106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}