{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nimport os\nimport re\n\nimport numpy\nimport pandas\nfrom fuzzywuzzy import fuzz\nfrom sklearn.model_selection import train_test_split\nfrom nltk.corpus import stopwords","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"directory = r\"../input/coleridgeinitiative-show-us-the-data/\"\n\ntrain_csv = pandas.read_csv(directory + \"/train.csv\")\nsample_submission = pandas.read_csv(directory + \"/sample_submission.csv\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Retrieve json data and clean it","metadata":{}},{"cell_type":"code","source":"def retrieve_text(filename, type):\n    json_path = os.path.join(directory, type, filename + \".json\")\n\n    section_title = []\n    contents = []\n    with open(json_path, mode='r') as recurse:\n        json_contents = json.load(recurse)\n\n        for data in json_contents:\n            contents.append(data.get('section_title'))\n            contents.append(data.get('text'))\n\n        # section_title = data_cleaning(\" \".join(section_title))\n        contents = data_cleaning(\" \".join(contents))\n\n    return contents","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_cleaning(text):\n    text = re.sub('[^A-Za-z0-9]+', \" \", text)\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    cleaned_text = emoji_pattern.sub(r'', text)\n\n    return cleaned_text.lower()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_json():\n    train_csv['json-content'] = train_csv['Id'].apply(retrieve_text, args=('train',))\n    test_set['json-content'] = sample_submission['Id'].apply(retrieve_text, args=('test',))\n    # train_csv['acronym'] = train_csv['dataset_title'].progress_apply(create_patterns)\n    # train_csv['fuzzy-ratio'] = train_csv.progress_apply(get_fuzzy_score, axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set = pandas.DataFrame()\ntest_set['Id'] = sample_submission['Id']\nload_json()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess data - retrive useful info","metadata":{}},{"cell_type":"code","source":"def preprocess_data(dataframe):\n    unique_dataset_titles = dataframe['dataset_title'].unique()\n\n    for dataset_title in unique_dataset_titles:\n        try:\n            if '(' in str(dataset_title):\n                tmp_title = str(dataset_title).split(\" \")\n                \n                tmp_title_without_braces = str(dataset_title).replace(\"(\", \"\")\n                tmp_title_without_braces = tmp_title_without_braces.replace(\")\", \"\").lower()\n                tmp_title_without_braces = re.sub('[^A-Za-z]+', \" \", tmp_title_without_braces)\n                    \n                for word in tmp_title:\n                    if '(' in word:\n                        acronyms_dict[str(word[1: -1]).lower()] = tmp_title_without_braces\n\n            else:\n                text = re.sub('[^A-Za-z]+', \" \", str(dataset_title))\n                clean_text = text.lower().split()\n                clean_text = [clean_word for clean_word in clean_text if not clean_word in set(stop_words)]\n\n                acronym_text = []\n                for word in clean_text:\n                    acronym_text.append(word[0: 1])\n\n                acronyms_dict[\"\".join(acronym_text)] = str(dataset_title).lower()\n\n            tmp_title = str(dataset_title)\n            tmp_title_without_braces = str(dataset_title).lower().split(\" \")\n            tmp_title = re.sub('[^A-Za-z0-9]+', \" \", tmp_title).lower()\n            tmp_title_without_braces = [word for word in tmp_title_without_braces if not '(' in word]\n            tmp_title_without_braces = re.sub('[^A-Za-z0-9]+', \" \", str(tmp_title_without_braces)).lower()\n\n            titles_prior1.add(tmp_title.strip())\n            \n            if tmp_title_without_braces.strip() not in titles_prior1:\n                titles_prior2.add(tmp_title_without_braces.strip())\n                titles_dict[tmp_title_without_braces.strip()] = tmp_title.strip()\n\n        except:\n            print(\"exception occurred for title: \", dataset_title)\n            continue\n\n    return acronyms_dict, titles_dict, titles_prior1, titles_prior2","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stop_words = stopwords.words('english')\nacronyms = set()\ntitles_prior1 = set()\ntitles_prior2 = set()\nacronyms_dict = {}\ntitles_dict = {}\nacronyms_dict, titles_dict, titles_prior1, titles_prior2 = preprocess_data(train_csv)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"acronyms_dict","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"titles_prior1 = list(sorted(titles_prior1, key=len, reverse=True))\ntitles_prior2 = list(sorted(titles_prior2, key=len, reverse=True))\nunique_cleaned_matches = train_csv['cleaned_label'].unique()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(titles_prior1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Predict results and save it to submission file","metadata":{}},{"cell_type":"code","source":"acronyms = acronyms_dict.keys()\nmatch_out = []\nfor json_data in test_set['json-content']:\n    match = ''\n    tmp_set = set()\n\n    for word in json_data.split():\n        tmp_set.add(word)\n    \n    for clean_text in unique_cleaned_matches:\n        if clean_text in str(json_data) and clean_text not in match:\n            match += ('|' + clean_text if len(match) > 0 else clean_text)\n            \n    for query_prior1 in titles_prior1:\n        query_text = str(query_prior1).lower()\n\n        if query_text in str(json_data) and query_text not in match:\n            match += ('|' + query_text if len(match) > 0 else query_text)\n\n    for query_prior2 in titles_prior2:\n        query_text = str(query_prior2).lower()\n\n        if query_text in str(json_data) and query_text not in match:\n            match += ('|' + query_text if len(match) > 0 else query_text)\n\n    for query_text in acronyms:\n        if len(query_text) > 3 and query_text in tmp_set and query_text not in match:\n            match += ('|' + query_text if len(match) > 0 else query_text)\n\n    match_out.append(match)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(match_out)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"result = pandas.DataFrame()\nresult['Id'] = test_set['Id']\nresult['PredictionString'] = match_out\nresult.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# result = pandas.DataFrame()\n# result['Id'] = train_csv['Id']\n# result['title'] = train_csv['dataset_title']\n# result['clean'] = train_csv['cleaned_label']\n# result['PredictionString'] = match_out\n# result.to_csv('submission.csv', index=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}