{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"This notebook simply uses matching if a dataset is in the document, it \"predicts\" the title.  It uses the 180 dataset list from the train data and adds some hand curated govt dataset titles.","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\n\nsample_sub","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_data_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\n\ntrain_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_json_pub(filename, train_data_path=train_data_path, output='text'):\n    json_path = os.path.join(train_data_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9]+', ' ', str(text).lower()).strip()\n    text = re.sub(' +', ' ', text)\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    return text\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#temp_1 = [x.lower() for x in train_df['dataset_label'].unique()]\n#temp_2 = [x.lower() for x in train_df['dataset_title'].unique()]\n#temp_3 = [x.lower() for x in train_df['cleaned_label'].unique()]\n\n#existing_labels = set(temp_1 + temp_2 + temp_3)\n#df2 = pd.DataFrame(list(existing_labels))\n\n#df2.rename(columns = {0:'title'}, inplace = True)\n\n#df2['title']=df2['title'].str.replace(\",\",\"\")\n#df2['title']=df2['title'].str.replace(\"-\",\" \")\n\n#df2['length'] = df2['title'].str.len()\n#df2.sort_values('title', ascending=True, inplace=True)\n\n#df2.head(50)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df2.to_csv('datasets.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2=pd.read_csv('../input/bigger-govt-dataset-list/data_set_800.csv')\n#df2['title']=df2['title'].str.replace(\" +\",\" \")\n#df2.sort_values('title', ascending=True, inplace=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df2.head(50)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# random sample for testing\ntrain_sample=train_df.sample(n = 10)\n\ntrain_sample","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = time.time()\n\n\n#### remove >.5 jaccard matches from predicitons\ndef jaccard_similarity(s1, s2):\n    l1 = s1.split(\" \")\n    l2 = s2.split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\n#############################\n#path=train_data_path\npath=test_data_path\n\n#for training use train_sample\n\n#for submission use sample_sub\n\n#############\n\ncolumn_names = [\"Id\", \"PredictionString\"]\n\nsubmission = pd.DataFrame(columns = column_names)\n\nto_append=[]\nfor index, row in sample_sub.iterrows():\n    to_append=[row['Id'],'']\n    large_string = str(read_json_pub(row['Id'],path))\n    clean_string=text_cleaning(large_string)\n    for index, row2 in df2.iterrows():\n        query_string = str(row2['title'])\n        if query_string in clean_string:\n            if to_append[1]!='' and clean_text(query_string) not in to_append[1]:\n                to_append[1]=to_append[1]+'|'+clean_text(query_string)\n            if to_append[1]=='':\n                to_append[1]=clean_text(query_string)\n     \n    ###### remove similar jaccard\n    #got_label=to_append[1].split('|')\n    #filtered=[]\n    #filtered_labels = ''\n    #for label in sorted(got_label, key=len):\n        #label = clean_text(label)\n        #if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 1.0 for got_label in filtered):\n            #filtered.append(label)\n            #if filtered_labels!='':\n                #filtered_labels=filtered_labels+'|'+label\n            #if filtered_labels=='':\n                #filtered_labels=label\n    #to_append[1] = filtered_labels         \n    #print ('################')\n    #print (to_append)\n    #print (large_string)\n    #print ('################')\n    ###### remove similar jaccard\n    df_length = len(submission)\n    submission.loc[df_length] = to_append\nsubmission.to_csv('submission.csv', index = False)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}