{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in list(filenames)[:25]:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-18T22:09:11.909255Z","iopub.execute_input":"2021-06-18T22:09:11.909683Z","iopub.status.idle":"2021-06-18T22:09:23.157237Z","shell.execute_reply.started":"2021-06-18T22:09:11.909641Z","shell.execute_reply":"2021-06-18T22:09:23.155720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import the dataset","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.158833Z","iopub.execute_input":"2021-06-18T22:09:23.159236Z","iopub.status.idle":"2021-06-18T22:09:23.165665Z","shell.execute_reply.started":"2021-06-18T22:09:23.159177Z","shell.execute_reply":"2021-06-18T22:09:23.163981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.168300Z","iopub.execute_input":"2021-06-18T22:09:23.168777Z","iopub.status.idle":"2021-06-18T22:09:23.181171Z","shell.execute_reply.started":"2021-06-18T22:09:23.168732Z","shell.execute_reply":"2021-06-18T22:09:23.179235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/coleridgeinitiative-show-us-the-data/test/8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60.json') as sample:\n    sample_json = json.load(sample)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.183503Z","iopub.execute_input":"2021-06-18T22:09:23.183970Z","iopub.status.idle":"2021-06-18T22:09:23.205097Z","shell.execute_reply.started":"2021-06-18T22:09:23.183924Z","shell.execute_reply":"2021-06-18T22:09:23.204221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_json[:3]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.206425Z","iopub.execute_input":"2021-06-18T22:09:23.206910Z","iopub.status.idle":"2021-06-18T22:09:23.216642Z","shell.execute_reply.started":"2021-06-18T22:09:23.206867Z","shell.execute_reply":"2021-06-18T22:09:23.215284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [{}, {}, {}]\nfor i in sample_json:\n    print(i.keys())","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.218823Z","iopub.execute_input":"2021-06-18T22:09:23.220295Z","iopub.status.idle":"2021-06-18T22:09:23.234429Z","shell.execute_reply.started":"2021-06-18T22:09:23.220249Z","shell.execute_reply":"2021-06-18T22:09:23.233402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## ","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.235939Z","iopub.execute_input":"2021-06-18T22:09:23.236372Z","iopub.status.idle":"2021-06-18T22:09:23.246049Z","shell.execute_reply.started":"2021-06-18T22:09:23.236330Z","shell.execute_reply":"2021-06-18T22:09:23.244543Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"## Preprocessing","metadata":{}},{"cell_type":"code","source":"import nltk\nfrom typing import List\nfrom nltk.tokenize import word_tokenize","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.248865Z","iopub.execute_input":"2021-06-18T22:09:23.249499Z","iopub.status.idle":"2021-06-18T22:09:23.984045Z","shell.execute_reply.started":"2021-06-18T22:09:23.249452Z","shell.execute_reply":"2021-06-18T22:09:23.982911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nltk.download('stopwords')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:23.986053Z","iopub.execute_input":"2021-06-18T22:09:23.986395Z","iopub.status.idle":"2021-06-18T22:09:44.031339Z","shell.execute_reply.started":"2021-06-18T22:09:23.986358Z","shell.execute_reply":"2021-06-18T22:09:44.029977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords\nenglish_stopwords = stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.032951Z","iopub.execute_input":"2021-06-18T22:09:44.033334Z","iopub.status.idle":"2021-06-18T22:09:44.047697Z","shell.execute_reply.started":"2021-06-18T22:09:44.033298Z","shell.execute_reply":"2021-06-18T22:09:44.045893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_stopwords(input_text: str, stopwords: List[str]) -> str:\n    \"\"\"Given an input text, removes all stopwords and punctuation marks\"\"\"\n    text_tokens = word_tokenize(input_text)\n    return [word.lower() for word in text_tokens if not word.lower() in stopwords and word.lower().isalnum()]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.049703Z","iopub.execute_input":"2021-06-18T22:09:44.050273Z","iopub.status.idle":"2021-06-18T22:09:44.058036Z","shell.execute_reply.started":"2021-06-18T22:09:44.050223Z","shell.execute_reply":"2021-06-18T22:09:44.056374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"remove_stopwords(sample_json[0]['text'], english_stopwords)[:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.060036Z","iopub.execute_input":"2021-06-18T22:09:44.060461Z","iopub.status.idle":"2021-06-18T22:09:44.099123Z","shell.execute_reply.started":"2021-06-18T22:09:44.060422Z","shell.execute_reply":"2021-06-18T22:09:44.097673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.100674Z","iopub.execute_input":"2021-06-18T22:09:44.101023Z","iopub.status.idle":"2021-06-18T22:09:44.105998Z","shell.execute_reply.started":"2021-06-18T22:09:44.100987Z","shell.execute_reply":"2021-06-18T22:09:44.104972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_text = remove_stopwords(sample_json[0]['text'], english_stopwords)\n\nvectorizer = CountVectorizer()","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.107419Z","iopub.execute_input":"2021-06-18T22:09:44.107762Z","iopub.status.idle":"2021-06-18T22:09:44.128426Z","shell.execute_reply.started":"2021-06-18T22:09:44.107726Z","shell.execute_reply":"2021-06-18T22:09:44.127514Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus = [\" \".join(remove_stopwords(text_body['text'] ,english_stopwords)) for text_body in sample_json]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.129752Z","iopub.execute_input":"2021-06-18T22:09:44.130327Z","iopub.status.idle":"2021-06-18T22:09:44.260565Z","shell.execute_reply.started":"2021-06-18T22:09:44.130288Z","shell.execute_reply":"2021-06-18T22:09:44.259751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus[0][:100]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.261863Z","iopub.execute_input":"2021-06-18T22:09:44.262427Z","iopub.status.idle":"2021-06-18T22:09:44.268548Z","shell.execute_reply.started":"2021-06-18T22:09:44.262390Z","shell.execute_reply":"2021-06-18T22:09:44.267399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = vectorizer.fit_transform(corpus)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.269782Z","iopub.execute_input":"2021-06-18T22:09:44.270107Z","iopub.status.idle":"2021-06-18T22:09:44.300698Z","shell.execute_reply.started":"2021-06-18T22:09:44.270075Z","shell.execute_reply":"2021-06-18T22:09:44.297181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.304762Z","iopub.execute_input":"2021-06-18T22:09:44.305505Z","iopub.status.idle":"2021-06-18T22:09:44.310913Z","shell.execute_reply.started":"2021-06-18T22:09:44.305442Z","shell.execute_reply":"2021-06-18T22:09:44.309692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(X.toarray(), columns=vectorizer.get_feature_names())","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.312753Z","iopub.execute_input":"2021-06-18T22:09:44.313213Z","iopub.status.idle":"2021-06-18T22:09:44.378846Z","shell.execute_reply.started":"2021-06-18T22:09:44.313143Z","shell.execute_reply":"2021-06-18T22:09:44.377428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# TF-IDF Vectorizer","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nimport re","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.380730Z","iopub.execute_input":"2021-06-18T22:09:44.381483Z","iopub.status.idle":"2021-06-18T22:09:44.387767Z","shell.execute_reply.started":"2021-06-18T22:09:44.381422Z","shell.execute_reply":"2021-06-18T22:09:44.385370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_string(filename) -> str:\n    ...\n#     Load file from memory\n    with open(filename, 'r') as f:\n        doc_json = json.load(f)\n    body = ' '.join([dictionary['text'] for dictionary in doc_json])\n    return ' '.join([word for word in body.split(' ') if (word.isalpha() and word.isascii())])","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.389840Z","iopub.execute_input":"2021-06-18T22:09:44.390381Z","iopub.status.idle":"2021-06-18T22:09:44.414540Z","shell.execute_reply.started":"2021-06-18T22:09:44.390321Z","shell.execute_reply":"2021-06-18T22:09:44.413074Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidf_vectorizer = TfidfVectorizer(stop_words=english_stopwords, max_features=5000)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:09:44.416260Z","iopub.execute_input":"2021-06-18T22:09:44.416905Z","iopub.status.idle":"2021-06-18T22:09:44.435290Z","shell.execute_reply.started":"2021-06-18T22:09:44.416862Z","shell.execute_reply":"2021-06-18T22:09:44.434078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_files = []\nfor dirname, _, filenames in os.walk('/kaggle/input/coleridgeinitiative-show-us-the-data/train/'):\n    for filename in filenames:\n        if filename.endswith('.json'):\n            train_files += [os.path.join(dirname, filename)]\n\n# train_files = train_files[:1000]\n            \ndoc_strings = [get_string(file_path) for file_path in tqdm(train_files)]\n    \n    \nX = tfidf_vectorizer.fit_transform(doc_strings)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:22:31.648234Z","iopub.execute_input":"2021-06-18T22:22:31.648655Z","iopub.status.idle":"2021-06-18T22:25:43.869847Z","shell.execute_reply.started":"2021-06-18T22:22:31.648620Z","shell.execute_reply":"2021-06-18T22:25:43.868676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfidf_df = pd.DataFrame(X.toarray(), columns=tfidf_vectorizer.get_feature_names())\ntfidf_df","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:14:51.503839Z","iopub.execute_input":"2021-06-18T22:14:51.504451Z","iopub.status.idle":"2021-06-18T22:14:51.619484Z","shell.execute_reply.started":"2021-06-18T22:14:51.504407Z","shell.execute_reply":"2021-06-18T22:14:51.618338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def calculate_similarity(X, vectorizor, query, top_k=20):\n    \"\"\" Vectorizes the `query` via `vectorizor` and calculates the cosine similarity of\n    the `query` and `X` (all the documents) and returns the `top_k` similar documents.\"\"\"\n\n    # Vectorize the query to the same length as documents\n    query_vec = vectorizor.transform(query)\n    # Compute the cosine similarity between query_vec and all the documents\n    cosine_similarities = cosine_similarity(X,query_vec).flatten()\n    # Sort the similar documents from the most similar to less similar and return the indices\n    most_similar_doc_indices = np.argsort(cosine_similarities, axis=0)[:-top_k-1:-1]\n    return (most_similar_doc_indices, cosine_similarities)\n\ndef show_similar_documents(df, cosine_similarities, similar_doc_indices, query):\n    \"\"\" Prints the most similar documents using indices in the `similar_doc_indices` vector.\"\"\"\n    counter = 1\n    for index in similar_doc_indices:\n        print('Top-{}, Similarity = {}'.format(counter, cosine_similarities[index]))\n        filename = train_files[index].split('/')[-1].split('.')[0]\n        print(f\"Filename: {filename}\")\n        cleaned_label = list(train[train['Id'] == filename]['cleaned_label'])\n        print(f\"cleaned_label: {cleaned_label}\")\n        print('body: {}, '.format(df[index])[:100])\n        print()\n        counter += 1\n        if bool([c for c in cleaned_label if c == query])","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:39:52.683791Z","iopub.execute_input":"2021-06-18T22:39:52.684310Z","iopub.status.idle":"2021-06-18T22:39:52.694092Z","shell.execute_reply.started":"2021-06-18T22:39:52.684265Z","shell.execute_reply":"2021-06-18T22:39:52.693002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# query = ['national science foundation survey of industrial research and development']\n# sim_vecs, cosine_similarities = calculate_similarity(X, tfidf_vectorizer, query)\n\nfor query in unique_dataset_labels:\n    sim_vecs, cosine_similarities = calculate_similarity(X, tfidf_vectorizer, query)\n    show_similar_documents(doc_strings, cosine_similarities, sim_vecs, query)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:39:53.166096Z","iopub.execute_input":"2021-06-18T22:39:53.166725Z","iopub.status.idle":"2021-06-18T22:39:53.349050Z","shell.execute_reply.started":"2021-06-18T22:39:53.166688Z","shell.execute_reply":"2021-06-18T22:39:53.347921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:33:26.274458Z","iopub.execute_input":"2021-06-18T22:33:26.275030Z","iopub.status.idle":"2021-06-18T22:33:26.307166Z","shell.execute_reply.started":"2021-06-18T22:33:26.274975Z","shell.execute_reply":"2021-06-18T22:33:26.305560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train 130 different classifiers\nOne for each dataset label","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:13:36.455299Z","iopub.execute_input":"2021-06-18T22:13:36.455762Z","iopub.status.idle":"2021-06-18T22:13:36.628595Z","shell.execute_reply.started":"2021-06-18T22:13:36.455725Z","shell.execute_reply":"2021-06-18T22:13:36.627384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_dataset_labels = train['cleaned_label'].unique()\nprint(unique_dataset_labels)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T22:19:21.855529Z","iopub.execute_input":"2021-06-18T22:19:21.856116Z","iopub.status.idle":"2021-06-18T22:19:21.867161Z","shell.execute_reply.started":"2021-06-18T22:19:21.856078Z","shell.execute_reply":"2021-06-18T22:19:21.866019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load dataset values","metadata":{}},{"cell_type":"code","source":"train_results = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:01.940679Z","iopub.execute_input":"2021-06-18T20:54:01.941254Z","iopub.status.idle":"2021-06-18T20:54:02.022750Z","shell.execute_reply.started":"2021-06-18T20:54:01.941187Z","shell.execute_reply":"2021-06-18T20:54:02.021493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_results.groupby('Id').count().sort_values(by='pub_title',ascending=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:02.030764Z","iopub.execute_input":"2021-06-18T20:54:02.031115Z","iopub.status.idle":"2021-06-18T20:54:02.086300Z","shell.execute_reply.started":"2021-06-18T20:54:02.031085Z","shell.execute_reply":"2021-06-18T20:54:02.085112Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_results[train_results['Id']=='170113f9-399c-489e-ab53-2faf5c64c5bc']","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:02.121120Z","iopub.execute_input":"2021-06-18T20:54:02.121521Z","iopub.status.idle":"2021-06-18T20:54:02.147462Z","shell.execute_reply.started":"2021-06-18T20:54:02.121488Z","shell.execute_reply":"2021-06-18T20:54:02.146085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:02.191789Z","iopub.execute_input":"2021-06-18T20:54:02.192135Z","iopub.status.idle":"2021-06-18T20:54:02.201399Z","shell.execute_reply.started":"2021-06-18T20:54:02.192105Z","shell.execute_reply":"2021-06-18T20:54:02.199721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv', 'r') as f:\n    sample_submission = f.read()\n","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:02.562302Z","iopub.execute_input":"2021-06-18T20:54:02.562889Z","iopub.status.idle":"2021-06-18T20:54:02.568647Z","shell.execute_reply.started":"2021-06-18T20:54:02.562833Z","shell.execute_reply":"2021-06-18T20:54:02.567602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Submission format\n| Id | PredictionString |\n| --- | --- | \n| 2100032a-7c33-4bff-97ef-690822c43466 |adni|alzheimer s disease neuroimaging initiati |\n| 2f392438-e215-4169-bebf-21ac4ff253e1 |common core of data\\|nces common core of data|t |\n| 3f316b38-1a24-45a9-8d8c-4e05a42257c6 |slosh model\\|noaa storm surge inundation|sea la |\n| 8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60 |rural urban continuum codes |","metadata":{}},{"cell_type":"code","source":"print(sample_submission)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T21:11:30.304054Z","iopub.execute_input":"2021-06-18T21:11:30.304533Z","iopub.status.idle":"2021-06-18T21:11:30.310805Z","shell.execute_reply.started":"2021-06-18T21:11:30.304493Z","shell.execute_reply":"2021-06-18T21:11:30.309228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Turn document into single string","metadata":{}},{"cell_type":"code","source":"def get_string(filename) -> str:\n    ...\n#     Load file from memory\n    with open(filename, 'r') as f:\n        doc_json = json.load(f)\n    return ' '.join([dictionary['text'] for dictionary in doc_json])\n        ","metadata":{"execution":{"iopub.status.busy":"2021-06-18T21:11:31.641141Z","iopub.execute_input":"2021-06-18T21:11:31.641800Z","iopub.status.idle":"2021-06-18T21:11:31.646922Z","shell.execute_reply.started":"2021-06-18T21:11:31.641735Z","shell.execute_reply":"2021-06-18T21:11:31.646065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"get_string('/kaggle/input/coleridgeinitiative-show-us-the-data/test/8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60.json')[:100] + '...'","metadata":{"execution":{"iopub.status.busy":"2021-06-18T21:11:32.029485Z","iopub.execute_input":"2021-06-18T21:11:32.030050Z","iopub.status.idle":"2021-06-18T21:11:32.046249Z","shell.execute_reply.started":"2021-06-18T21:11:32.030014Z","shell.execute_reply":"2021-06-18T21:11:32.043634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Create set of all possible datasets","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\nprint(len(train['dataset_title'].unique()))\nprint(len(train['dataset_label'].unique()))\nprint(len(train['cleaned_label'].unique()))\nprint(train.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T21:12:23.516795Z","iopub.execute_input":"2021-06-18T21:12:23.517240Z","iopub.status.idle":"2021-06-18T21:12:23.611372Z","shell.execute_reply.started":"2021-06-18T21:12:23.517180Z","shell.execute_reply":"2021-06-18T21:12:23.610015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(train.groupby('dataset_title'))[0][1]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:03.838407Z","iopub.execute_input":"2021-06-18T20:54:03.838896Z","iopub.status.idle":"2021-06-18T20:54:03.870952Z","shell.execute_reply.started":"2021-06-18T20:54:03.838847Z","shell.execute_reply":"2021-06-18T20:54:03.869577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_dataset_map() -> dict:\n    label_map = {}\n\n    for ndx, row in train.iterrows():\n        if not label_map.get(row['dataset_label'],False):\n            label_map[row['dataset_label']] = []\n\n        if not label_map.get(row['dataset_title'],False):\n            label_map[row['dataset_title']] = []\n\n        label_map[row['dataset_label']] += [row['cleaned_label']]\n        label_map[row['dataset_title']] += [row['cleaned_label']]\n\n    for k, v in label_map.items():\n        label_map[k] = list(set(v))\n\n    return label_map","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:03.933386Z","iopub.execute_input":"2021-06-18T20:54:03.933835Z","iopub.status.idle":"2021-06-18T20:54:03.942824Z","shell.execute_reply.started":"2021-06-18T20:54:03.933798Z","shell.execute_reply":"2021-06-18T20:54:03.941405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_map = get_dataset_map()\nlabel_map","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:04.111818Z","iopub.execute_input":"2021-06-18T20:54:04.112250Z","iopub.status.idle":"2021-06-18T20:54:06.331598Z","shell.execute_reply.started":"2021-06-18T20:54:04.112211Z","shell.execute_reply":"2021-06-18T20:54:06.330468Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def find_datasets_in_string(corpus: str, label_map: dict) -> list:\n    predictions = []\n    for k, v in label_map.items():\n        if k in corpus:\n            predictions+=v\n    predictions_set = set(predictions)\n    return '|'.join(predictions_set)\n    \n    \n    ","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:06.333291Z","iopub.execute_input":"2021-06-18T20:54:06.333578Z","iopub.status.idle":"2021-06-18T20:54:06.339141Z","shell.execute_reply.started":"2021-06-18T20:54:06.333549Z","shell.execute_reply":"2021-06-18T20:54:06.338010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_doc = get_string('/kaggle/input/coleridgeinitiative-show-us-the-data/test/8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60.json')\nprint(find_datasets_in_string(sample_doc, label_map))","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:06.341069Z","iopub.execute_input":"2021-06-18T20:54:06.341481Z","iopub.status.idle":"2021-06-18T20:54:06.358863Z","shell.execute_reply.started":"2021-06-18T20:54:06.341449Z","shell.execute_reply":"2021-06-18T20:54:06.357946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train['Id'] =='8e6996b4-ca08-4c0b-bed2-aaf07a4c6a60']","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:06.360304Z","iopub.execute_input":"2021-06-18T20:54:06.360732Z","iopub.status.idle":"2021-06-18T20:54:06.379649Z","shell.execute_reply.started":"2021-06-18T20:54:06.360699Z","shell.execute_reply":"2021-06-18T20:54:06.378815Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Search Document for all datasets in set","metadata":{}},{"cell_type":"code","source":"files = []\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        if filename.endswith('.json'):\n            files += [os.path.join(dirname, filename)]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:06.380779Z","iopub.execute_input":"2021-06-18T20:54:06.381227Z","iopub.status.idle":"2021-06-18T20:54:06.455654Z","shell.execute_reply.started":"2021-06-18T20:54:06.381172Z","shell.execute_reply":"2021-06-18T20:54:06.454687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"results = {}\nfor doc in tqdm(files):\n    results[doc.split('/')[-1].split('.')[0]] = find_datasets_in_string(get_string(doc), label_map)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:54:06.457108Z","iopub.execute_input":"2021-06-18T20:54:06.457747Z","iopub.status.idle":"2021-06-18T20:55:01.015831Z","shell.execute_reply.started":"2021-06-18T20:54:06.457691Z","shell.execute_reply":"2021-06-18T20:55:01.014679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(results.items())[:4]","metadata":{"execution":{"iopub.status.busy":"2021-06-18T21:10:46.333686Z","iopub.execute_input":"2021-06-18T21:10:46.334386Z","iopub.status.idle":"2021-06-18T21:10:46.341362Z","shell.execute_reply.started":"2021-06-18T21:10:46.334348Z","shell.execute_reply":"2021-06-18T21:10:46.340029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Format Submission","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"raw","source":"def get_csv_df(results):\n#     csv_string = \"Id,PredictionString\\n\"\n    csv_string = []\n    for k, v in results.items():\n#         csv_string+= f\"{k},{v}\\n\"\n        csv_string.append([k,v])\n    return pd.DataFrame(csv_string,columns=['Id','PredictionString'])","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:41:56.164276Z","iopub.execute_input":"2021-06-18T20:41:56.164734Z","iopub.status.idle":"2021-06-18T20:41:56.171914Z","shell.execute_reply.started":"2021-06-18T20:41:56.164684Z","shell.execute_reply":"2021-06-18T20:41:56.170524Z"}}},{"cell_type":"code","source":"get_csv_df(results)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T21:11:20.605586Z","iopub.execute_input":"2021-06-18T21:11:20.606172Z","iopub.status.idle":"2021-06-18T21:11:20.620124Z","shell.execute_reply.started":"2021-06-18T21:11:20.606135Z","shell.execute_reply":"2021-06-18T21:11:20.618622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create Test Set Submission","metadata":{}},{"cell_type":"code","source":"files = []\nfor dirname, _, filenames in os.walk('/kaggle/input/coleridgeinitiative-show-us-the-data/test/'):\n    for filename in filenames:\n        if filename.endswith('.json'):\n            files += [os.path.join(dirname, filename)]\nresults = {}\nfor doc in tqdm(files):\n    results[doc.split('/')[-1].split('.')[0]] = find_datasets_in_string(get_string(doc), label_map)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:55:01.098268Z","iopub.execute_input":"2021-06-18T20:55:01.098749Z","iopub.status.idle":"2021-06-18T20:55:01.141716Z","shell.execute_reply.started":"2021-06-18T20:55:01.098699Z","shell.execute_reply":"2021-06-18T20:55:01.140649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df = get_csv_df(results)\nsubmission_df","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:55:01.143237Z","iopub.execute_input":"2021-06-18T20:55:01.143590Z","iopub.status.idle":"2021-06-18T20:55:01.158259Z","shell.execute_reply.started":"2021-06-18T20:55:01.143555Z","shell.execute_reply":"2021-06-18T20:55:01.157157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:55:01.160008Z","iopub.execute_input":"2021-06-18T20:55:01.160346Z","iopub.status.idle":"2021-06-18T20:55:01.170570Z","shell.execute_reply.started":"2021-06-18T20:55:01.160315Z","shell.execute_reply":"2021-06-18T20:55:01.169296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open('submission.csv' , 'r') as f:\n    print(f.read())","metadata":{"execution":{"iopub.status.busy":"2021-06-18T20:55:01.172186Z","iopub.execute_input":"2021-06-18T20:55:01.172643Z","iopub.status.idle":"2021-06-18T20:55:01.186561Z","shell.execute_reply.started":"2021-06-18T20:55:01.172610Z","shell.execute_reply":"2021-06-18T20:55:01.185653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Modeling","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}