{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Efficient Keyword Matching\n'in' method can help in seaching from a few thounsand dataset names, but if you are having a dataset name list consisting of Millions of names(like [this](https://www.kaggle.com/googleai/dataset-search-metadata-for-datasets)), then it is not going to help.
\nThis kernel does that by - \n+ Splits paragraph into words\n+ Creates sets of dataset names with different word count\n+ Then picks ‘n’ consecutive words, joins them, and searches in the set
","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimg = plt.imread('../input/kaggle-images/matching.png')\nplt.figure(figsize=(15, 5))\nplt.imshow(img)\nplt.axis('off')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T09:59:27.785623Z","iopub.execute_input":"2021-06-22T09:59:27.786006Z","iopub.status.idle":"2021-06-22T09:59:28.414357Z","shell.execute_reply.started":"2021-06-22T09:59:27.785973Z","shell.execute_reply":"2021-06-22T09:59:28.4135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport gc\nimport os\nimport json\nimport pandas as pd\nfrom tqdm.auto import tqdm\n\ntqdm.pandas()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T09:30:59.57829Z","iopub.execute_input":"2021-06-22T09:30:59.579071Z","iopub.status.idle":"2021-06-22T09:30:59.588837Z","shell.execute_reply.started":"2021-06-22T09:30:59.579017Z","shell.execute_reply":"2021-06-22T09:30:59.587627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Utility functions","metadata":{}},{"cell_type":"code","source":"test_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\n\ndef num_words(x):\n try:\n return len(x.split())\n except:\n return 0\n \ndef clean_text(txt):\n return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())\n\ndef read_json_pub(filename, train_data_path=test_files_path, output='text'):\n json_path = os.path.join(train_data_path, (filename+'.json'))\n headings = []\n contents = []\n combined = []\n \n with open(json_path, 'r') as f:\n json_decode = json.load(f)\n for data in json_decode:\n headings.append(data.get('section_title'))\n contents.append(data.get('text'))\n combined.append(data.get('section_title'))\n combined.append(data.get('text'))\n \n all_headings = ' '.join(headings)\n all_contents = ' '.join(contents)\n all_data = '. '.join(combined)\n \n if output == 'text':\n return all_contents\n elif output == 'head':\n return all_headings\n else:\n return all_data","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:00:03.487695Z","iopub.execute_input":"2021-06-22T10:00:03.488304Z","iopub.status.idle":"2021-06-22T10:00:03.498966Z","shell.execute_reply.started":"2021-06-22T10:00:03.488246Z","shell.execute_reply":"2021-06-22T10:00:03.497487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Get all the dataset names and their word count in a dataframe","metadata":{}},{"cell_type":"code","source":"df1 = pd.read_csv('../input/dataset-search-metadata-for-datasets/dataset_metadata_2020_10_16.csv')[['name']]\ndf1['length'] = df1.name.progress_apply(num_words)\n\ngc.collect()\n\ndf2 = pd.read_csv('../input/dataset-search-metadata-for-datasets/dataset_metadata_2020_08_17.csv/dataset_metadata_2020_08_17.csv')[['name']]\ndf2['length'] = df2.name.progress_apply(num_words)\n\ngc.collect()\n\ndf = pd.concat([df1, df2]).reset_index(drop=True)\n\ndel df1, df2\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2021-06-22T09:25:36.903181Z","iopub.execute_input":"2021-06-22T09:25:36.90346Z","iopub.status.idle":"2021-06-22T09:30:26.752991Z","shell.execute_reply.started":"2021-06-22T09:25:36.903435Z","shell.execute_reply":"2021-06-22T09:30:26.751883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sets based on number of words in the dataset name","metadata":{}},{"cell_type":"code","source":"dict_of_set_of_datasets = {}\n\nfor i in range(1, 25):\n dict_of_set_of_datasets[i] = set(df[df.length==i].name.values)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-22T09:30:26.754888Z","iopub.execute_input":"2021-06-22T09:30:26.755232Z","iopub.status.idle":"2021-06-22T09:30:30.911136Z","shell.execute_reply.started":"2021-06-22T09:30:26.7552Z","shell.execute_reply":"2021-06-22T09:30:30.909268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Keyword matching","metadata":{}},{"cell_type":"code","source":"match_predictions = pd.DataFrame(index=[i.replace('.json', '') for i in os.listdir('../input/coleridgeinitiative-show-us-the-data/train')],\n columns = ['PredictionString'])\n\nfor id_, row in tqdm(match_predictions.iterrows(), total=len(match_predictions)):\n \n predictions = []\n \n large_string = str(read_json_pub(id_, test_files_path))\n text_list = clean_text(large_string).split()\n \n for indx, words in enumerate(zip(*[text_list[s:] + ['random_text']*s for s in range(26)])):\n for j in range(1, 25):\n if ' '.join(words[:j]) in dict_of_set_of_datasets[j]:\n predictions.append(' '.join(words[:j]))\n break\n \n predictions = '|'.join(predictions)\n if predictions!='':\n match_predictions.loc[id_,'PredictionString'] = predictions\n\nmatch_predictions.to_csv('submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-22T10:02:52.634708Z","iopub.execute_input":"2021-06-22T10:02:52.635067Z"},"trusted":true},"execution_count":null,"outputs":[]}]}