{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\nfrom collections import defaultdict\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        break\n        \n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-08T08:56:22.232176Z","iopub.execute_input":"2021-06-08T08:56:22.232666Z","iopub.status.idle":"2021-06-08T08:56:33.769055Z","shell.execute_reply.started":"2021-06-08T08:56:22.232549Z","shell.execute_reply":"2021-06-08T08:56:33.768090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/train.csv')\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:56:33.770613Z","iopub.execute_input":"2021-06-08T08:56:33.771081Z","iopub.status.idle":"2021-06-08T08:56:33.939376Z","shell.execute_reply.started":"2021-06-08T08:56:33.771046Z","shell.execute_reply":"2021-06-08T08:56:33.938041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:56:33.941688Z","iopub.execute_input":"2021-06-08T08:56:33.942134Z","iopub.status.idle":"2021-06-08T08:56:33.976299Z","shell.execute_reply.started":"2021-06-08T08:56:33.942078Z","shell.execute_reply":"2021-06-08T08:56:33.975061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dir = '/kaggle/input/coleridgeinitiative-show-us-the-data/train/'\ntest_dir = '/kaggle/input/coleridgeinitiative-show-us-the-data/test/'","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:56:33.978109Z","iopub.execute_input":"2021-06-08T08:56:33.978423Z","iopub.status.idle":"2021-06-08T08:56:33.983672Z","shell.execute_reply.started":"2021-06-08T08:56:33.978392Z","shell.execute_reply":"2021-06-08T08:56:33.982438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reading the training json files**","metadata":{}},{"cell_type":"code","source":"def getText(filenames, Dir=train_dir):\n    text_list = []\n    append = text_list.append\n    for filename in filenames:\n        file = pd.read_json(Dir+filename+'.json')\n        text = ' '.join(file['text'].values)\n        append(text)\n    return text_list","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:56:33.985902Z","iopub.execute_input":"2021-06-08T08:56:33.986355Z","iopub.status.idle":"2021-06-08T08:56:33.995278Z","shell.execute_reply.started":"2021-06-08T08:56:33.986309Z","shell.execute_reply":"2021-06-08T08:56:33.994125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_list = getText(train['Id'].values)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:56:33.996814Z","iopub.execute_input":"2021-06-08T08:56:33.997159Z","iopub.status.idle":"2021-06-08T08:59:00.571285Z","shell.execute_reply.started":"2021-06-08T08:56:33.997126Z","shell.execute_reply":"2021-06-08T08:59:00.570032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['text'] = text_list","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.573109Z","iopub.execute_input":"2021-06-08T08:59:00.573562Z","iopub.status.idle":"2021-06-08T08:59:00.585872Z","shell.execute_reply.started":"2021-06-08T08:59:00.573511Z","shell.execute_reply":"2021-06-08T08:59:00.584303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.589988Z","iopub.execute_input":"2021-06-08T08:59:00.590337Z","iopub.status.idle":"2021-06-08T08:59:00.611883Z","shell.execute_reply.started":"2021-06-08T08:59:00.590305Z","shell.execute_reply":"2021-06-08T08:59:00.610795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.613881Z","iopub.execute_input":"2021-06-08T08:59:00.614333Z","iopub.status.idle":"2021-06-08T08:59:00.623338Z","shell.execute_reply.started":"2021-06-08T08:59:00.614286Z","shell.execute_reply":"2021-06-08T08:59:00.622457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Combining the Existing Labels**","metadata":{}},{"cell_type":"code","source":"data_title = list(map(clean_text, train['dataset_title'].unique()))\ndata_label = list(map(clean_text, train['dataset_label'].unique()))\ncleaned_label = list(train['cleaned_label'].unique())","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.624921Z","iopub.execute_input":"2021-06-08T08:59:00.625334Z","iopub.status.idle":"2021-06-08T08:59:00.643014Z","shell.execute_reply.started":"2021-06-08T08:59:00.625292Z","shell.execute_reply":"2021-06-08T08:59:00.641879Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = set(data_title + data_label + cleaned_label)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.644375Z","iopub.execute_input":"2021-06-08T08:59:00.644805Z","iopub.status.idle":"2021-06-08T08:59:00.648964Z","shell.execute_reply.started":"2021-06-08T08:59:00.644771Z","shell.execute_reply":"2021-06-08T08:59:00.647945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.650038Z","iopub.execute_input":"2021-06-08T08:59:00.650423Z","iopub.status.idle":"2021-06-08T08:59:00.674730Z","shell.execute_reply.started":"2021-06-08T08:59:00.650395Z","shell.execute_reply":"2021-06-08T08:59:00.673661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subText = getText(sample_submission['Id'].values, test_dir)\nsample_submission['text'] = subText","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.675742Z","iopub.execute_input":"2021-06-08T08:59:00.676132Z","iopub.status.idle":"2021-06-08T08:59:00.734143Z","shell.execute_reply.started":"2021-06-08T08:59:00.676102Z","shell.execute_reply":"2021-06-08T08:59:00.732935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping = dict()\nfor row in train.iterrows():\n    cleaned_label = row[1]['cleaned_label'].strip()\n    title = clean_text(row[1]['dataset_title']).strip()\n    dataSetLabel = clean_text(row[1]['dataset_label']).strip()\n    mapTitle = mapping.get(title, set())\n    mapTitle.add(cleaned_label)\n    mapping[title] = mapTitle\n    mapLabel = mapping.get(dataSetLabel, set())\n    mapLabel.add(cleaned_label)\n    mapping[dataSetLabel] = mapLabel\n    mapCleanedLabel = mapping.get(cleaned_label, set())\n    mapCleanedLabel.add(cleaned_label)\n    mapping[dataSetLabel] = mapLabel","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:00.736562Z","iopub.execute_input":"2021-06-08T08:59:00.736958Z","iopub.status.idle":"2021-06-08T08:59:02.897972Z","shell.execute_reply.started":"2021-06-08T08:59:00.736925Z","shell.execute_reply":"2021-06-08T08:59:02.896995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:02.899334Z","iopub.execute_input":"2021-06-08T08:59:02.899692Z","iopub.status.idle":"2021-06-08T08:59:02.914397Z","shell.execute_reply.started":"2021-06-08T08:59:02.899661Z","shell.execute_reply":"2021-06-08T08:59:02.913585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:02.915547Z","iopub.execute_input":"2021-06-08T08:59:02.916071Z","iopub.status.idle":"2021-06-08T08:59:02.935466Z","shell.execute_reply.started":"2021-06-08T08:59:02.916039Z","shell.execute_reply":"2021-06-08T08:59:02.934282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Matching the string from labels to the test text**","metadata":{}},{"cell_type":"code","source":"strings = []\nfor text in sample_submission['text'].values:\n    text = clean_text(text)\n    predictString = set()\n    for label in labels:\n        if label in text:\n            for string in mapping[label.strip()]:\n                predictString.add(string)\n    predictString = \"|\".join(predictString)\n    strings.append(predictString)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:02.936890Z","iopub.execute_input":"2021-06-08T08:59:02.937342Z","iopub.status.idle":"2021-06-08T08:59:03.004316Z","shell.execute_reply.started":"2021-06-08T08:59:02.937311Z","shell.execute_reply":"2021-06-08T08:59:03.003233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Submission !!**","metadata":{}},{"cell_type":"code","source":"sample_submission['PredictionString'] = strings\nsample_submission.drop(['text'], axis=1).to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T08:59:03.005533Z","iopub.execute_input":"2021-06-08T08:59:03.005860Z","iopub.status.idle":"2021-06-08T08:59:03.016246Z","shell.execute_reply.started":"2021-06-08T08:59:03.005828Z","shell.execute_reply":"2021-06-08T08:59:03.014799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}