{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport json\nimport re","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:56:11.814379Z","iopub.execute_input":"2021-06-08T13:56:11.814792Z","iopub.status.idle":"2021-06-08T13:56:11.819225Z","shell.execute_reply.started":"2021-06-08T13:56:11.814756Z","shell.execute_reply":"2021-06-08T13:56:11.818243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data import","metadata":{"execution":{"iopub.status.busy":"2021-06-08T11:38:29.443314Z","iopub.execute_input":"2021-06-08T11:38:29.443638Z","iopub.status.idle":"2021-06-08T11:38:29.446915Z","shell.execute_reply.started":"2021-06-08T11:38:29.443607Z","shell.execute_reply":"2021-06-08T11:38:29.445995Z"}}},{"cell_type":"code","source":"df_train=pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\nsample=pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:43:13.804473Z","iopub.execute_input":"2021-06-08T13:43:13.804944Z","iopub.status.idle":"2021-06-08T13:43:13.874296Z","shell.execute_reply.started":"2021-06-08T13:43:13.804916Z","shell.execute_reply":"2021-06-08T13:43:13.873586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Basic Checks","metadata":{}},{"cell_type":"code","source":"print(\"No.of Row's and Columns of data \\n *********************** \\n  \",df_train.shape,\n      \"\\n\\n Top 5 records of the data \\n ************************\",df_train.head(5),\n      \"\\n\\n ************************** \\n \\n Bottom 5 records of the data \\n\\n ******************\",df_train.tail(5))","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:47:34.244369Z","iopub.execute_input":"2021-06-08T13:47:34.244765Z","iopub.status.idle":"2021-06-08T13:47:34.257661Z","shell.execute_reply.started":"2021-06-08T13:47:34.244718Z","shell.execute_reply":"2021-06-08T13:47:34.256889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#creating id values as list for json data import\nid_values=df_train.Id.values.tolist()\n\n#Creating a function to import json data\nidwise_json_txt=[]\ndef json_txt_extraction():    \n    for id_lst in range(len(id_values)):\n#         print(id_lst,id_values[id_lst])\n        f = open(\"../input/coleridgeinitiative-show-us-the-data/train/\"+ id_values[id_lst] +\".json\",)\n        data = json.load(f)\n        dict_txt_lst=[]\n        for inp_txt in range(len(data)):\n            txt=data[inp_txt]\n            txtn=txt['text']\n            dict_txt_lst.append(txtn)\n        idwise_json_txt_str=' '.join(dict_txt_lst)\n        idwise_json_txt.append(idwise_json_txt_str)\n    return idwise_json_txt \njson_txt_extraction()\n\n#updating train data with json text data\ndf_train['text']=idwise_json_txt","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:50:15.758935Z","iopub.execute_input":"2021-06-08T13:50:15.759315Z","iopub.status.idle":"2021-06-08T13:50:30.947925Z","shell.execute_reply.started":"2021-06-08T13:50:15.759280Z","shell.execute_reply":"2021-06-08T13:50:30.946841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:50:52.984442Z","iopub.execute_input":"2021-06-08T13:50:52.985094Z","iopub.status.idle":"2021-06-08T13:50:53.007693Z","shell.execute_reply.started":"2021-06-08T13:50:52.985056Z","shell.execute_reply":"2021-06-08T13:50:53.006496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:56:16.205272Z","iopub.execute_input":"2021-06-08T13:56:16.205786Z","iopub.status.idle":"2021-06-08T13:56:16.210338Z","shell.execute_reply.started":"2021-06-08T13:56:16.205754Z","shell.execute_reply":"2021-06-08T13:56:16.209254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_title = list(map(clean_text, df_train['dataset_title'].unique()))\ndata_label = list(map(clean_text, df_train['dataset_label'].unique()))\ncleaned_label = list(df_train['cleaned_label'].unique())","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:57:12.524007Z","iopub.execute_input":"2021-06-08T13:57:12.524540Z","iopub.status.idle":"2021-06-08T13:57:12.537401Z","shell.execute_reply.started":"2021-06-08T13:57:12.524494Z","shell.execute_reply":"2021-06-08T13:57:12.536674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data_title\n# data_label","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:57:31.653395Z","iopub.execute_input":"2021-06-08T13:57:31.653784Z","iopub.status.idle":"2021-06-08T13:57:31.657829Z","shell.execute_reply.started":"2021-06-08T13:57:31.653752Z","shell.execute_reply":"2021-06-08T13:57:31.656589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = set(data_title + data_label + cleaned_label)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:57:43.453980Z","iopub.execute_input":"2021-06-08T13:57:43.454479Z","iopub.status.idle":"2021-06-08T13:57:43.459131Z","shell.execute_reply.started":"2021-06-08T13:57:43.454436Z","shell.execute_reply":"2021-06-08T13:57:43.458176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# labels","metadata":{"execution":{"iopub.status.busy":"2021-06-08T13:58:22.949028Z","iopub.execute_input":"2021-06-08T13:58:22.951214Z","iopub.status.idle":"2021-06-08T13:58:22.954493Z","shell.execute_reply.started":"2021-06-08T13:58:22.951174Z","shell.execute_reply":"2021-06-08T13:58:22.953875Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# importing test data\n#creating id values as list for json data import\nid_values_text=sample.Id.values.tolist()\n\n#Creating a function to import json data\nidwise_json_txt_test=[]\ndef json_txt_extraction_test():    \n    for id_lst in range(len(id_values_text)):\n#         print(id_lst,id_values[id_lst])\n        f = open(\"../input/coleridgeinitiative-show-us-the-data/test/\"+ id_values_text[id_lst] +\".json\",)\n        data = json.load(f)\n        dict_txt_lst=[]\n        for inp_txt in range(len(data)):\n            txt=data[inp_txt]\n            txtn=txt['text']\n            dict_txt_lst.append(txtn)\n        idwise_json_txt_str=' '.join(dict_txt_lst)\n        idwise_json_txt_test.append(idwise_json_txt_str)\n    return idwise_json_txt_test \njson_txt_extraction_test()\n\n#updating train data with json text data\nsample['text']=idwise_json_txt_test\n","metadata":{"execution":{"iopub.status.busy":"2021-06-08T14:01:56.703957Z","iopub.execute_input":"2021-06-08T14:01:56.704339Z","iopub.status.idle":"2021-06-08T14:01:56.741145Z","shell.execute_reply.started":"2021-06-08T14:01:56.704305Z","shell.execute_reply":"2021-06-08T14:01:56.740400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2021-06-08T14:03:34.617855Z","iopub.execute_input":"2021-06-08T14:03:34.618240Z","iopub.status.idle":"2021-06-08T14:03:34.630853Z","shell.execute_reply.started":"2021-06-08T14:03:34.618206Z","shell.execute_reply":"2021-06-08T14:03:34.629817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapping = dict()\nfor row in df_train.iterrows():\n    cleaned_label = row[1]['cleaned_label'].strip()\n    title = clean_text(row[1]['dataset_title']).strip()\n    dataSetLabel = clean_text(row[1]['dataset_label']).strip()\n    mapTitle = mapping.get(title, set())\n    mapTitle.add(cleaned_label)\n    mapping[title] = mapTitle\n    mapLabel = mapping.get(dataSetLabel, set())\n    mapLabel.add(cleaned_label)\n    mapping[dataSetLabel] = mapLabel\n    mapCleanedLabel = mapping.get(cleaned_label, set())\n    mapCleanedLabel.add(cleaned_label)\n    mapping[dataSetLabel] = mapLabel","metadata":{"execution":{"iopub.status.busy":"2021-06-08T14:13:36.232647Z","iopub.execute_input":"2021-06-08T14:13:36.233017Z","iopub.status.idle":"2021-06-08T14:13:38.416723Z","shell.execute_reply.started":"2021-06-08T14:13:36.232986Z","shell.execute_reply":"2021-06-08T14:13:38.415725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strings = []\nfor text in sample['text'].values:\n    text = clean_text(text)\n    predictString = set()\n    for label in labels:\n        if label in text:\n            for string in mapping[label.strip()]:\n                predictString.add(string)\n    predictString = \"|\".join(predictString)\n    strings.append(predictString)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T14:13:51.522382Z","iopub.execute_input":"2021-06-08T14:13:51.522731Z","iopub.status.idle":"2021-06-08T14:13:51.582869Z","shell.execute_reply.started":"2021-06-08T14:13:51.522694Z","shell.execute_reply":"2021-06-08T14:13:51.582011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strings","metadata":{"execution":{"iopub.status.busy":"2021-06-08T14:14:10.782020Z","iopub.execute_input":"2021-06-08T14:14:10.782391Z","iopub.status.idle":"2021-06-08T14:14:10.786782Z","shell.execute_reply.started":"2021-06-08T14:14:10.782360Z","shell.execute_reply":"2021-06-08T14:14:10.786128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['PredictionString'] = strings\nsample.drop(['text'], axis=1).to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-08T14:14:46.033135Z","iopub.execute_input":"2021-06-08T14:14:46.033668Z","iopub.status.idle":"2021-06-08T14:14:46.043220Z","shell.execute_reply.started":"2021-06-08T14:14:46.033635Z","shell.execute_reply":"2021-06-08T14:14:46.042420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}