{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-11T12:56:11.215973Z","iopub.execute_input":"2021-06-11T12:56:11.216510Z","iopub.status.idle":"2021-06-11T12:56:12.795709Z","shell.execute_reply.started":"2021-06-11T12:56:11.216475Z","shell.execute_reply":"2021-06-11T12:56:12.794016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nstart_time = time.time()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.798025Z","iopub.execute_input":"2021-06-11T12:56:12.798329Z","iopub.status.idle":"2021-06-11T12:56:12.804314Z","shell.execute_reply.started":"2021-06-11T12:56:12.798293Z","shell.execute_reply":"2021-06-11T12:56:12.803343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport os\nimport random\nimport json\nfrom tqdm import tqdm\nimport re\nimport glob\nfrom textblob import TextBlob\nfrom functools import partial\nimport string\nimport nltk\nimport spacy\nfrom nltk.probability import FreqDist","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.805956Z","iopub.execute_input":"2021-06-11T12:56:12.806310Z","iopub.status.idle":"2021-06-11T12:56:12.816590Z","shell.execute_reply.started":"2021-06-11T12:56:12.806206Z","shell.execute_reply":"2021-06-11T12:56:12.815606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"RANDOM_SEED = 42\ndef seed_everything(seed=RANDOM_SEED):\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    random.seed(seed)\nseed_everything()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.818024Z","iopub.execute_input":"2021-06-11T12:56:12.818375Z","iopub.status.idle":"2021-06-11T12:56:12.831270Z","shell.execute_reply.started":"2021-06-11T12:56:12.818345Z","shell.execute_reply":"2021-06-11T12:56:12.830332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/train.csv\")\ntrain.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.833710Z","iopub.execute_input":"2021-06-11T12:56:12.833955Z","iopub.status.idle":"2021-06-11T12:56:12.926406Z","shell.execute_reply.started":"2021-06-11T12:56:12.833929Z","shell.execute_reply":"2021-06-11T12:56:12.925475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in train.columns:\n    print(col + \":\" + str(len(train[col].unique())))\n\nprint(\"all rows = \" + str(len(train)))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.930084Z","iopub.execute_input":"2021-06-11T12:56:12.930382Z","iopub.status.idle":"2021-06-11T12:56:12.952607Z","shell.execute_reply.started":"2021-06-11T12:56:12.930353Z","shell.execute_reply":"2021-06-11T12:56:12.951683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data Description train.csv -labels and metadata for the training set train/tezt directory - the full text of the training/test set's publications in JSON format, broken into sections with section titles\n\nid - publication id - note that there are multiple rows for some training documents, indicating multiple mentioned datasets. pub_title - title of the publication (a small number of publications have the same title). dataset_title - the title of the dataset that is mentioned within the publication. dataset_label - a portion of the text that indicates the dataset. cleaned_label - the dataset_label, as passed through the clean_text function from the Evaluation page. sample_submission.csv - a sample submission file in the correct format.\n\nId - publication id. PredictionString - To be filled with equivalent of cleaned_label of train data.\n","metadata":{}},{"cell_type":"code","source":"sample = pd.read_csv(\"../input/coleridgeinitiative-show-us-the-data/sample_submission.csv\")\nsample","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.954177Z","iopub.execute_input":"2021-06-11T12:56:12.954548Z","iopub.status.idle":"2021-06-11T12:56:12.967111Z","shell.execute_reply.started":"2021-06-11T12:56:12.954510Z","shell.execute_reply":"2021-06-11T12:56:12.966353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.968327Z","iopub.execute_input":"2021-06-11T12:56:12.968626Z","iopub.status.idle":"2021-06-11T12:56:12.979516Z","shell.execute_reply.started":"2021-06-11T12:56:12.968597Z","shell.execute_reply":"2021-06-11T12:56:12.978599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_path = \"../input/coleridgeinitiative-show-us-the-data/train\"\ntest_path = \"../input/coleridgeinitiative-show-us-the-data/test\"","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.980706Z","iopub.execute_input":"2021-06-11T12:56:12.980990Z","iopub.status.idle":"2021-06-11T12:56:12.987673Z","shell.execute_reply.started":"2021-06-11T12:56:12.980954Z","shell.execute_reply":"2021-06-11T12:56:12.986791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_train_path = [os.path.join(train_path,s) + \".json\" for s in train[\"Id\"]]\nall_test_path = [os.path.join(test_path,s) + \".json\" for s in sample[\"Id\"]]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:12.989081Z","iopub.execute_input":"2021-06-11T12:56:12.989330Z","iopub.status.idle":"2021-06-11T12:56:13.030674Z","shell.execute_reply.started":"2021-06-11T12:56:12.989305Z","shell.execute_reply":"2021-06-11T12:56:13.029552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_path = all_train_path[0]\njson_path","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:13.031907Z","iopub.execute_input":"2021-06-11T12:56:13.032166Z","iopub.status.idle":"2021-06-11T12:56:13.037411Z","shell.execute_reply.started":"2021-06-11T12:56:13.032140Z","shell.execute_reply":"2021-06-11T12:56:13.036454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with open(json_path, 'r') as f:\n        json_decode = json.load(f)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:13.038482Z","iopub.execute_input":"2021-06-11T12:56:13.038994Z","iopub.status.idle":"2021-06-11T12:56:13.050326Z","shell.execute_reply.started":"2021-06-11T12:56:13.038964Z","shell.execute_reply":"2021-06-11T12:56:13.048750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"jsontest = pd.DataFrame(json_decode)\njsontest","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:13.053417Z","iopub.execute_input":"2021-06-11T12:56:13.053795Z","iopub.status.idle":"2021-06-11T12:56:13.072655Z","shell.execute_reply.started":"2021-06-11T12:56:13.053757Z","shell.execute_reply":"2021-06-11T12:56:13.071357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts = \"\"\n\nfor a in jsontest.values:\n    texts += a[0] +\" \"+ a[1] + \" \"","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:13.074101Z","iopub.execute_input":"2021-06-11T12:56:13.074616Z","iopub.status.idle":"2021-06-11T12:56:13.082572Z","shell.execute_reply.started":"2021-06-11T12:56:13.074538Z","shell.execute_reply":"2021-06-11T12:56:13.081691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts[:300]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:13.084480Z","iopub.execute_input":"2021-06-11T12:56:13.084876Z","iopub.status.idle":"2021-06-11T12:56:13.097636Z","shell.execute_reply.started":"2021-06-11T12:56:13.084835Z","shell.execute_reply":"2021-06-11T12:56:13.096712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"json_path = all_train_path[0]\nwith open(json_path, 'r') as f:\n        json_decode = json.load(f)\n\njsontest = pd.DataFrame(json_decode)\n\ntexts = \"\"\n\nfor a in jsontest.values:\n    texts += a[0] +\" \"+ a[1] +\" \"","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:13.099147Z","iopub.execute_input":"2021-06-11T12:56:13.099495Z","iopub.status.idle":"2021-06-11T12:56:13.110818Z","shell.execute_reply.started":"2021-06-11T12:56:13.099467Z","shell.execute_reply":"2021-06-11T12:56:13.109476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nalltexts = []\n\nfor json_path in tqdm(all_train_path):\n\n    with open(json_path, 'r') as f:\n            json_decode = json.load(f)\n    jsontest = pd.DataFrame(json_decode)\n\n    texts = \"\"\n\n    for a in jsontest.values:\n        texts += a[0] +\" \"+ a[1] + \" \"\n        \n    alltexts.append(texts)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:13.112812Z","iopub.execute_input":"2021-06-11T12:56:13.113211Z","iopub.status.idle":"2021-06-11T12:56:39.504243Z","shell.execute_reply.started":"2021-06-11T12:56:13.113169Z","shell.execute_reply":"2021-06-11T12:56:39.503180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"text\"] = alltexts\ntrain","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.505268Z","iopub.execute_input":"2021-06-11T12:56:39.505508Z","iopub.status.idle":"2021-06-11T12:56:39.526608Z","shell.execute_reply.started":"2021-06-11T12:56:39.505477Z","shell.execute_reply":"2021-06-11T12:56:39.525669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.527874Z","iopub.execute_input":"2021-06-11T12:56:39.528150Z","iopub.status.idle":"2021-06-11T12:56:39.539831Z","shell.execute_reply.started":"2021-06-11T12:56:39.528124Z","shell.execute_reply":"2021-06-11T12:56:39.538888Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train[\"Id\"].unique()))\nprint(len(train))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.540935Z","iopub.execute_input":"2021-06-11T12:56:39.541245Z","iopub.status.idle":"2021-06-11T12:56:39.550054Z","shell.execute_reply.started":"2021-06-11T12:56:39.541194Z","shell.execute_reply":"2021-06-11T12:56:39.549346Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Idgroup = train.groupby(\"Id\")[\"dataset_label\"].count().reset_index()\nIdgroup.columns = [\"Id\",\"count\"]\nIdgroup = Idgroup.sort_values(\"count\").reset_index(drop=True)\nIdgroup","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.551114Z","iopub.execute_input":"2021-06-11T12:56:39.551509Z","iopub.status.idle":"2021-06-11T12:56:39.583187Z","shell.execute_reply.started":"2021-06-11T12:56:39.551473Z","shell.execute_reply":"2021-06-11T12:56:39.582133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostId = train[train[\"Id\"] == Idgroup[\"Id\"].iloc[-1]]\nmostId.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.584373Z","iopub.execute_input":"2021-06-11T12:56:39.584639Z","iopub.status.idle":"2021-06-11T12:56:39.672957Z","shell.execute_reply.started":"2021-06-11T12:56:39.584613Z","shell.execute_reply":"2021-06-11T12:56:39.672000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostIdlist = mostId[\"cleaned_label\"].to_list()\nmostIdlist","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.674198Z","iopub.execute_input":"2021-06-11T12:56:39.674490Z","iopub.status.idle":"2021-06-11T12:56:39.681194Z","shell.execute_reply.started":"2021-06-11T12:56:39.674464Z","shell.execute_reply":"2021-06-11T12:56:39.680155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mostIdlist = (\"|\").join(mostIdlist)\nmostIdlist","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.682551Z","iopub.execute_input":"2021-06-11T12:56:39.682850Z","iopub.status.idle":"2021-06-11T12:56:39.692280Z","shell.execute_reply.started":"2021-06-11T12:56:39.682823Z","shell.execute_reply":"2021-06-11T12:56:39.691140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.693757Z","iopub.execute_input":"2021-06-11T12:56:39.694043Z","iopub.status.idle":"2021-06-11T12:56:39.708424Z","shell.execute_reply.started":"2021-06-11T12:56:39.694014Z","shell.execute_reply":"2021-06-11T12:56:39.707271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.709896Z","iopub.execute_input":"2021-06-11T12:56:39.710343Z","iopub.status.idle":"2021-06-11T12:56:39.717703Z","shell.execute_reply.started":"2021-06-11T12:56:39.710276Z","shell.execute_reply":"2021-06-11T12:56:39.716675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check = []\nfor a in range(len(train)):\n    \n    if clean_text(train[\"dataset_label\"].iloc[a]) == train[\"cleaned_label\"].iloc[a]:\n        check.append(1)\n    else:\n        check.append(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:39.718973Z","iopub.execute_input":"2021-06-11T12:56:39.719218Z","iopub.status.idle":"2021-06-11T12:56:40.101957Z","shell.execute_reply.started":"2021-06-11T12:56:39.719194Z","shell.execute_reply":"2021-06-11T12:56:40.101017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(check)/len(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.103050Z","iopub.execute_input":"2021-06-11T12:56:40.103300Z","iopub.status.idle":"2021-06-11T12:56:40.110332Z","shell.execute_reply.started":"2021-06-11T12:56:40.103274Z","shell.execute_reply":"2021-06-11T12:56:40.109142Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"check\"] = check\ncheckdf = train[train[\"check\"]==0]\ncheckdf.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.112080Z","iopub.execute_input":"2021-06-11T12:56:40.112520Z","iopub.status.idle":"2021-06-11T12:56:40.137449Z","shell.execute_reply.started":"2021-06-11T12:56:40.112477Z","shell.execute_reply":"2021-06-11T12:56:40.136244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkdf[\"dataset_label\"].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.139039Z","iopub.execute_input":"2021-06-11T12:56:40.139450Z","iopub.status.idle":"2021-06-11T12:56:40.146023Z","shell.execute_reply.started":"2021-06-11T12:56:40.139409Z","shell.execute_reply":"2021-06-11T12:56:40.144883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"checkdf[\"cleaned_label\"].unique()","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.147285Z","iopub.execute_input":"2021-06-11T12:56:40.147792Z","iopub.status.idle":"2021-06-11T12:56:40.160018Z","shell.execute_reply.started":"2021-06-11T12:56:40.147760Z","shell.execute_reply":"2021-06-11T12:56:40.159135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clean_text(checkdf[\"dataset_label\"].iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.165922Z","iopub.execute_input":"2021-06-11T12:56:40.166199Z","iopub.status.idle":"2021-06-11T12:56:40.172023Z","shell.execute_reply.started":"2021-06-11T12:56:40.166173Z","shell.execute_reply":"2021-06-11T12:56:40.170906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cleanlabel = []\nfor a in tqdm(train[\"cleaned_label\"]):\n    if a[-1] == \" \":\n        cleanlabel.append(a[:-1])\n    else:\n        cleanlabel.append(a)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.172953Z","iopub.execute_input":"2021-06-11T12:56:40.173186Z","iopub.status.idle":"2021-06-11T12:56:40.197214Z","shell.execute_reply.started":"2021-06-11T12:56:40.173163Z","shell.execute_reply":"2021-06-11T12:56:40.196335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"check = []\nfor a in range(len(train)):\n    \n    if clean_text(train[\"dataset_label\"].iloc[a]) == cleanlabel[a]:\n        check.append(1)\n    else:\n        check.append(0)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.198259Z","iopub.execute_input":"2021-06-11T12:56:40.198506Z","iopub.status.idle":"2021-06-11T12:56:40.417371Z","shell.execute_reply.started":"2021-06-11T12:56:40.198475Z","shell.execute_reply":"2021-06-11T12:56:40.416698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(check)/len(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.418282Z","iopub.execute_input":"2021-06-11T12:56:40.418659Z","iopub.status.idle":"2021-06-11T12:56:40.425431Z","shell.execute_reply.started":"2021-06-11T12:56:40.418623Z","shell.execute_reply":"2021-06-11T12:56:40.424659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"cleaned_label\"]=cleanlabel","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.426771Z","iopub.execute_input":"2021-06-11T12:56:40.427133Z","iopub.status.idle":"2021-06-11T12:56:40.438138Z","shell.execute_reply.started":"2021-06-11T12:56:40.427097Z","shell.execute_reply":"2021-06-11T12:56:40.437246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.439111Z","iopub.execute_input":"2021-06-11T12:56:40.439393Z","iopub.status.idle":"2021-06-11T12:56:40.458755Z","shell.execute_reply.started":"2021-06-11T12:56:40.439366Z","shell.execute_reply":"2021-06-11T12:56:40.457755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"text\"] = [clean_text(s) for s in tqdm(train[\"text\"])]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:56:40.460114Z","iopub.execute_input":"2021-06-11T12:56:40.460368Z","iopub.status.idle":"2021-06-11T12:57:45.646413Z","shell.execute_reply.started":"2021-06-11T12:56:40.460341Z","shell.execute_reply":"2021-06-11T12:57:45.645510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dslabel = [clean_text(s) for s in train[\"dataset_label\"].unique()]\nlen(dslabel)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:57:45.648038Z","iopub.execute_input":"2021-06-11T12:57:45.648299Z","iopub.status.idle":"2021-06-11T12:57:45.656280Z","shell.execute_reply.started":"2021-06-11T12:57:45.648272Z","shell.execute_reply":"2021-06-11T12:57:45.655311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_control = []\nall_labels = []\nlabel_len = []\n\nfor a in tqdm(train[\"text\"]):\n    labels = []\n    for b in dslabel:\n        if b in a:\n            labels.append(clean_text(b))\n            break\n    if len(labels)==0:\n        label_control.append(0)\n    else:\n        label_control.append(1)\n        \n    #all_labels.append(\"|\".join(labels))\n    #label_len.append(len(labels))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T12:57:45.657459Z","iopub.execute_input":"2021-06-11T12:57:45.657744Z","iopub.status.idle":"2021-06-11T12:58:04.287656Z","shell.execute_reply.started":"2021-06-11T12:57:45.657719Z","shell.execute_reply":"2021-06-11T12:58:04.286633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.sum(label_control)/len(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:48.230625Z","iopub.execute_input":"2021-06-11T13:00:48.231128Z","iopub.status.idle":"2021-06-11T13:00:48.237883Z","shell.execute_reply.started":"2021-06-11T13:00:48.231090Z","shell.execute_reply":"2021-06-11T13:00:48.236755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:48.269285Z","iopub.execute_input":"2021-06-11T13:00:48.269637Z","iopub.status.idle":"2021-06-11T13:00:48.278806Z","shell.execute_reply.started":"2021-06-11T13:00:48.269604Z","shell.execute_reply":"2021-06-11T13:00:48.277669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\nalltexts = []\n\nfor json_path in tqdm(all_test_path):\n\n    with open(json_path, 'r') as f:\n            json_decode = json.load(f)\n    jsontest = pd.DataFrame(json_decode)\n\n    texts = \"\"\n\n    for a in jsontest.values:\n        texts += a[0] + \" \" + a[1] + \" \"\n        \n    alltexts.append(texts)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:48.297908Z","iopub.execute_input":"2021-06-11T13:00:48.298237Z","iopub.status.idle":"2021-06-11T13:00:48.336826Z","shell.execute_reply.started":"2021-06-11T13:00:48.298208Z","shell.execute_reply":"2021-06-11T13:00:48.335617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[\"text\"] = alltexts","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:48.338720Z","iopub.execute_input":"2021-06-11T13:00:48.339007Z","iopub.status.idle":"2021-06-11T13:00:48.343550Z","shell.execute_reply.started":"2021-06-11T13:00:48.338975Z","shell.execute_reply":"2021-06-11T13:00:48.342461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:48.957513Z","iopub.execute_input":"2021-06-11T13:00:48.957859Z","iopub.status.idle":"2021-06-11T13:00:48.969864Z","shell.execute_reply.started":"2021-06-11T13:00:48.957829Z","shell.execute_reply":"2021-06-11T13:00:48.968628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[\"text\"] = [clean_text(s) for s in tqdm(sample[\"text\"])]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:48.981963Z","iopub.execute_input":"2021-06-11T13:00:48.982265Z","iopub.status.idle":"2021-06-11T13:00:49.014210Z","shell.execute_reply.started":"2021-06-11T13:00:48.982240Z","shell.execute_reply":"2021-06-11T13:00:49.013304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(dslabel))\ndstitle = [clean_text(s) for s in train[\"dataset_title\"].unique()]\ndslabel = set(dslabel + dstitle) \nlen(dslabel)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:49.018239Z","iopub.execute_input":"2021-06-11T13:00:49.018525Z","iopub.status.idle":"2021-06-11T13:00:49.028687Z","shell.execute_reply.started":"2021-06-11T13:00:49.018497Z","shell.execute_reply":"2021-06-11T13:00:49.027462Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_control = []  \nall_labels = []\nlabel_len = []\n\nfor a in tqdm(sample[\"text\"]):\n    labels = []\n    for b in dslabel:\n        if b in a:\n            labels.append(clean_text(b))\n            \n    if len(labels)==0:\n        label_control.append(0)\n    else:\n        label_control.append(1)\n    \n    all_labels.append(\"|\".join(labels))\n    label_len.append(len(labels))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:49.041422Z","iopub.execute_input":"2021-06-11T13:00:49.041736Z","iopub.status.idle":"2021-06-11T13:00:49.075357Z","shell.execute_reply.started":"2021-06-11T13:00:49.041709Z","shell.execute_reply":"2021-06-11T13:00:49.074505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[\"PredictionString\"] = all_labels\nsample","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:49.076935Z","iopub.execute_input":"2021-06-11T13:00:49.077179Z","iopub.status.idle":"2021-06-11T13:00:49.088562Z","shell.execute_reply.started":"2021-06-11T13:00:49.077154Z","shell.execute_reply":"2021-06-11T13:00:49.087882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample[\"PredictionString\"].iloc[2]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:49.093885Z","iopub.execute_input":"2021-06-11T13:00:49.094318Z","iopub.status.idle":"2021-06-11T13:00:49.101184Z","shell.execute_reply.started":"2021-06-11T13:00:49.094278Z","shell.execute_reply":"2021-06-11T13:00:49.100160Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = sample[[\"Id\",\"PredictionString\"]]","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:49.110263Z","iopub.execute_input":"2021-06-11T13:00:49.110943Z","iopub.status.idle":"2021-06-11T13:00:49.115759Z","shell.execute_reply.started":"2021-06-11T13:00:49.110911Z","shell.execute_reply":"2021-06-11T13:00:49.114814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:49.133591Z","iopub.execute_input":"2021-06-11T13:00:49.134122Z","iopub.status.idle":"2021-06-11T13:00:49.142700Z","shell.execute_reply.started":"2021-06-11T13:00:49.134090Z","shell.execute_reply":"2021-06-11T13:00:49.141552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:01:26.053916Z","iopub.execute_input":"2021-06-11T13:01:26.054369Z","iopub.status.idle":"2021-06-11T13:01:26.064034Z","shell.execute_reply.started":"2021-06-11T13:01:26.054338Z","shell.execute_reply":"2021-06-11T13:01:26.062803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"--- %s seconds ---\" % (time.time() - start_time))","metadata":{"execution":{"iopub.status.busy":"2021-06-11T13:00:49.201819Z","iopub.execute_input":"2021-06-11T13:00:49.202145Z","iopub.status.idle":"2021-06-11T13:00:49.207214Z","shell.execute_reply.started":"2021-06-11T13:00:49.202118Z","shell.execute_reply":"2021-06-11T13:00:49.206233Z"},"trusted":true},"execution_count":null,"outputs":[]}]}