{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!ls ../input/tf-installing-packageofficial-model-and-text","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:07:54.127116Z","iopub.execute_input":"2021-06-16T13:07:54.127827Z","iopub.status.idle":"2021-06-16T13:07:54.893715Z","shell.execute_reply.started":"2021-06-16T13:07:54.127704Z","shell.execute_reply":"2021-06-16T13:07:54.892699Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:07:54.896578Z","iopub.execute_input":"2021-06-16T13:07:54.897410Z","iopub.status.idle":"2021-06-16T13:07:55.677252Z","shell.execute_reply.started":"2021-06-16T13:07:54.897353Z","shell.execute_reply":"2021-06-16T13:07:55.675951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp ../input/tf-installing-packageofficial-model-and-text/cached_property-1.5.2-py2.py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/gast-0.4.0-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/gin_config-0.4.0-py2.py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/grpcio-1.34.1-cp37-cp37m-manylinux2014_x86_64.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/h5py-3.1.0-cp37-cp37m-manylinux1_x86_64.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/keras_nightly-2.5.0.dev2021032900-py2.py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/portalocker-2.0.0-py2.py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/py-cpuinfo-8.0.0.xyz.csv /tmp/pip/cache/py-cpuinfo-8.0.0.tar.gz\n!cp ../input/tf-installing-packageofficial-model-and-text/pycocotools-2.0.2.xyz.csv /tmp/pip/cache/pycocotools-2.0.2.tar.gz\n!cp ../input/tf-installing-packageofficial-model-and-text/sacrebleu-1.5.1-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/seqeval-1.2.2-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tensorboard-2.5.0-py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tensorboard_data_server-0.6.1-py3-none-manylinux2010_x86_64.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tensorflow-2.5.0-cp37-cp37m-manylinux2010_x86_64.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tensorflow_estimator-2.5.0-py2.py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tensorflow_model_optimization-0.5.0-py2.py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tensorflow_text-2.5.0-cp37-cp37m-manylinux1_x86_64.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tf_models_official-2.5.0-py2.py3-none-any.whl /tmp/pip/cache/\n!cp ../input/tf-installing-packageofficial-model-and-text/tf_slim-1.1.0-py2.py3-none-any.whl /tmp/pip/cache/\n","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:07:55.680415Z","iopub.execute_input":"2021-06-16T13:07:55.680771Z","iopub.status.idle":"2021-06-16T13:08:14.656046Z","shell.execute_reply.started":"2021-06-16T13:07:55.680736Z","shell.execute_reply":"2021-06-16T13:08:14.654866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /tmp/pip/cache/\n!pip install --no-index --find-links /tmp/pip/cache/ tensorflow-text\n!pip install --no-index --find-links /tmp/pip/cache/ tf-models-official","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:08:14.657986Z","iopub.execute_input":"2021-06-16T13:08:14.658456Z","iopub.status.idle":"2021-06-16T13:09:21.664348Z","shell.execute_reply.started":"2021-06-16T13:08:14.658406Z","shell.execute_reply":"2021-06-16T13:09:21.663168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\nimport tensorflow as tf\nimport tensorflow_hub as hub\nimport tensorflow_text as text\nfrom official.nlp import optimization\nfrom tqdm import tqdm\nimport os\nimport glob\nimport datetime\nimport json\nimport re\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, OneHotEncoder\n","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:21.665895Z","iopub.execute_input":"2021-06-16T13:09:21.666235Z","iopub.status.idle":"2021-06-16T13:09:25.106020Z","shell.execute_reply.started":"2021-06-16T13:09:21.666203Z","shell.execute_reply":"2021-06-16T13:09:25.104954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tfhub_handle_encoder = \"/kaggle/input/bertpreprocessing-v3-en-uncased-l12-v4/small_bert_bert_en_uncased_L-8_H-512_A-8_2\"\n#tfhub_handle_preprocess = \"/kaggle/input/bertpreprocessing-v3-en-uncased-l12-v4/bert_en_uncased_preprocess_3\"\n#train_path = \"/kaggle/input/coleridgeinitiative-show-us-the-data/train/\"\ntest_path = \"/kaggle/input/coleridgeinitiative-show-us-the-data/test/\"","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.107566Z","iopub.execute_input":"2021-06-16T13:09:25.108060Z","iopub.status.idle":"2021-06-16T13:09:25.117059Z","shell.execute_reply.started":"2021-06-16T13:09:25.107992Z","shell.execute_reply":"2021-06-16T13:09:25.114788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Get Columns Name","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\ndf = df.drop(['pub_title','dataset_title','dataset_label'], axis =1)","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.119882Z","iopub.execute_input":"2021-06-16T13:09:25.120384Z","iopub.status.idle":"2021-06-16T13:09:25.306922Z","shell.execute_reply.started":"2021-06-16T13:09:25.120334Z","shell.execute_reply":"2021-06-16T13:09:25.305857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labelencoder = LabelEncoder()\ndf['cleaned_label'] = labelencoder.fit_transform(df['cleaned_label'])\nohencoder = OneHotEncoder()\nonehot_label = df.iloc[:,-1].values\nonehot_label = onehot_label.reshape(-1,1)\nonehot_label = ohencoder.fit_transform(onehot_label)\nonehot_label = pd.DataFrame(onehot_label.todense())","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.310746Z","iopub.execute_input":"2021-06-16T13:09:25.311111Z","iopub.status.idle":"2021-06-16T13:09:25.357396Z","shell.execute_reply.started":"2021-06-16T13:09:25.311077Z","shell.execute_reply":"2021-06-16T13:09:25.356347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"onehot_label = pd.DataFrame(onehot_label)\ndataset_resource = list()\nfor col in onehot_label.columns:\n    dataset_resource.append(col)\ndataset_resource = labelencoder.inverse_transform(dataset_resource)","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.359527Z","iopub.execute_input":"2021-06-16T13:09:25.359959Z","iopub.status.idle":"2021-06-16T13:09:25.366981Z","shell.execute_reply.started":"2021-06-16T13:09:25.359916Z","shell.execute_reply":"2021-06-16T13:09:25.365721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Making Predictions","metadata":{}},{"cell_type":"markdown","source":"Get Testing Text","metadata":{}},{"cell_type":"code","source":"test_file_list = !ls ../input/coleridgeinitiative-show-us-the-data/test\ntest_file_list = [id.strip('.json') for id in test_file_list]","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.368755Z","iopub.execute_input":"2021-06-16T13:09:25.369261Z","iopub.status.idle":"2021-06-16T13:09:25.404145Z","shell.execute_reply.started":"2021-06-16T13:09:25.369209Z","shell.execute_reply":"2021-06-16T13:09:25.402963Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(text):\n    text = re.sub(r\"https?://[A-Za-z0-9./]+\", ' ', text)#remove url\n    text = re.sub(r\"\\S+@\\S+\", ' ', text) #remove email address\n    text = re.sub(r\"[^a-zA-Z.!?']\", ' ', text) #remove special symbol\n    text = re.sub(r\" +\", ' ', text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.406149Z","iopub.execute_input":"2021-06-16T13:09:25.406526Z","iopub.status.idle":"2021-06-16T13:09:25.413024Z","shell.execute_reply.started":"2021-06-16T13:09:25.406484Z","shell.execute_reply":"2021-06-16T13:09:25.411935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_test_json(test_file_list, path): #Get Training Text\n    file_list = []\n    text_data = []\n    for i, rec_id in tqdm(enumerate(test_file_list), total = len(test_file_list)):\n        location = f'{path}{rec_id}.json'\n        file_list.append(rec_id)\n        add_text = str(\"\")\n        with open(location, 'r') as f:\n            text_data.append(json.load(f))\n    for j in range(len(text_data)):\n        temp_dict = []\n        add_text = str(\"\")\n        temp_dict = text_data[j]\n        for k in range(1): #only get abstract\n            temp_text_iter = dict(temp_dict[k])\n            add_text= add_text + temp_text_iter['text']    \n        text_data[j] = add_text\n        text_data[j] = clean_text(text_data[j])\n        \n    text_df = pd.DataFrame(list(zip(file_list, text_data)),\n                           columns =['ID', 'text'])\n    \n    return text_df","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.414241Z","iopub.execute_input":"2021-06-16T13:09:25.414514Z","iopub.status.idle":"2021-06-16T13:09:25.427124Z","shell.execute_reply.started":"2021-06-16T13:09:25.414487Z","shell.execute_reply":"2021-06-16T13:09:25.425944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_df_test = read_test_json(test_file_list, test_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.428734Z","iopub.execute_input":"2021-06-16T13:09:25.429050Z","iopub.status.idle":"2021-06-16T13:09:25.469997Z","shell.execute_reply.started":"2021-06-16T13:09:25.429000Z","shell.execute_reply":"2021-06-16T13:09:25.468873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_model = tf.saved_model.load('../input/my-pre-trained-bert-model/temp_001')\n","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:25.471355Z","iopub.execute_input":"2021-06-16T13:09:25.471641Z","iopub.status.idle":"2021-06-16T13:09:48.425436Z","shell.execute_reply.started":"2021-06-16T13:09:25.471614Z","shell.execute_reply":"2021-06-16T13:09:48.424508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_prediction(text):\n    result = tf.constant([text])\n    result = classifier_model(result)\n    result = tf.sigmoid(result)\n    result = result.numpy()\n    result = result[0][:]\n    \n    return result","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:48.429545Z","iopub.execute_input":"2021-06-16T13:09:48.430250Z","iopub.status.idle":"2021-06-16T13:09:48.436295Z","shell.execute_reply.started":"2021-06-16T13:09:48.430196Z","shell.execute_reply":"2021-06-16T13:09:48.435527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_sub = text_df_test[\"text\"].apply(lambda x: get_prediction(x))\ny_sub","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:48.437790Z","iopub.execute_input":"2021-06-16T13:09:48.438452Z","iopub.status.idle":"2021-06-16T13:09:51.110471Z","shell.execute_reply.started":"2021-06-16T13:09:48.438402Z","shell.execute_reply":"2021-06-16T13:09:51.109219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(y_sub.shape[0]):\n    \n    a = y_sub[i]\n    a = a >= 0.05\n    if True not in a:\n        a = y_sub[i]\n        a = a.max() == a\n    \n    y_sub[i] = a\n    \ny_sub","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:51.112031Z","iopub.execute_input":"2021-06-16T13:09:51.112441Z","iopub.status.idle":"2021-06-16T13:09:51.129116Z","shell.execute_reply.started":"2021-06-16T13:09:51.112397Z","shell.execute_reply":"2021-06-16T13:09:51.128267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"label_sub = []\nfor i in range(y_sub.shape[0]):\n    temp_label = []\n    temp_text = str('')\n    for j in range(y_sub[0].shape[0]):\n        if y_sub[i][j] == True:\n            temp_label = dataset_resource[j]\n            temp_text = str(temp_text+'|'+temp_label)\n            temp_text = temp_text.strip()  \n            temp_text = temp_text.strip('|')\n    label_sub.append(temp_text)","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:51.130464Z","iopub.execute_input":"2021-06-16T13:09:51.131318Z","iopub.status.idle":"2021-06-16T13:09:51.145273Z","shell.execute_reply.started":"2021-06-16T13:09:51.131280Z","shell.execute_reply":"2021-06-16T13:09:51.144226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.DataFrame(list(zip(test_file_list,\n                                   label_sub)),\n                          columns =['Id', 'PredictionString'])","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:51.146868Z","iopub.execute_input":"2021-06-16T13:09:51.147589Z","iopub.status.idle":"2021-06-16T13:09:51.158829Z","shell.execute_reply.started":"2021-06-16T13:09:51.147549Z","shell.execute_reply":"2021-06-16T13:09:51.157834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:51.160548Z","iopub.execute_input":"2021-06-16T13:09:51.161231Z","iopub.status.idle":"2021-06-16T13:09:51.183575Z","shell.execute_reply.started":"2021-06-16T13:09:51.161131Z","shell.execute_reply":"2021-06-16T13:09:51.182219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.iloc[3,1]","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:13:23.693814Z","iopub.execute_input":"2021-06-16T13:13:23.694509Z","iopub.status.idle":"2021-06-16T13:13:23.702304Z","shell.execute_reply.started":"2021-06-16T13:13:23.694463Z","shell.execute_reply":"2021-06-16T13:13:23.701134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)\n","metadata":{"execution":{"iopub.status.busy":"2021-06-16T13:09:51.193064Z","iopub.execute_input":"2021-06-16T13:09:51.193840Z","iopub.status.idle":"2021-06-16T13:09:51.208893Z","shell.execute_reply.started":"2021-06-16T13:09:51.193789Z","shell.execute_reply":"2021-06-16T13:09:51.207760Z"},"trusted":true},"execution_count":null,"outputs":[]}]}