{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# MLiP Group 15\nUsing this notebook, we will perform NER using a fine-tuned Sci-BERT model, combined with string matching.\n\nNote: we used a notebook from tungmphung (https://www.kaggle.com/tungmphung/coleridge-matching-bert-ner) as a starting point.","metadata":{}},{"cell_type":"markdown","source":"# Settings","metadata":{}},{"cell_type":"code","source":"ABELVALIDATIONSIZE = 4000\nSCIBERTDATASETNAME = \"uniquevalidation-10neg-1pos\"\n\nvalidation_version = True\n","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:52:34.232832Z","iopub.execute_input":"2021-06-09T13:52:34.233362Z","iopub.status.idle":"2021-06-09T13:52:34.237438Z","shell.execute_reply.started":"2021-06-09T13:52:34.233313Z","shell.execute_reply":"2021-06-09T13:52:34.236624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Install offline packages","metadata":{}},{"cell_type":"code","source":"# Install offline libraries\n!pip install datasets --no-index --find-links=file:///kaggle/input/coleridge-packages/packages/datasets\n!pip install ../input/coleridge-packages/seqeval-1.2.2-py3-none-any.whl\n!pip install ../input/coleridge-packages/tokenizers-0.10.1-cp37-cp37m-manylinux1_x86_64.whl\n!pip install ../input/coleridge-packages/transformers-4.5.0.dev0-py3-none-any.whl\n","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:52:34.239326Z","iopub.execute_input":"2021-06-09T13:52:34.239828Z","iopub.status.idle":"2021-06-09T13:53:57.36422Z","shell.execute_reply.started":"2021-06-09T13:52:34.239785Z","shell.execute_reply":"2021-06-09T13:53:57.363253Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import os\nimport re\nimport json\nimport time\nimport datetime\nimport random\nimport glob\nimport importlib\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\n\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nrandom.seed(123)\nnp.random.seed(456)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-09T13:53:57.368464Z","iopub.execute_input":"2021-06-09T13:53:57.368805Z","iopub.status.idle":"2021-06-09T13:53:57.377174Z","shell.execute_reply.started":"2021-06-09T13:53:57.368771Z","shell.execute_reply":"2021-06-09T13:53:57.376371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load data from competition","metadata":{}},{"cell_type":"code","source":"train_path = '../input/coleridgeinitiative-show-us-the-data/train.csv'\n# Load and display the train data from train.csv\nunique = True\n\nif unique:\n    train = pd.read_csv(train_path)\n    list_unique = ['Baccalaureate and Beyond','National Assessment of Education Progress','World Ocean Database','Survey of Industrial Research and Development','Survey of Doctorate Recipients',\"COVID-19 Deaths data\",\"The National Institute on Aging Genetics of Alzheimer's Disease Data Storage Site (NIAGADS)\",'National Assessment of Education Progress','National Teacher and Principal Survey','Beginning Postsecondary Student','Rural-Urban Continuum Codes', 'NOAA Tide Gauge','National Education Longitudinal Study'] \n   \n    boolean_series_test = train.dataset_title.isin(list_unique)\n    boolean_series_train = ~train.dataset_title.isin(list_unique)\n    test = train[boolean_series_test]\n    train = train[boolean_series_train]\nelse:\n    train = pd.read_csv(train_path)\n\n#display(train)\n#display(test)\n\n# Load all the train papers into a dictionary\npaper_train_folder = '../input/coleridgeinitiative-show-us-the-data/train'\npapers = {}\nfor paper_id in tqdm(train['Id'].unique(), ascii=True, desc=\"Loading papers\"):\n    with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n        paper = json.load(f)\n        papers[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:53:57.380986Z","iopub.execute_input":"2021-06-09T13:53:57.381289Z","iopub.status.idle":"2021-06-09T13:54:04.815549Z","shell.execute_reply.started":"2021-06-09T13:53:57.38126Z","shell.execute_reply":"2021-06-09T13:54:04.813489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load and display sample submission\nsample_submission_path = '../input/coleridgeinitiative-show-us-the-data/sample_submission.csv'\nsample_submission = pd.read_csv(sample_submission_path)\ndisplay(sample_submission)\n\n# load the test data\nif validation_version:\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/train'\n    papers_test = {}\n    for paper_id in tqdm(test['Id'].unique(), ascii=True, desc=\"Loading papers\"):\n        with open(f'{paper_train_folder}/{paper_id}.json', 'r') as f:\n            paper = json.load(f)\n            papers_test[paper_id] = paper\n    sample_submission = test\nelse:\n    paper_test_folder = '../input/coleridgeinitiative-show-us-the-data/test'\n    papers_test = {}\n    for paper_id in sample_submission['Id']:\n        with open(f'{paper_test_folder}/{paper_id}.json', 'r') as f:\n            paper = json.load(f)\n            papers_test[paper_id] = paper","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:54:04.817099Z","iopub.execute_input":"2021-06-09T13:54:04.817503Z","iopub.status.idle":"2021-06-09T13:54:07.195561Z","shell.execute_reply.started":"2021-06-09T13:54:04.817462Z","shell.execute_reply":"2021-06-09T13:54:07.194706Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Literal string matching","metadata":{}},{"cell_type":"markdown","source":"### Create a set of all found datasets","metadata":{}},{"cell_type":"code","source":"all_labels = set() # all dataset names\n\nfor label_1, label_2, label_3 in train[['dataset_title', 'dataset_label', 'cleaned_label']].itertuples(index=False):\n    all_labels.add(str(label_1).lower())\n    all_labels.add(str(label_2).lower())\n    all_labels.add(str(label_3).lower())\n    \nprint(f'No. different labels: {len(all_labels)}')\nif len(all_labels) > 5:\n    print(\"Examples:\")\n    print(list(dict.fromkeys(all_labels))[:10])","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:54:07.19689Z","iopub.execute_input":"2021-06-09T13:54:07.197291Z","iopub.status.idle":"2021-06-09T13:54:07.248171Z","shell.execute_reply.started":"2021-06-09T13:54:07.197197Z","shell.execute_reply":"2021-06-09T13:54:07.247535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Matching on test data","metadata":{}},{"cell_type":"code","source":"# Cleaning text by removing all but single spaces and lowercase characters/letters\ndef clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()\n\ndef totally_clean_text(txt):\n    txt = clean_text(txt)\n    txt = re.sub(' +', ' ', txt)\n    return txt","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:54:07.251124Z","iopub.execute_input":"2021-06-09T13:54:07.251379Z","iopub.status.idle":"2021-06-09T13:54:07.257877Z","shell.execute_reply.started":"2021-06-09T13:54:07.251355Z","shell.execute_reply":"2021-06-09T13:54:07.256941Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# loop through all papers, lowercase/clean the text and match it with the set of datasets\nliteral_preds = []\n\nfor paper_id in tqdm(sample_submission['Id'], ascii=True, desc=\"matching papers\"):\n    paper = papers_test[paper_id]\n    text_1 = '. '.join(section['text'] for section in paper).lower()\n    text_2 = totally_clean_text(text_1)\n    \n    labels = set()\n    for label in all_labels:\n        if label in text_1 or label in text_2 or clean_text(label) in text_2:\n            labels.add(clean_text(label))\n    \n    literal_preds.append('|'.join(labels))\n\nprint(len(literal_preds))\nliteral_preds[:10]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:54:07.261013Z","iopub.execute_input":"2021-06-09T13:54:07.261282Z","iopub.status.idle":"2021-06-09T13:55:31.764832Z","shell.execute_reply.started":"2021-06-09T13:54:07.261257Z","shell.execute_reply":"2021-06-09T13:55:31.76418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Sci-BERT prediction","metadata":{"trusted":true}},{"cell_type":"markdown","source":"### Paths and Hyperparameters","metadata":{}},{"cell_type":"code","source":"# TODO: look into this preprocessing for prediction\nMAX_LENGTH = 64 # max no. words for each sentence.\nOVERLAP = 20 # if a sentence exceeds MAX_LENGTH, we split it to multiple sentences with overlapping\n\nPREDICT_BATCH = 64000 \n\nPRETRAINED_PATH = '../input/' + SCIBERTDATASETNAME + '/output'\nTEST_INPUT_SAVE_PATH = './input_data'\nTEST_NER_DATA_FILE = 'test_ner_input.json'\nTRAIN_PATH = '../input/' + SCIBERTDATASETNAME + '/train_ner.json'\nVAL_PATH = '../input/' + SCIBERTDATASETNAME + '/train_ner.json'\n\n# Output of the kaggle_run_ner.py predictions\nPREDICTION_SAVE_PATH = './pred'\nPREDICTION_FILE = 'test_predictions.txt'","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:55:31.766205Z","iopub.execute_input":"2021-06-09T13:55:31.766649Z","iopub.status.idle":"2021-06-09T13:55:31.773163Z","shell.execute_reply.started":"2021-06-09T13:55:31.766606Z","shell.execute_reply":"2021-06-09T13:55:31.771935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Transform data to NER format","metadata":{}},{"cell_type":"markdown","source":"Group by publication, training labels should have the same form as expected output.","metadata":{}},{"cell_type":"code","source":"# note: train there is the train.csv dataframe\ntrain = train.groupby('Id').agg({\n    'pub_title': 'first',\n    'dataset_title': '|'.join,\n    'dataset_label': '|'.join,\n    'cleaned_label': '|'.join\n}).reset_index()\n\nprint(f'No. grouped training rows: {len(train)}')\n\ndisplay(train)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:55:31.774429Z","iopub.execute_input":"2021-06-09T13:55:31.774925Z","iopub.status.idle":"2021-06-09T13:55:32.093162Z","shell.execute_reply.started":"2021-06-09T13:55:31.77489Z","shell.execute_reply":"2021-06-09T13:55:32.092339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# does NOT do lowercasing. Which is important as we use SciBERT-cased\ndef clean_SciBERT_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt)).strip()\n\n# for sliding window. Sci-BERT cannot handle too long inputs\ndef shorten_sentences(sentences):\n    short_sentences = []\n    for sentence in sentences:\n        words = sentence.split()\n        if len(words) > MAX_LENGTH:\n            for p in range(0, len(words), MAX_LENGTH - OVERLAP):\n                short_sentences.append(' '.join(words[p:p+MAX_LENGTH]))\n        else:\n            short_sentences.append(sentence)\n    return short_sentences","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:55:32.094802Z","iopub.execute_input":"2021-06-09T13:55:32.095183Z","iopub.status.idle":"2021-06-09T13:55:32.10241Z","shell.execute_reply.started":"2021-06-09T13:55:32.095147Z","shell.execute_reply":"2021-06-09T13:55:32.101397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_rows = [] # test data in NER format\npaper_length = [] # store the number of sentences each paper has\n\nfor paper_id in tqdm(sample_submission['Id'], ascii=True, desc=\"preprocessing papers\"): # loop over test papers\n    # load paper\n    paper = papers_test[paper_id]\n    \n    # extract sentences from the text part of the paper\n    sentences = [clean_SciBERT_text(sentence) for section in paper \n                 for sentence in section['text'].split('.')\n                ]\n    sentences = shorten_sentences(sentences) # make sentences short\n    #sentences = [sentence for sentence in sentences if len(sentence) > 10] # only accept sentences with length > 10 words\n    \n    # only look at sentences which contain these words\n    sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study', 'statistics', 'compilation', 'dossier', 'dataset', 'reports', 'studies', 'measurements', 'file', 'archive', 'set', 'public', 'toy', 'synthetic'])]\n    #sentences = [sentence for sentence in sentences if any(word in sentence.lower() for word in ['data', 'study'])]\n    \n    # collect all sentences in json\n    for sentence in sentences:\n        sentence_words = sentence.split()\n        dummy_tags = ['O']*len(sentence_words) # TODO: figure out why this happens\n        test_rows.append({'tokens' : sentence_words, 'tags' : dummy_tags})\n    \n    # track which sentence belongs to which data point\n    paper_length.append(len(sentences))\n    \nprint(f'total number of sentences: {len(test_rows)}')","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:55:32.105962Z","iopub.execute_input":"2021-06-09T13:55:32.106338Z","iopub.status.idle":"2021-06-09T13:56:18.834312Z","shell.execute_reply.started":"2021-06-09T13:55:32.106301Z","shell.execute_reply":"2021-06-09T13:56:18.833428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Do predict and collect results using kaggle_run_ner.py","metadata":{}},{"cell_type":"code","source":"os.environ[\"MODEL_PATH\"] = f\"{PRETRAINED_PATH}\"\nos.environ[\"TRAIN_FILE\"] = f\"{TRAIN_PATH}\"\nos.environ[\"VALIDATION_FILE\"] = f\"{VAL_PATH}\"\nos.environ[\"TEST_FILE\"] = f\"{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}\"\nos.environ[\"OUTPUT_DIR\"] = f\"{PREDICTION_SAVE_PATH}\"","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:56:18.835895Z","iopub.execute_input":"2021-06-09T13:56:18.836507Z","iopub.status.idle":"2021-06-09T13:56:18.842048Z","shell.execute_reply.started":"2021-06-09T13:56:18.836468Z","shell.execute_reply":"2021-06-09T13:56:18.840904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# copy my_seqeval.py to the working directory because the input directory is non-writable\n!cp /kaggle/input/coleridge-packages/my_seqeval.py ./\n\n# make necessary directories and files\nos.makedirs(TEST_INPUT_SAVE_PATH, exist_ok=True)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:56:18.843376Z","iopub.execute_input":"2021-06-09T13:56:18.843725Z","iopub.status.idle":"2021-06-09T13:56:19.515618Z","shell.execute_reply.started":"2021-06-09T13:56:18.84369Z","shell.execute_reply":"2021-06-09T13:56:19.51464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#--seed 123 \\\ndef bert_predict():\n    !python ../input/kaggle-ner-utils/kaggle_run_ner.py \\\n    --model_name_or_path \"$MODEL_PATH\" \\\n    --train_file \"$TRAIN_FILE\" \\\n    --validation_file \"$VALIDATION_FILE\" \\\n    --test_file \"$TEST_FILE\" \\\n    --output_dir \"$OUTPUT_DIR\" \\\n    --report_to 'none' \\\n    --seed 123 \\\n    --do_predict","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:56:19.519143Z","iopub.execute_input":"2021-06-09T13:56:19.51943Z","iopub.status.idle":"2021-06-09T13:56:19.524736Z","shell.execute_reply.started":"2021-06-09T13:56:19.5194Z","shell.execute_reply":"2021-06-09T13:56:19.523818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load the outputs from the prediction file\nbert_outputs = [] # these contain the ner labels for each sentence\n\nfor batch_begin in range(0, len(test_rows), PREDICT_BATCH):\n    # write data rows to input file\n    with open(f'{TEST_INPUT_SAVE_PATH}/{TEST_NER_DATA_FILE}', 'w') as f:\n        for row in test_rows[batch_begin:batch_begin+PREDICT_BATCH]:\n            json.dump(row, f)\n            f.write('\\n')\n    \n    # remove output dir\n    !rm -r \"$OUTPUT_DIR\"\n    \n    # do predict\n    bert_predict()\n    \n    # read predictions\n    with open(f'{PREDICTION_SAVE_PATH}/{PREDICTION_FILE}') as f:\n        this_preds = f.read().split('\\n')[:-1]\n        bert_outputs += [pred.split() for pred in this_preds]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T13:56:19.526283Z","iopub.execute_input":"2021-06-09T13:56:19.526947Z","iopub.status.idle":"2021-06-09T14:30:06.353127Z","shell.execute_reply.started":"2021-06-09T13:56:19.526909Z","shell.execute_reply":"2021-06-09T14:30:06.352295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Restore Dataset labels from predictions","metadata":{}},{"cell_type":"code","source":"# get test sentences\ntest_sentences = [row['tokens'] for row in test_rows]\n\n#print(test_sentences)\n#print(bert_outputs)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T14:30:06.356786Z","iopub.execute_input":"2021-06-09T14:30:06.357068Z","iopub.status.idle":"2021-06-09T14:30:06.408256Z","shell.execute_reply.started":"2021-06-09T14:30:06.357037Z","shell.execute_reply":"2021-06-09T14:30:06.407413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_dataset_labels = [] # store all dataset labels for each publication\n\n# Check for B and I tags to reconstruct dataset names\nfor length in paper_length:\n    labels = set()\n    for sentence, pred in zip(test_sentences[:length], bert_outputs[:length]):\n        curr_phrase = ''\n        for word, tag in zip(sentence, pred):\n            if tag == 'B': # start a new phrase\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n                curr_phrase = word\n            elif tag == 'I' and curr_phrase: # continue the phrase\n                curr_phrase += ' ' + word\n            else: # end last phrase (if any)\n                if curr_phrase:\n                    labels.add(curr_phrase)\n                    curr_phrase = ''\n        # check if the label is the suffix of the sentence\n        if curr_phrase:\n            labels.add(curr_phrase)\n            curr_phrase = ''\n    \n    # record dataset labels for this publication\n    bert_dataset_labels.append(labels)\n    \n    del test_sentences[:length], bert_outputs[:length]\n\n#bert_dataset_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-09T14:30:06.409508Z","iopub.execute_input":"2021-06-09T14:30:06.409889Z","iopub.status.idle":"2021-06-09T14:30:08.703553Z","shell.execute_reply.started":"2021-06-09T14:30:06.409851Z","shell.execute_reply":"2021-06-09T14:30:08.702646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Filter based on Jaccard score and clean","metadata":{}},{"cell_type":"code","source":"# TODO: look into this\ndef jaccard_similarity(s1, s2):\n    l1 = s1.lower().split(\" \")\n    l2 = s2.lower().split(\" \")    \n    intersection = len(list(set(l1).intersection(l2)))\n    union = (len(l1) + len(l2)) - intersection\n    return float(intersection) / union\n\nfiltered_bert_labels = []\n#test['cleaned_label']\n\n\nif validation_version:\n    validatable_labels = []\n    truths = [x for x in test['cleaned_label']]\n    for labels in bert_dataset_labels:\n        filtered = []\n        for label in sorted(labels, key=len):\n            label = clean_text(label)\n            if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n                filtered.append(label)\n        validatable_labels.append(filtered)\n    \n    TP = 0\n    FP = 0\n    FN = 0\n    for i in range(len(validatable_labels)):\n        for p in validatable_labels[i]:\n            if jaccard_similarity(p, truths[i]) >= 0.5:\n                TP+=1\n            else:\n                FP+=1        \n        if len(validatable_labels[i]) == 0:\n            FN+=1\n    \n    print(\"score is:\")\n    print((5*TP)/((5*TP)+(4*FP)+FN))\n        \nelse:\n    for labels in bert_dataset_labels:\n        filtered = []\n\n        for label in sorted(labels, key=len):\n            label = clean_text(label)\n            if len(filtered) == 0 or all(jaccard_similarity(label, got_label) < 0.75 for got_label in filtered):\n                filtered.append(label)\n\n        filtered_bert_labels.append('|'.join(filtered))\n    \n#filtered_bert_labels[:5]","metadata":{"execution":{"iopub.status.busy":"2021-06-09T14:30:08.704936Z","iopub.execute_input":"2021-06-09T14:30:08.70546Z","iopub.status.idle":"2021-06-09T14:30:08.782049Z","shell.execute_reply.started":"2021-06-09T14:30:08.70542Z","shell.execute_reply":"2021-06-09T14:30:08.781131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Aggregate final predictions and write submission file","metadata":{}},{"cell_type":"code","source":"#final_predictions = literal_preds\nfinal_predictions = filtered_bert_labels\n\n\n\n#final_predictions = []\n#for literal_match, bert_pred in zip(literal_preds, filtered_bert_labels):\n#    if literal_match:\n#        final_predictions.append(literal_match)\n#    else:\n#        final_predictions.append(bert_pred)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T14:30:08.783556Z","iopub.execute_input":"2021-06-09T14:30:08.783905Z","iopub.status.idle":"2021-06-09T14:30:08.787642Z","shell.execute_reply.started":"2021-06-09T14:30:08.783868Z","shell.execute_reply":"2021-06-09T14:30:08.786816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['PredictionString'] = final_predictions\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-09T14:30:08.789032Z","iopub.execute_input":"2021-06-09T14:30:08.789517Z","iopub.status.idle":"2021-06-09T14:30:08.821743Z","shell.execute_reply.started":"2021-06-09T14:30:08.789479Z","shell.execute_reply":"2021-06-09T14:30:08.820161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_csv(f'submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-09T14:30:08.822905Z","iopub.status.idle":"2021-06-09T14:30:08.82364Z"},"trusted":true},"execution_count":null,"outputs":[]}]}