{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Coleridge Initiative - Show US the Data","metadata":{}},{"cell_type":"markdown","source":"I tried two approaches (no dataset label string matching)\n\n* Regular expression - 0.5/0.56\n* BERT NER - 0.37/0.52\n\nEnsemble, searching among known datasets and other approaches with BERT did not improve the results\n\n## Regular expressions\n* Looking for uppercase letters\n* Looking for nearby words with these letters\n* Just looking for words beginning with uppercase letters\n* I memorize all found and search among other documents\n## BERT NER\n* I divide the document into sentences of 200-400 characters\n* Select potential candidates\n* I take 90% of sentences with tags and 10% without tags\n* Use 3 classes - no class, first word and last word of dataset name\n* Predict and select good candidates","metadata":{}},{"cell_type":"markdown","source":"# Moduls","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore', category=DeprecationWarning)\nwarnings.filterwarnings('ignore', category=FutureWarning)\nimport sys\nsys.path.insert(0, \"../input/transformers/transformers-master/\")\n\nfrom collections import Counter\nimport os, re, json, torch\n\nfrom functools import partial\n\nimport pandas as pd\nimport numpy as np\n\nimport Levenshtein\n\nfrom transformers import BertTokenizer,BertTokenizerFast, BertForSequenceClassification, BertConfig, BertForTokenClassification,AutoModelForTokenClassification","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-23T05:32:39.434008Z","iopub.execute_input":"2021-06-23T05:32:39.43484Z","iopub.status.idle":"2021-06-23T05:32:41.580524Z","shell.execute_reply.started":"2021-06-23T05:32:39.434717Z","shell.execute_reply":"2021-06-23T05:32:41.579771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"code","source":"# reading csv files and train & test file paths\ntrain_df = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/train.csv')\nsample_sub = pd.read_csv('../input/coleridgeinitiative-show-us-the-data/sample_submission.csv')\ntrain_files_path = '../input/coleridgeinitiative-show-us-the-data/train'\ntest_files_path = '../input/coleridgeinitiative-show-us-the-data/test'","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-23T05:32:41.583215Z","iopub.execute_input":"2021-06-23T05:32:41.583493Z","iopub.status.idle":"2021-06-23T05:32:41.700673Z","shell.execute_reply.started":"2021-06-23T05:32:41.583444Z","shell.execute_reply":"2021-06-23T05:32:41.699905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_append_return(filename, train_files_path=train_files_path, output='text'):\n    \"\"\"\n    Function to read json file and then return the text data from them and append to the dataframe\n    \"\"\"\n    json_path = os.path.join(train_files_path, (filename+'.json'))\n    headings = []\n    contents = []\n    combined = []\n    with open(json_path, 'r') as f:\n        json_decode = json.load(f)\n        for data in json_decode:\n            headings.append(data.get('section_title'))\n            contents.append(data.get('text'))\n            combined.append(data.get('section_title'))\n            combined.append(data.get('text'))\n    \n    all_headings = ' '.join(headings)\n    all_contents = ' '.join(contents)\n    all_data = '. '.join(combined)\n    \n    if output == 'text':\n        return all_contents\n    elif output == 'head':\n        return all_headings\n    else:\n        return all_data","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-23T05:32:41.701957Z","iopub.execute_input":"2021-06-23T05:32:41.702298Z","iopub.status.idle":"2021-06-23T05:32:41.710497Z","shell.execute_reply.started":"2021-06-23T05:32:41.702263Z","shell.execute_reply":"2021-06-23T05:32:41.709515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_text(txt):\n    return re.sub('[^A-Za-z0-9]+', ' ', str(txt).lower()).strip()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2021-06-23T05:32:41.712255Z","iopub.execute_input":"2021-06-23T05:32:41.712875Z","iopub.status.idle":"2021-06-23T05:32:41.718241Z","shell.execute_reply.started":"2021-06-23T05:32:41.712837Z","shell.execute_reply":"2021-06-23T05:32:41.717313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def jaccard_similarity(str1: str, str2: str) -> float:\n    a = set(str1.split()) \n    b = set(str2.split())\n    c = a.intersection(b)\n    return float(len(c)) / (len(a) + len(b) - len(c))","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.721527Z","iopub.execute_input":"2021-06-23T05:32:41.72185Z","iopub.status.idle":"2021-06-23T05:32:41.727326Z","shell.execute_reply.started":"2021-06-23T05:32:41.721791Z","shell.execute_reply":"2021-06-23T05:32:41.726445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def text_cleaning(text):\n    '''\n    Converts all text to lower case, Removes special charecters, emojis and multiple spaces\n    text - Sentence that needs to be cleaned\n    '''\n    text = re.sub('[^A-Za-z0-9\\.\\:\\,\\!\\?\\;\\&]+', ' ', str(text)).strip()\n    emoji_pattern = re.compile(\"[\"\n                               u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                               u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                               u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                               u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                               \"]+\", flags=re.UNICODE)\n    text = emoji_pattern.sub(r'', text)\n    text = re.sub('\\s', ' ', text)\n    return text.strip()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.729514Z","iopub.execute_input":"2021-06-23T05:32:41.729955Z","iopub.status.idle":"2021-06-23T05:32:41.738059Z","shell.execute_reply.started":"2021-06-23T05:32:41.729911Z","shell.execute_reply":"2021-06-23T05:32:41.737386Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#train_df['text'] = train_df['Id'].apply(read_append_return)\nsample_sub['text'] = sample_sub['Id'].apply(read_append_return, train_files_path=test_files_path)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.740694Z","iopub.execute_input":"2021-06-23T05:32:41.741049Z","iopub.status.idle":"2021-06-23T05:32:41.787634Z","shell.execute_reply.started":"2021-06-23T05:32:41.741015Z","shell.execute_reply":"2021-06-23T05:32:41.786777Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# re.search","metadata":{}},{"cell_type":"code","source":"sub = {}\nnew_existing_labels = []\ng2 = ['survey',\"study\",\"database\",\"data\",\"dataset\",\"initiative\",'research']\ng = ['survey',\"study\",\"database\",\"dataset\"]\nbad_w = ['consortium','organization','bureau','development','center','table','department','university','bank','class','user'\n         'appendix','supplementary','supplement','major','association','journal','commission','associates','board','agency',\n        'administration','federation','ministry','form','score','management','accounts','account','feasibility']\nbad1 = ['USGS','GWAS','ECLS','aDAS','NCDC','NDBC','UDS','GTD','ISC','DGP','EDC','FDA','TSE','DEA','CDA','IDB','NGDC','JODC','EDM','FADN','LRD','DBDM','DMC','WSC']\n\nfor index, row in sample_sub.iterrows():#sample_sub train_df.iloc[2019:2020]\n    sub[row['Id']] = ''\n    sample_text = text_cleaning(row['text'])\n    sample_text1 = sample_text.replace(\",\",\"\").lower()\n    \n    b = []\n    c = []    \n    for i,cl in enumerate(re.finditer(r'[a-z]*[A-Z]{2,5}[a-z]*[A-Z]{1,5}', sample_text)):\n        if cl[0].upper() in bad1:\n            continue\n        x1 = \"[a-z ]+\".join(list(cl[0].upper()))+'[a-z]+(?: [Dd]ata| [Ss]urvey| [Ss]ample| [Ss]tudy)*'\n\n        s = False\n        ans = re.search(x1, sample_text[cl.start()-100:cl.start()])\n        if ans:\n            s = clean_text(ans[0])\n        if s:\n            it_bad = False\n            for w2 in s.split():\n                if w2 in bad_w: \n                    it_bad = True\n            if not it_bad:\n                if s.split()[-1] in g2:\n                    b.append(clean_text(s)) #.rstrip(\"s\")\n                    c.append(clean_text(cl[0]))\n\n    \n\n    for s in c:\n        if len(sub[row['Id']]) and not s in sub[row['Id']]:\n            sub[row['Id']] = sub[row['Id']] + '|'+ s\n        elif not len(sub[row['Id']]):\n            sub[row['Id']] = s\n    \n    for s in b:\n        if len(sub[row['Id']]) and not s in sub[row['Id']]:\n            it_bad = False\n            for p in sub[row['Id']].split(\"|\"):\n                if p in s:\n                    it_bad = True\n                    break\n\n            if not it_bad:\n                sub[row['Id']] = sub[row['Id']] + '|'+ s\n                new_existing_labels.append(s)\n        elif not len(sub[row['Id']]):\n            sub[row['Id']] = s\n            new_existing_labels.append(s)\n\n\n    a = re.findall(r'(?<=[^\\.] )[A-Z][a-z]{3,20} (?:(?:[A-Z][a-z]{2,20}|of|up|to|and|the|in|on|COVID-19|s|for)[- \\.,]){0,10}(?:[A-Z][a-z]{3,20})(?: data| survey| sample| study)*', sample_text)\n    a = [s for s in a if len(re.findall(r'[A-Z]',s))>2]\n    cnt = Counter(a).most_common()\n    \n    cnt = {k[0]:k[1] for k in cnt if k[1] > 0}\n    \n    for s in cnt.keys():\n        s = clean_text(s)\n        if \"\".join([w[0] for w in s.split()]).upper() in bad1:\n            continue\n            \n        for w in s.split():\n            if w in g:\n                it_bad = False\n                for w2 in s.split():\n                    if w2 in bad_w: \n                        it_bad = True\n                if not it_bad:\n                    if not len(sub[row['Id']]):\n                        sub[row['Id']] = s\n                        new_existing_labels.append(s)\n                    elif len(sub[row['Id']]) and not s in sub[row['Id']]:\n                        for p in sub[row['Id']].split(\"|\"):\n                            if p in s or Levenshtein.distance(p,s)/len(s)<0.2:\n                                it_bad = True\n                                break\n                        if not it_bad:\n                            sub[row['Id']] = sub[row['Id']] + '|'+ s\n                            new_existing_labels.append(s)\n\n                    break\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.788951Z","iopub.execute_input":"2021-06-23T05:32:41.789479Z","iopub.status.idle":"2021-06-23T05:32:41.803878Z","shell.execute_reply.started":"2021-06-23T05:32:41.789427Z","shell.execute_reply":"2021-06-23T05:32:41.803144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ncnt = Counter(new_existing_labels).most_common()\ncnt = {k[0]:k[1] for k in cnt if k[1] > 1}\nnew_existing_labels = list(cnt.keys())\n\nnew_existing_labels = sorted(new_existing_labels, key=len)[::-1]\n\nfor index, row in sample_sub.iterrows():\n    if not row['Id'] in sub:\n        sub[row['Id']] = ''    \n    sample_text = text_cleaning(row['text'])\n    for s in new_existing_labels:#\n        if ' ' in s and s in sample_text.lower():\n            if len(sub[row['Id']]) and not s in sub[row['Id']]:\n                it_bad = False\n                for p in sub[row['Id']].split(\"|\"):\n                    if p in s or Levenshtein.distance(p,s)/len(s)<0.2:\n                        it_bad = True\n                if not it_bad:\n                    sub[row['Id']] = sub[row['Id']] + '|'+ s\n            elif not len(sub[row['Id']]):\n                sub[row['Id']] = s\n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.80487Z","iopub.execute_input":"2021-06-23T05:32:41.80573Z","iopub.status.idle":"2021-06-23T05:32:41.816582Z","shell.execute_reply.started":"2021-06-23T05:32:41.805703Z","shell.execute_reply":"2021-06-23T05:32:41.815771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# BertForTokenClassification","metadata":{}},{"cell_type":"code","source":"class VTBDataset(torch.utils.data.Dataset):\n    def __init__(self, encodings, size_s, labels = None):\n        self.encodings = encodings\n        self.labels = labels\n        self.size_s = size_s\n    def __getitem__(self, idx):\n        self.item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n        if self.labels:\n            self.item['labels'] = torch.tensor(self.labels[idx])\n        return self.item\n\n    def __len__(self):\n        return self.size_s","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.817921Z","iopub.execute_input":"2021-06-23T05:32:41.8182Z","iopub.status.idle":"2021-06-23T05:32:41.828251Z","shell.execute_reply.started":"2021-06-23T05:32:41.818176Z","shell.execute_reply":"2021-06-23T05:32:41.827317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def encode_tags(labels, encodings):\n    encoded_labels = []\n    for doc_labels, doc_offset in zip(labels, encodings.offset_mapping):\n        # create an empty array of -100\n        doc_enc_labels = np.ones(len(doc_offset),dtype=int) * -100\n        arr_offset = np.array(doc_offset)\n        # set labels whose first offset position is 0 and the second is not 0\n        doc_enc_labels[(arr_offset[:,0] == 0) & (arr_offset[:,1] != 0)] = doc_labels[:sum((arr_offset[:,0] == 0) & (arr_offset[:,1] != 0))]\n\n        encoded_labels.append(doc_enc_labels.tolist())\n    return encoded_labels","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.829523Z","iopub.execute_input":"2021-06-23T05:32:41.829966Z","iopub.status.idle":"2021-06-23T05:32:41.839022Z","shell.execute_reply.started":"2021-06-23T05:32:41.82993Z","shell.execute_reply":"2021-06-23T05:32:41.838279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BERT_MODEL_PATH = '../input/bert-base-cased'\ntokenizer = BertTokenizerFast.from_pretrained(BERT_MODEL_PATH,do_lower_case=False)\nbatch_size_test = 100\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nSigmoid = torch.nn.Sigmoid()","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.840624Z","iopub.execute_input":"2021-06-23T05:32:41.84102Z","iopub.status.idle":"2021-06-23T05:32:41.988953Z","shell.execute_reply.started":"2021-06-23T05:32:41.840983Z","shell.execute_reply":"2021-06-23T05:32:41.988029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Предсказание","metadata":{}},{"cell_type":"code","source":"\nmodel = BertForTokenClassification.from_pretrained(BERT_MODEL_PATH,num_labels=3)\nmodel = model.to(device)\n\nckpt = torch.load(\"../input/07-model/0.94model.bin\",map_location=device)\nmodel.load_state_dict(ckpt)   \nmodel.eval()\ngood_word = ['resource','report','research','survey','agriculture','service',\"study\",\"database\",\"program\",\"data\",\"dataset\",\"assessment\",'monitoring','surveys','initiative','system','student',\n    'observation','census','directory','reports','statistics','codes','student','students','baccalaureate','sample','project','initiatives']\nfor index, row in sample_sub.iterrows():#\n    ans = []\n    sample_text = text_cleaning(row['text'])\n    sample_text = sample_text.split(\". \")\n    \n    sent = ''\n    new_s = []\n    for i,sentens in enumerate(sample_text):\n        if not sent:\n            \n            if len(sentens)>200 and len(sentens)<400:\n                new_s.append(sentens)\n            elif len(sentens)>=400:\n                new_s.extend(re.findall(r'(?: |^).{0,150}[A-Z][a-z]{2,20} (?:(?:[A-Z][a-z]{2,20}|of|up|to|and|the|in|on|COVID-19|s|for|[0-9]{4}})[- \\.,]){0,10}(?:[A-Z][a-z]{2,20})(?: data| survey| sample| study| [0-9]{2,4})*.{0,150}(?:[\\. ]|$)', sentens))\n                new_s.extend(re.findall(r'(?: |^).{0,200}(?: [Dd]ata| [Rr]egistry|[Gg]enome [Ss]equence| [Mm]odel| [Ss]tudy| [Ss]urvey).{0,200}(?:[\\. ]|$)', sentens))\n                new_s.extend(re.findall(r'(?: |^).{0,200}[A-Z]{4,10}.{0,200}(?:[\\. ]|$)', sentens))                \n            else:\n                sent = sentens\n        else:\n            if len(sent + sentens) >= 400:\n                new_s.append(sent)\n                sent = ''\n                if len(sentens)>200 and len(sentens)<400:\n                    new_s.append(sentens)\n                elif len(sentens)>=400:\n                    new_s.extend(re.findall(r'(?: |^).{0,150}[A-Z][a-z]{2,20} (?:(?:[A-Z][a-z]{2,20}|of|up|to|and|the|in|on|COVID-19|s|for|[0-9]{4}})[- \\.,]){0,10}(?:[A-Z][a-z]{2,20})(?: data| survey| sample| study| [0-9]{2,4})*.{0,150}(?:[\\. ]|$)', sentens))\n                    new_s.extend(re.findall(r'(?: |^).{0,200}(?: [Dd]ata| [Rr]egistry|[Gg]enome [Ss]equence| [Mm]odel| [Ss]tudy| [Ss]urvey).{0,200}(?:[\\. ]|$)', sentens))\n                    new_s.extend(re.findall(r'(?: |^).{0,200}[A-Z]{4,10}.{0,200}(?:[\\. ]|$)', sentens))    \n                else:\n                    sent = sentens\n            else:\n                sent = sent +'. ' + sentens\n    if sent:\n        new_s.append(sent)\n    \n    new_s_2 = []\n    for s in new_s:\n        \n        a = re.findall(r'(?:(?:[A-Z][a-z]{2,20}|of|in|COVID-19|s|for|and) ){3,6}', s)\n        \n        a.extend(re.findall(r'(?: [Dd]ata| [Rr]egistry|[Gg]enome [Ss]equence| [Mm]odel| [Ss]tudy| [Ss]urvey)', s))\n        a.extend(re.findall(r'[A-Z]{4,10}', s))\n        if a:\n            new_s_2.append(s)\n    if new_s_2:\n        t_x = [s.split() for s in new_s_2]\n        valid_y = [[1]*len(x) for x in t_x]\n        \n        val_encodings = tokenizer(t_x, is_split_into_words=True, return_offsets_mapping=True, padding=True, truncation=True,max_length = 256)    \n        val_labels = encode_tags(valid_y, val_encodings)    \n        val_encodings.pop(\"offset_mapping\")\n        \n        valid_dataset = VTBDataset(val_encodings,len(t_x),val_labels)\n        valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size_test, shuffle=False)\n        \n        valid = pd.DataFrame()\n        valid['val_labels'] = val_encodings['input_ids']\n        val_labels = len(val_encodings['input_ids'])\n        len_val = len(val_encodings['input_ids'][0])\n\n        valid_preds1 = np.zeros((val_labels,len_val), dtype = np.float32)\n        valid_preds2 = np.zeros((val_labels,len_val), dtype = np.float32)\n\n        avg_accuracy = 0.\n        with torch.no_grad():\n            for i,(batch)  in enumerate(valid_loader):\n                input_ids = batch['input_ids'].to(device)\n                attention_mask = batch['attention_mask'].to(device)\n                labels = batch['labels']\n                outputs = model(input_ids, attention_mask=attention_mask, labels=None)\n\n                logits1 = outputs[0][:,:,1].detach()\n                logits1[labels<0] = -10\n                logits2 = outputs[0][:,:,2].detach()\n                logits2[labels<0] = -10                   \n                valid_preds2[i*batch_size_test:(i+1)*batch_size_test,:]=logits2.cpu().numpy()  \n                valid_preds1[i*batch_size_test:(i+1)*batch_size_test,:]=logits1.cpu().numpy()                             \n\n    \n    ans = []\n    for index, row_1 in valid.iterrows():\n    \n        preds1, = np.where(valid_preds1[index]>1)\n        preds2, = np.where(valid_preds2[index]>0)\n        \n        preds2.sort()\n        max_a = 0\n        g_all = []\n        for min_a in preds1:\n            if max_a > min_a:\n                continue\n            g = ''\n            max_a = 0\n            for min_b in preds2:\n                if min_b>min_a:\n                    max_a = min_b\n                    break\n            if max_a == 0 and valid_preds1[index][min_a]>2:\n                max_a = min_a  \n            if max_a-min_a > 10:\n                continue\n            if max_a>=min_a and min_a>0:\n                k = 0\n                  \n                b = np.array(row_1[\"val_labels\"])\n\n                s = tokenizer.convert_ids_to_tokens(b[min_a:])\n                for j,w in enumerate(s):\n                    \n                    if j<=max_a - min_a or \"##\" in w  :\n                        g += w + ' '\n                    else:    \n                        break\n            g = g.replace(\" ##\",\"\").strip()\n            \n            if g and sum(map(str.isupper,g))/len(g.split())>0.5:\n                g = clean_text(g)\n                it_bad = False\n                for w2 in g.split():\n                    if w2 in bad_w: \n                        it_bad = True\n                if not it_bad:\n                    for w in g.split():\n                        if w in good_word or len(g.split())==1:\n                            g_all.append(g)\n                            break            \n    \n        ans.extend(g_all)\n\n    if not row['Id'] in sub:\n        sub[row['Id']] = ''\n    for s in ans:\n        if len(sub[row['Id']]) and not s in sub[row['Id']]:\n            it_bad = False\n            for p in sub[row['Id']].split(\"|\"):\n                if p in s or Levenshtein.distance(p,s)/len(s)<0.2:\n                    it_bad = True\n            if not it_bad:              \n                sub[row['Id']] = sub[row['Id']] + '|'+ s\n        elif not len(sub[row['Id']]):\n            sub[row['Id']] = s    \n","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:32:41.992195Z","iopub.execute_input":"2021-06-23T05:32:41.992473Z","iopub.status.idle":"2021-06-23T05:33:04.110715Z","shell.execute_reply.started":"2021-06-23T05:32:41.992436Z","shell.execute_reply":"2021-06-23T05:33:04.109956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Answer","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame.from_dict(sub, orient='index', columns=['PredictionString']).reset_index()\nsubmission.columns = [\"Id\",\"PredictionString\"]\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-23T05:33:04.112036Z","iopub.execute_input":"2021-06-23T05:33:04.112355Z","iopub.status.idle":"2021-06-23T05:33:04.383041Z","shell.execute_reply.started":"2021-06-23T05:33:04.112322Z","shell.execute_reply":"2021-06-23T05:33:04.382297Z"},"trusted":true},"execution_count":null,"outputs":[]}]}