{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\n# import os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"fdf0cc74-07c7-4559-81e2-6ba7ef20505f","_cell_guid":"57b584ae-49f6-45ee-bc13-04c7ff359395","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:17:56.737358Z","iopub.execute_input":"2022-06-30T06:17:56.737969Z","iopub.status.idle":"2022-06-30T06:17:56.762396Z","shell.execute_reply.started":"2022-06-30T06:17:56.737860Z","shell.execute_reply":"2022-06-30T06:17:56.761563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport random\n# import re\n# import time\n# import pickle\nimport glob\nimport json\nfrom typing import Optional, Tuple\n\n# import numpy as np\nimport pandas as pd\n# from matplotlib import pyplot as plt\nfrom tqdm.notebook import tqdm\n\nimport torch\nfrom torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n# from torch.utils.data import random_split\n# import torch.nn.functional as f\nfrom torch.nn import MSELoss\nimport torch.nn as nn\n# from torch.optim import AdamW\n\nimport transformers\nfrom transformers import RobertaModel, RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig\nfrom transformers import AutoModel, AutoModelForSequenceClassification, AutoTokenizer\n# from transformers import TrainingArguments, Trainer\n# from transformers import get_scheduler\n\nimport datasets\nfrom datasets import load_metric\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold, GroupShuffleSplit\n# from sklearn.metrics import confusion_matrix, classification_report, f1_score, recall_score, precision_score, accuracy_score\n# from sklearn.utils.class_weight import compute_class_weight\n\nprint(transformers.__version__)","metadata":{"_uuid":"89b7f705-63bf-4d1d-b84a-5e292ba3e5e8","_cell_guid":"33b20ae5-84f7-45f0-abf0-f1dfb698be1c","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:17:56.858200Z","iopub.execute_input":"2022-06-30T06:17:56.858537Z","iopub.status.idle":"2022-06-30T06:18:05.008401Z","shell.execute_reply.started":"2022-06-30T06:17:56.858506Z","shell.execute_reply":"2022-06-30T06:18:05.006228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\"","metadata":{"_uuid":"fbdcb32d-b75f-4a3b-999d-0efb01236de5","_cell_guid":"341b572f-e38c-48e6-90cd-dac892dbbbc2","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.010554Z","iopub.execute_input":"2022-06-30T06:18:05.011219Z","iopub.status.idle":"2022-06-30T06:18:05.016423Z","shell.execute_reply.started":"2022-06-30T06:18:05.011179Z","shell.execute_reply":"2022-06-30T06:18:05.015005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seeds(seed=42):\n    random.seed(seed)\n#     np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.cuda.manual_seed_all(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = False\nset_seeds()","metadata":{"_uuid":"c6bc4e46-69b1-4616-88c4-d18d0b57651a","_cell_guid":"5c5c62f5-33b7-4067-955b-ba559f9282a5","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.018375Z","iopub.execute_input":"2022-06-30T06:18:05.018967Z","iopub.status.idle":"2022-06-30T06:18:05.030301Z","shell.execute_reply.started":"2022-06-30T06:18:05.018924Z","shell.execute_reply":"2022-06-30T06:18:05.029420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\ndevice","metadata":{"_uuid":"9577339b-64f8-4b29-9c6a-38f35d230cc9","_cell_guid":"5363d04c-5934-4860-8977-f204d0fa4d29","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.033351Z","iopub.execute_input":"2022-06-30T06:18:05.033767Z","iopub.status.idle":"2022-06-30T06:18:05.105264Z","shell.execute_reply.started":"2022-06-30T06:18:05.033710Z","shell.execute_reply":"2022-06-30T06:18:05.104330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load dataset","metadata":{"_uuid":"101c6930-0fca-4ed2-b7da-729dba4565ad","_cell_guid":"892fb94b-cc0c-4b1a-9f81-4206b23e085f","trusted":true}},{"cell_type":"code","source":"def read_notebook(path: str) -> pd.DataFrame:\n    with open(path) as file:\n        df = pd.DataFrame(json.load(file))\n    df[\"id\"] = os.path.splitext(os.path.basename(path))[0]\n    return df\n\ndef expand_order(row: Tuple[str, str]) -> pd.DataFrame:\n    cell_ids = row[1].split(\" \")\n    df = pd.DataFrame(\n        {\n            \"id\": [row[0] for _ in range(len(cell_ids))],\n            \"cell_id\": cell_ids,\n            \"rank\": range(len(cell_ids)),\n        }\n    )\n    df[\"pct_rank\"] = df[\"rank\"] / len(df)\n    return df","metadata":{"_uuid":"2e4a5389-14ba-4e85-bf23-3413e48cf3e5","_cell_guid":"078a5673-0d59-4569-9443-0275eb058d30","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.107125Z","iopub.execute_input":"2022-06-30T06:18:05.107612Z","iopub.status.idle":"2022-06-30T06:18:05.120618Z","shell.execute_reply.started":"2022-06-30T06:18:05.107574Z","shell.execute_reply":"2022-06-30T06:18:05.119131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BASE_MODEL = '../input/robertabase/roberta-base'\nSEQ_LEN = 512","metadata":{"_uuid":"e5e52a87-2eb5-4ca7-8050-066f6d30db85","_cell_guid":"dc73ce0a-70eb-4c43-9af7-d356b531bacd","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.123578Z","iopub.execute_input":"2022-06-30T06:18:05.125036Z","iopub.status.idle":"2022-06-30T06:18:05.132996Z","shell.execute_reply.started":"2022-06-30T06:18:05.124991Z","shell.execute_reply":"2022-06-30T06:18:05.131391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = transformers.AutoTokenizer.from_pretrained(BASE_MODEL, do_lower_case=True)","metadata":{"_uuid":"96a91750-a2ca-4681-9e9a-731bdfdaf215","_cell_guid":"b20278ff-f644-423b-90ca-41a5be7324a3","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.134647Z","iopub.execute_input":"2022-06-30T06:18:05.135127Z","iopub.status.idle":"2022-06-30T06:18:05.266188Z","shell.execute_reply.started":"2022-06-30T06:18:05.135088Z","shell.execute_reply":"2022-06-30T06:18:05.265163Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_encoding(source):\n    return tokenizer(\n        source,\n        add_special_tokens=True,\n        max_length = SEQ_LEN,\n        truncation = True,\n        padding = \"max_length\",\n        return_tensors = 'pt'\n      )","metadata":{"execution":{"iopub.status.busy":"2022-06-30T06:18:05.267448Z","iopub.execute_input":"2022-06-30T06:18:05.267823Z","iopub.status.idle":"2022-06-30T06:18:05.274636Z","shell.execute_reply.started":"2022-06-30T06:18:05.267780Z","shell.execute_reply":"2022-06-30T06:18:05.273535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomDataset(torch.utils.data.Dataset):\n    def __init__(self, texts, labels=None):\n        self.texts = texts\n        self.labels = labels\n\n    def __getitem__(self, idx):\n        item = create_encoding([self.texts[idx]])\n#         item = {key: val[idx].clone().detach() for key, val in encodings.items()}\n        if self.labels is not None:\n            item['pct_rank'] = torch.tensor(self.labels[idx])\n        return item\n\n    def __len__(self):\n        return len(self.texts)","metadata":{"_uuid":"f470b23d-fcf3-4ff6-b124-ef1d4a89543d","_cell_guid":"4953496a-19ac-41e5-82e8-7131a6677fee","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.276408Z","iopub.execute_input":"2022-06-30T06:18:05.277525Z","iopub.status.idle":"2022-06-30T06:18:05.285445Z","shell.execute_reply.started":"2022-06-30T06:18:05.277488Z","shell.execute_reply":"2022-06-30T06:18:05.284498Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"batch_size = 2","metadata":{"_uuid":"a6583b8b-eb09-4986-8869-f758d5631de6","_cell_guid":"cd530420-ba90-41c6-b077-d7d7fd6d2a58","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.290499Z","iopub.execute_input":"2022-06-30T06:18:05.290814Z","iopub.status.idle":"2022-06-30T06:18:05.295371Z","shell.execute_reply.started":"2022-06-30T06:18:05.290789Z","shell.execute_reply":"2022-06-30T06:18:05.294315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create_encoding(\"my name is\")['input_ids'].shape\n# # for k, v in create_encoding(\"my name is\"):\n# #     print(k)\n# #     print(v)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T06:18:05.296713Z","iopub.execute_input":"2022-06-30T06:18:05.297797Z","iopub.status.idle":"2022-06-30T06:18:05.305032Z","shell.execute_reply.started":"2022-06-30T06:18:05.297759Z","shell.execute_reply":"2022-06-30T06:18:05.304206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for k,v in create_encoding([\"my name is\", 'nam']).items():\n#     print(k)\n#     print(v)\n#     print(v.shape)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T06:18:05.306467Z","iopub.execute_input":"2022-06-30T06:18:05.306935Z","iopub.status.idle":"2022-06-30T06:18:05.314164Z","shell.execute_reply.started":"2022-06-30T06:18:05.306899Z","shell.execute_reply":"2022-06-30T06:18:05.313181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# fine-tune","metadata":{"_uuid":"d2d50111-2b5e-4dce-a47a-5a4dac49e57d","_cell_guid":"64d8c38f-12de-40bc-b380-449f96c80a90","trusted":true}},{"cell_type":"code","source":"class RobertaClassificationHead(nn.Module):\n    \"\"\"Head for sentence-level classification tasks.\"\"\"\n\n    def __init__(self, config):\n        super().__init__()\n        self.dense = nn.Linear(config.hidden_size, config.hidden_size)\n        classifier_dropout = (\n            config.classifier_dropout if config.classifier_dropout is not None else config.hidden_dropout_prob\n        )\n        self.dropout = nn.Dropout(classifier_dropout)\n        self.out_proj = nn.Linear(config.hidden_size, config.num_labels)\n\n    def forward(self, features, **kwargs):\n        x = features[:, 0, :]  # take <s> token (equiv. to [CLS])\n        x = self.dropout(x)\n        x = self.dense(x)\n        x = torch.tanh(x)\n        x = self.dropout(x)\n        x = self.out_proj(x)\n        return x","metadata":{"_uuid":"97041eac-d4fa-4c02-a570-4183e08275ca","_cell_guid":"79b29c1b-2136-462e-9c34-22ca1084db48","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.315803Z","iopub.execute_input":"2022-06-30T06:18:05.316045Z","iopub.status.idle":"2022-06-30T06:18:05.327363Z","shell.execute_reply.started":"2022-06-30T06:18:05.316023Z","shell.execute_reply":"2022-06-30T06:18:05.326484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CustomModel(nn.Module):\n    def __init__(self, model_name):\n        super(CustomModel, self).__init__()\n        self.num_labels = 1\n        self.roberta = AutoModel.from_pretrained(model_name, add_pooling_layer=False)\n        self.config = RobertaConfig.from_pretrained(model_name)\n        self.config.num_labels = self.num_labels\n        self.classifier=RobertaClassificationHead(self.config)\n        \n    def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, output_attentions=None, output_hidden_states=None, return_dict=None, pct_rank=None):\n        outputs = self.roberta(\n            input_ids,\n            attention_mask=attention_mask,\n            token_type_ids=token_type_ids,\n            position_ids=position_ids,\n            head_mask=head_mask,\n            inputs_embeds=inputs_embeds,\n            output_attentions=output_attentions,\n            output_hidden_states=output_hidden_states,\n            return_dict=return_dict\n        )\n        sequence_output = outputs[0]\n        logits = self.classifier(sequence_output)\n        \n        loss = None\n        if pct_rank is not None:\n            loss_fct = MSELoss()\n            loss = loss_fct(logits.squeeze(), pct_rank)\n        \n        return loss, logits","metadata":{"_uuid":"dc3a047f-c411-4563-99f0-b085ba65667f","_cell_guid":"9146d26b-f030-42ac-b201-a23a6fd12516","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.328618Z","iopub.execute_input":"2022-06-30T06:18:05.328872Z","iopub.status.idle":"2022-06-30T06:18:05.341273Z","shell.execute_reply.started":"2022-06-30T06:18:05.328849Z","shell.execute_reply":"2022-06-30T06:18:05.340380Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Load model')\nmodel = CustomModel(model_name=BASE_MODEL)\nmodel = model.to(device)","metadata":{"_uuid":"3e4120ca-24e8-4beb-a71f-c362f3ba2b45","_cell_guid":"c0427f5d-995d-4146-8dce-55b8e9da5792","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:05.343344Z","iopub.execute_input":"2022-06-30T06:18:05.343601Z","iopub.status.idle":"2022-06-30T06:18:15.528111Z","shell.execute_reply.started":"2022-06-30T06:18:05.343578Z","shell.execute_reply":"2022-06-30T06:18:15.527108Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# inference","metadata":{"_uuid":"0733cda8-5ae7-42c6-acfa-000f7e92cf8e","_cell_guid":"4efa0281-1d33-467f-8892-85bcfe719bfd","trusted":true}},{"cell_type":"code","source":"paths = glob.glob(os.path.join('/', 'kaggle', \"input\", 'AI4Code', 'test', \"*.json\"))\n\ntest_df = pd.concat([read_notebook(x) for x in paths])\ntest_df = test_df.rename_axis(\"cell_id\").reset_index()\n\n# test_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\n# test_df[\"pct_rank\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)\n\ndisplay(test_df)\n# print(len(paths), len(test_df))","metadata":{"_uuid":"75f6191d-9491-4ac7-a03c-33a3a9f43777","_cell_guid":"13b43784-f37b-48f3-89ae-7de8820fd90f","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:15.529337Z","iopub.execute_input":"2022-06-30T06:18:15.529762Z","iopub.status.idle":"2022-06-30T06:18:15.604922Z","shell.execute_reply.started":"2022-06-30T06:18:15.529723Z","shell.execute_reply":"2022-06-30T06:18:15.603813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenizing 할때 문제 발생하는 듯\n# test_encoding = create_encoding(test_df[test_df['cell_type'] == 'markdown']['source'].tolist())\n# len(test_encoding['input_ids'])","metadata":{"_uuid":"e1332f40-9df0-4707-b9ea-0f0d22a08078","_cell_guid":"32ac1bff-dbc8-4211-b5d8-e577858c01d1","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:15.606345Z","iopub.execute_input":"2022-06-30T06:18:15.606874Z","iopub.status.idle":"2022-06-30T06:18:15.612012Z","shell.execute_reply.started":"2022-06-30T06:18:15.606817Z","shell.execute_reply":"2022-06-30T06:18:15.611028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_dataset = CustomDataset(test_encoding, [1]*len(test_encoding['input_ids']))\ntest_dataset = CustomDataset(test_df[test_df['cell_type'] == 'markdown']['source'].tolist())\nlen(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-06-30T06:18:15.613325Z","iopub.execute_input":"2022-06-30T06:18:15.614274Z","iopub.status.idle":"2022-06-30T06:18:15.629263Z","shell.execute_reply.started":"2022-06-30T06:18:15.614236Z","shell.execute_reply":"2022-06-30T06:18:15.628415Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset[0]['input_ids'].shape","metadata":{"execution":{"iopub.status.busy":"2022-06-30T06:18:15.631524Z","iopub.execute_input":"2022-06-30T06:18:15.632711Z","iopub.status.idle":"2022-06-30T06:18:15.650006Z","shell.execute_reply.started":"2022-06-30T06:18:15.632672Z","shell.execute_reply":"2022-06-30T06:18:15.649183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataloader = DataLoader(\n    test_dataset, \n    batch_size=batch_size,\n    num_workers=2,\n    pin_memory=True,\n)","metadata":{"_uuid":"fb569eee-614b-4c02-94ae-28f57484f5fa","_cell_guid":"98f4ac2b-0808-4248-a4a0-eee82488b4ce","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:15.651199Z","iopub.execute_input":"2022-06-30T06:18:15.651510Z","iopub.status.idle":"2022-06-30T06:18:15.656506Z","shell.execute_reply.started":"2022-06-30T06:18:15.651477Z","shell.execute_reply":"2022-06-30T06:18:15.655418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.load_state_dict(torch.load('../input/robertabase20036407541/roberta-base-2-0.0364-0.7541.pt'))\n\n\npreds = []\n\nmodel.eval()\n\nfor step, batch in enumerate(tqdm(test_dataloader)):\n    batch = {k: v.squeeze(1).to(device) for k, v in batch.items()}\n\n    with torch.no_grad():\n#         input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, output_attentions=None, output_hidden_states=None, return_dict=None, pct_rank=None\n#         _, logits = model(input_ids=batch['input_ids'], attention_mask=batch['attention_mask'])\n#         print(batch['input_ids'].shape)\n        _, logits = model(**batch)\n        logits = logits.detach().cpu().numpy()\n        preds.extend(logits)\n        \nlen(preds)","metadata":{"_uuid":"cc8c36c7-2788-4f2a-8261-b9b80b380651","_cell_guid":"c6fce88e-03fa-4f0c-a867-1d5b2a6ea87a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:15.658215Z","iopub.execute_input":"2022-06-30T06:18:15.658984Z","iopub.status.idle":"2022-06-30T06:18:22.526371Z","shell.execute_reply.started":"2022-06-30T06:18:15.658946Z","shell.execute_reply":"2022-06-30T06:18:22.525326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[test_df['cell_type']=='markdown', 'pct_rank'] = preds\ntest_df = test_df.sort_values(\"pct_rank\").groupby(\"id\", as_index=False)[\"cell_id\"].apply(lambda x: \" \".join(x))\ntest_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\n# test_df","metadata":{"_uuid":"cc39d1ab-5e6e-479a-859a-632211997f28","_cell_guid":"728d58a4-fa75-4fb1-a27a-cc9020bc4952","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:22.528103Z","iopub.execute_input":"2022-06-30T06:18:22.529002Z","iopub.status.idle":"2022-06-30T06:18:22.545466Z","shell.execute_reply.started":"2022-06-30T06:18:22.528970Z","shell.execute_reply":"2022-06-30T06:18:22.544598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('submission.csv', columns=['id', 'cell_order'], index=False)","metadata":{"_uuid":"559201df-ae10-448d-9dbc-a1906e408be1","_cell_guid":"6079350e-9081-43be-b2f2-e19def6bc293","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-30T06:18:22.548118Z","iopub.execute_input":"2022-06-30T06:18:22.548680Z","iopub.status.idle":"2022-06-30T06:18:22.558997Z","shell.execute_reply.started":"2022-06-30T06:18:22.548633Z","shell.execute_reply":"2022-06-30T06:18:22.558058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"e22643a1-a1de-41c0-9f90-fe71aed76bb9","_cell_guid":"e7012b6f-4198-4a56-948b-4a93f3ce640a","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}