{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# AI4Code Pytorch DistilBert Baseline + Toc Post-processing\n\n\n\nNearly 3% of notebooks have a table of content that can be used to reorder code. Notebooks with Toc are usually longer with 80 cases on average v.s. 45 cells without. In this notebooks I show how to read most of these ToC then how to post process predictions based on this.\n\n\n\nInitially using https://www.kaggle.com/code/aerdem4/ai4code-pytorch-distilbert-baseline\nNow using: https://www.kaggle.com/code/yuanzhezhou/ai4code-distilbert-training-777","metadata":{"papermill":{"duration":0.031568,"end_time":"2022-05-12T10:15:13.890382","exception":false,"start_time":"2022-05-12T10:15:13.858814","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\nBERT_PATH = \"../input/huggingface-bert-variants/distilbert-base-uncased/distilbert-base-uncased\"\n\ndata_dir = Path('../input/AI4Code')","metadata":{"papermill":{"duration":0.122804,"end_time":"2022-05-12T10:15:14.04297","exception":false,"start_time":"2022-05-12T10:15:13.920166","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:42:19.082690Z","iopub.execute_input":"2022-05-17T11:42:19.083009Z","iopub.status.idle":"2022-05-17T11:42:19.182618Z","shell.execute_reply.started":"2022-05-17T11:42:19.082923Z","shell.execute_reply":"2022-05-17T11:42:19.181961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DEBUG = False\nNUM_TRAIN = 1000 if DEBUG else 10000\n\ndf_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\n\ndef read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\npaths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\nnotebooks_train = [\n    read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n]\ndf = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\n\ndf_orders = pd.read_csv(\n    data_dir / 'train_orders.csv',\n    index_col='id',\n    squeeze=True,\n).str.split()  # Split the string representation of cell_ids into a list\n\ndef get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\ndf_orders_ = df_orders.to_frame().join(\n    df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n    how='right',\n)\n\nranks = {}\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\ndf_ranks = (\n    pd.DataFrame\n    .from_dict(ranks, orient='index')\n    .rename_axis('id')\n    .apply(pd.Series.explode)\n    .set_index('cell_id', append=True)\n)\n\ndf = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\ndf[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")","metadata":{"papermill":{"duration":82.291505,"end_time":"2022-05-12T10:16:36.365197","exception":false,"start_time":"2022-05-12T10:15:14.073692","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:42:19.295088Z","iopub.execute_input":"2022-05-17T11:42:19.295305Z","iopub.status.idle":"2022-05-17T11:44:06.831495Z","shell.execute_reply.started":"2022-05-17T11:42:19.295282Z","shell.execute_reply":"2022-05-17T11:44:06.830772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Table of content EDA","metadata":{}},{"cell_type":"code","source":"df_md = df[df.cell_type=='markdown'].reset_index()\ndf_md['is_toc'] = df_md.source.str.lower().str.contains('table of content').astype('int')\ncontain_toc = df_md.groupby('id').is_toc.max()\n\nprint(f'proportion containing a table of content: {contain_toc.mean():.2%}')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:06.833107Z","iopub.execute_input":"2022-05-17T11:44:06.833362Z","iopub.status.idle":"2022-05-17T11:44:07.423666Z","shell.execute_reply.started":"2022-05-17T11:44:06.833326Z","shell.execute_reply":"2022-05-17T11:44:07.422928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame({'lenght':df.reset_index().groupby('id').cell_id.count(),'contain_toc':contain_toc}).groupby('contain_toc').mean()","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:07.424767Z","iopub.execute_input":"2022-05-17T11:44:07.425138Z","iopub.status.idle":"2022-05-17T11:44:07.607709Z","shell.execute_reply.started":"2022-05-17T11:44:07.425102Z","shell.execute_reply":"2022-05-17T11:44:07.606652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_md[df_md.is_toc==1].pct_rank.hist(bins=100)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:07.609888Z","iopub.execute_input":"2022-05-17T11:44:07.610219Z","iopub.status.idle":"2022-05-17T11:44:08.002633Z","shell.execute_reply.started":"2022-05-17T11:44:07.610182Z","shell.execute_reply":"2022-05-17T11:44:08.001969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can start with setting the position of the toc very low...","metadata":{}},{"cell_type":"code","source":"for i in range(2 if DEBUG else 10):\n    print(df_md[df_md.is_toc==1].iloc[i].source)\n    print('------')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:08.003903Z","iopub.execute_input":"2022-05-17T11:44:08.004164Z","iopub.status.idle":"2022-05-17T11:44:08.025492Z","shell.execute_reply.started":"2022-05-17T11:44:08.004112Z","shell.execute_reply":"2022-05-17T11:44:08.024840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\ndef process_toc(line):\n    lst1 = re.findall(r'\\\"\\#(.*)\\\"', line)\n    lst2 = re.findall(r'\\(\\#(.*)\\)', line)\n    lst3 = re.findall(r'\\- (.*)', line)\n    if len(lst1):\n        return(lst1)\n    if len(lst2):\n        return(lst2)\n    else:\n        return(lst3)\n    \nc = 0\nfor s in df_md[df_md.is_toc==1].source:\n    process_toc(s)\n    if len(process_toc(s)):\n        c+=1\n\nprint(f' ugly but seems to process {c/len(df_md[df_md.is_toc==1].source):.2%} table of contents')","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:08.026835Z","iopub.execute_input":"2022-05-17T11:44:08.027341Z","iopub.status.idle":"2022-05-17T11:44:08.048993Z","shell.execute_reply.started":"2022-05-17T11:44:08.027305Z","shell.execute_reply":"2022-05-17T11:44:08.048357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ids_toc = df_md[df_md.is_toc==1].id.unique()\nids_toc_i = 0\n\ndf_ex = df[(df['id'] == ids_toc[ids_toc_i])&(df['cell_type'] == 'markdown')]","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:08.050229Z","iopub.execute_input":"2022-05-17T11:44:08.050659Z","iopub.status.idle":"2022-05-17T11:44:08.181822Z","shell.execute_reply.started":"2022-05-17T11:44:08.050624Z","shell.execute_reply":"2022-05-17T11:44:08.181181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bs4 import BeautifulSoup\n\nreg_lists = ['\\=\\'(.*)\\'', '\\(\\#(.*)\\)', '\\=\\\"(.*)\\\">', '\\\"(.*)\\\"']\n\ndef get_id(line, reg_lists = reg_lists):\n    m = []\n    first = np.inf\n    output = ''\n    for r in reg_lists:\n        match = re.search(r, line)\n        if match:\n            if (match.start()<10):\n                first = match.start()\n                output = line[match.start():match.end()]\n    return(output)\n\ndef extract_id(line):\n    output=''\n    soup = BeautifulSoup(line)\n    div = soup.find('a')\n    if div is not None:\n        output = div.get('id')\n    return(output)\n\n\nids_toc[ids_toc_i]\n\nprint(df_md[(df_md.is_toc==1)&(df_md['id'] == ids_toc[ids_toc_i])].source.values[0])\nprint(process_toc(df_md[(df_md.is_toc==1)&(df_md['id'] == ids_toc[ids_toc_i])].source.values[0]))\nprint(df_ex.source.apply(extract_id))","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:08.183033Z","iopub.execute_input":"2022-05-17T11:44:08.183293Z","iopub.status.idle":"2022-05-17T11:44:08.416708Z","shell.execute_reply.started":"2022-05-17T11:44:08.183256Z","shell.execute_reply":"2022-05-17T11:44:08.416037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import GroupShuffleSplit\n\nNVALID = 0.1  # size of validation set\n\nsplitter = GroupShuffleSplit(n_splits=1, test_size=NVALID, random_state=0)\n\ntrain_ind, val_ind = next(splitter.split(df, groups=df[\"ancestor_id\"]))\n\ntrain_df = df.loc[train_ind].reset_index(drop=True)\nval_df = df.loc[val_ind].reset_index(drop=True)","metadata":{"papermill":{"duration":1.895199,"end_time":"2022-05-12T10:16:49.969199","exception":false,"start_time":"2022-05-12T10:16:48.074","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:08.417894Z","iopub.execute_input":"2022-05-17T11:44:08.418118Z","iopub.status.idle":"2022-05-17T11:44:10.186723Z","shell.execute_reply.started":"2022-05-17T11:44:08.418086Z","shell.execute_reply":"2022-05-17T11:44:10.185972Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from bisect import bisect\n\ndef count_inversions(a):\n    inversions = 0\n    sorted_so_far = []\n    for i, u in enumerate(a):\n        j = bisect(sorted_so_far, u)\n        inversions += i - j\n        sorted_so_far.insert(j, u)\n    return inversions\n\n\ndef kendall_tau(ground_truth, predictions):\n    total_inversions = 0\n    total_2max = 0  # twice the maximum possible inversions across all instances\n    for gt, pred in zip(ground_truth, predictions):\n        ranks = [gt.index(x) for x in pred]  # rank predicted order in terms of ground truth\n        total_inversions += count_inversions(ranks)\n        n = len(gt)\n        total_2max += n * (n - 1)\n    return 1 - 4 * total_inversions / total_2max","metadata":{"papermill":{"duration":0.262837,"end_time":"2022-05-12T10:16:51.011588","exception":false,"start_time":"2022-05-12T10:16:50.748751","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:10.189742Z","iopub.execute_input":"2022-05-17T11:44:10.190005Z","iopub.status.idle":"2022-05-17T11:44:10.196952Z","shell.execute_reply.started":"2022-05-17T11:44:10.189970Z","shell.execute_reply":"2022-05-17T11:44:10.196224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_dummy = val_df.groupby('id')['cell_id'].apply(list)\nkendall_tau(df_orders.loc[y_dummy.index], y_dummy)","metadata":{"papermill":{"duration":0.372005,"end_time":"2022-05-12T10:16:51.63863","exception":false,"start_time":"2022-05-12T10:16:51.266625","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:10.198261Z","iopub.execute_input":"2022-05-17T11:44:10.198989Z","iopub.status.idle":"2022-05-17T11:44:10.320761Z","shell.execute_reply.started":"2022-05-17T11:44:10.198952Z","shell.execute_reply":"2022-05-17T11:44:10.320067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df[\"cell_type\"].value_counts()","metadata":{"papermill":{"duration":0.27416,"end_time":"2022-05-12T10:16:52.16563","exception":false,"start_time":"2022-05-12T10:16:51.89147","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:10.321942Z","iopub.execute_input":"2022-05-17T11:44:10.322219Z","iopub.status.idle":"2022-05-17T11:44:10.339457Z","shell.execute_reply.started":"2022-05-17T11:44:10.322181Z","shell.execute_reply":"2022-05-17T11:44:10.337254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df_mark = train_df[train_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\n\nval_df_mark = val_df[val_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)","metadata":{"papermill":{"duration":0.371916,"end_time":"2022-05-12T10:16:52.797271","exception":false,"start_time":"2022-05-12T10:16:52.425355","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:10.341224Z","iopub.execute_input":"2022-05-17T11:44:10.342185Z","iopub.status.idle":"2022-05-17T11:44:10.457019Z","shell.execute_reply.started":"2022-05-17T11:44:10.342126Z","shell.execute_reply":"2022-05-17T11:44:10.456315Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import mean_squared_error\n\nmean_squared_error(val_df_mark[\"pct_rank\"], np.ones(val_df_mark.shape[0])*train_df_mark[\"pct_rank\"].mean())","metadata":{"papermill":{"duration":0.280778,"end_time":"2022-05-12T10:16:53.349877","exception":false,"start_time":"2022-05-12T10:16:53.069099","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:10.458263Z","iopub.execute_input":"2022-05-17T11:44:10.458602Z","iopub.status.idle":"2022-05-17T11:44:10.479785Z","shell.execute_reply.started":"2022-05-17T11:44:10.458564Z","shell.execute_reply":"2022-05-17T11:44:10.479056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport sys, os\nfrom transformers import DistilBertModel, DistilBertTokenizer\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\n\nMAX_LEN = 128\n\nclass MarkdownModel(nn.Module):\n    def __init__(self):\n        super(MarkdownModel, self).__init__()\n        self.distill_bert = DistilBertModel.from_pretrained(BERT_PATH)\n        self.top = nn.Linear(768, 1)\n        \n    def forward(self, ids, mask):\n        x = self.distill_bert(ids, mask)[0]\n        x = self.top(x[:, 0, :])\n        return x","metadata":{"papermill":{"duration":7.145711,"end_time":"2022-05-12T10:17:00.757077","exception":false,"start_time":"2022-05-12T10:16:53.611366","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:10.481014Z","iopub.execute_input":"2022-05-17T11:44:10.481648Z","iopub.status.idle":"2022-05-17T11:44:16.638386Z","shell.execute_reply.started":"2022-05-17T11:44:10.481610Z","shell.execute_reply":"2022-05-17T11:44:16.637544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch.utils.data import DataLoader, Dataset\n\nclass MarkdownDataset(Dataset):\n    \n    def __init__(self, df, max_len):\n        super().__init__()\n        self.df = df.reset_index(drop=True)\n        self.max_len = max_len\n        self.tokenizer = DistilBertTokenizer.from_pretrained(BERT_PATH, do_lower_case=True)\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index]\n        \n        inputs = self.tokenizer.encode_plus(\n            row.source,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_len,\n            padding=\"max_length\",\n            return_token_type_ids=True,\n            truncation=True\n        )\n        ids = torch.LongTensor(inputs['input_ids'])\n        mask = torch.LongTensor(inputs['attention_mask'])\n\n        return ids, mask, torch.FloatTensor([row.pct_rank])\n\n    def __len__(self):\n        return self.df.shape[0]\n    \ntrain_ds = MarkdownDataset(train_df_mark, max_len=MAX_LEN)\nval_ds = MarkdownDataset(val_df_mark, max_len=MAX_LEN)\n\nval_ds[0]","metadata":{"papermill":{"duration":0.474499,"end_time":"2022-05-12T10:17:01.487031","exception":false,"start_time":"2022-05-12T10:17:01.012532","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:16.639902Z","iopub.execute_input":"2022-05-17T11:44:16.640292Z","iopub.status.idle":"2022-05-17T11:44:16.849422Z","shell.execute_reply.started":"2022-05-17T11:44:16.640252Z","shell.execute_reply":"2022-05-17T11:44:16.848664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df_mark.iloc[0]","metadata":{"papermill":{"duration":0.317331,"end_time":"2022-05-12T10:17:02.061139","exception":false,"start_time":"2022-05-12T10:17:01.743808","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:16.850782Z","iopub.execute_input":"2022-05-17T11:44:16.851164Z","iopub.status.idle":"2022-05-17T11:44:16.858486Z","shell.execute_reply.started":"2022-05-17T11:44:16.851108Z","shell.execute_reply":"2022-05-17T11:44:16.857809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def adjust_lr(optimizer, epoch):\n    if epoch < 1:\n        lr = 5e-5\n    elif epoch < 2:\n        lr = 1e-5\n    elif epoch < 5:\n        lr = 5e-6\n    else:\n        lr = 1e-7\n\n    for p in optimizer.param_groups:\n        p['lr'] = lr\n    return lr\n    \ndef get_optimizer(net):\n    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, net.parameters()), lr=3e-4, betas=(0.9, 0.999),\n                                 eps=1e-08)\n    return optimizer","metadata":{"papermill":{"duration":0.265988,"end_time":"2022-05-12T10:17:02.580374","exception":false,"start_time":"2022-05-12T10:17:02.314386","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:16.859816Z","iopub.execute_input":"2022-05-17T11:44:16.860431Z","iopub.status.idle":"2022-05-17T11:44:16.869016Z","shell.execute_reply.started":"2022-05-17T11:44:16.860395Z","shell.execute_reply":"2022-05-17T11:44:16.867943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"BS = 32\nNW = 8\n\ntrain_loader = DataLoader(train_ds, batch_size=BS, shuffle=True, num_workers=NW,\n                          pin_memory=False, drop_last=True)\nval_loader = DataLoader(val_ds, batch_size=BS, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)","metadata":{"papermill":{"duration":0.298424,"end_time":"2022-05-12T10:17:03.132","exception":false,"start_time":"2022-05-12T10:17:02.833576","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:16.870319Z","iopub.execute_input":"2022-05-17T11:44:16.870644Z","iopub.status.idle":"2022-05-17T11:44:16.883735Z","shell.execute_reply.started":"2022-05-17T11:44:16.870608Z","shell.execute_reply":"2022-05-17T11:44:16.882671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data(data):\n    return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n\ndef validate(model, val_loader):\n    model.eval()\n    \n    tbar = tqdm(val_loader, file=sys.stdout)\n    \n    preds = []\n    labels = []\n\n    with torch.no_grad():\n        for idx, data in enumerate(tbar):\n            inputs, target = read_data(data)\n\n            pred = model(inputs[0], inputs[1])\n\n            preds.append(pred.detach().cpu().numpy().ravel())\n            labels.append(target.detach().cpu().numpy().ravel())\n    \n    return np.concatenate(labels), np.concatenate(preds)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:16.885430Z","iopub.execute_input":"2022-05-17T11:44:16.886370Z","iopub.status.idle":"2022-05-17T11:44:16.894951Z","shell.execute_reply.started":"2022-05-17T11:44:16.886330Z","shell.execute_reply":"2022-05-17T11:44:16.894207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndef train(model, train_loader, val_loader, epochs):\n    np.random.seed(0)\n    \n    optimizer = get_optimizer(model)\n\n    criterion = torch.nn.MSELoss()\n    \n    for e in range(epochs):   \n        model.train()\n        tbar = tqdm(train_loader, file=sys.stdout)\n        \n        lr = adjust_lr(optimizer, e)\n        \n        loss_list = []\n        preds = []\n        labels = []\n\n        for idx, data in enumerate(tbar):\n\n            inputs, target = read_data(data)\n\n            optimizer.zero_grad()\n            pred = model(inputs[0], inputs[1])\n\n            loss = criterion(pred, target)\n            loss.backward()\n            optimizer.step()\n\n            loss_list.append(loss.detach().cpu().item())\n            preds.append(pred.detach().cpu().numpy().ravel())\n            labels.append(target.detach().cpu().numpy().ravel())\n\n            avg_loss = np.round(np.mean(loss_list), 4)\n\n            tbar.set_description(f\"Epoch {e+1} Loss: {avg_loss} lr: {lr}\")\n            \n        y_val, y_pred = validate(model, val_loader)\n            \n        print(\"Validation MSE:\", np.round(mean_squared_error(y_val, y_pred), 4))\n        print()\n    return model, y_pred\n\n# model = MarkdownModel()\n# model = model.cuda()\n# model, y_pred = train(model, train_loader, val_loader, epochs = 1 if DEBUG else 2)","metadata":{"papermill":{"duration":987.160977,"end_time":"2022-05-12T10:33:30.548236","exception":false,"start_time":"2022-05-12T10:17:03.387259","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:44:16.896134Z","iopub.execute_input":"2022-05-17T11:44:16.896485Z","iopub.status.idle":"2022-05-17T11:44:16.911649Z","shell.execute_reply.started":"2022-05-17T11:44:16.896451Z","shell.execute_reply":"2022-05-17T11:44:16.910787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_test = list((data_dir / 'test').glob('*.json'))\nnotebooks_test = [\n    read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]\ntest_df = (\n    pd.concat(notebooks_test)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n).reset_index()\n\ntest_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\ntest_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)\n\ntest_df[\"pct_rank\"] = 0\ntest_ds = MarkdownDataset(test_df[test_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True), max_len=MAX_LEN)\ntest_loader = DataLoader(test_ds, batch_size=BS, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)\n\nlen(test_ds), test_ds[0]","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:44:16.914029Z","iopub.execute_input":"2022-05-17T11:44:16.914889Z","iopub.status.idle":"2022-05-17T11:44:17.023536Z","shell.execute_reply.started":"2022-05-17T11:44:16.914851Z","shell.execute_reply":"2022-05-17T11:44:17.022711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MarkdownModel()\nmodel = model.cuda()\nmodel.load_state_dict(torch.load('../input/mymodels2022v3/my_own_model_markdown_file_0_0.78169.bin'))\n\ny_val, y_pred = validate(model, val_loader)\ny_test = validate(model, test_loader)[1]","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:50:11.283369Z","iopub.execute_input":"2022-05-17T11:50:11.284045Z","iopub.status.idle":"2022-05-17T11:50:54.714953Z","shell.execute_reply.started":"2022-05-17T11:50:11.284008Z","shell.execute_reply":"2022-05-17T11:50:54.714089Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df[\"pred\"] = val_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)\nval_df.loc[val_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_pred","metadata":{"papermill":{"duration":3.695855,"end_time":"2022-05-12T10:33:37.305989","exception":false,"start_time":"2022-05-12T10:33:33.610134","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:50:59.066486Z","iopub.execute_input":"2022-05-17T11:50:59.067350Z","iopub.status.idle":"2022-05-17T11:50:59.134852Z","shell.execute_reply.started":"2022-05-17T11:50:59.067311Z","shell.execute_reply":"2022-05-17T11:50:59.134178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_dummy = val_df.sort_values(\"pred\").groupby('id')['cell_id'].apply(list)\nkendall_tau(df_orders.loc[y_dummy.index], y_dummy)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:51:00.041865Z","iopub.execute_input":"2022-05-17T11:51:00.042293Z","iopub.status.idle":"2022-05-17T11:51:00.200923Z","shell.execute_reply.started":"2022-05-17T11:51:00.042256Z","shell.execute_reply":"2022-05-17T11:51:00.200212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def toc_post_process_sort(val_df):\n    # sort values to match toc chapters\n    val_df['is_toc'] = val_df.source.str.lower().str.contains('table of content')\n    val_ids = val_df[val_df['is_toc']].id.unique()\n    for id_str in val_ids:\n        df_id = val_df[val_df['id']==id_str].copy()\n        chapters = process_toc(df_id[df_id['is_toc']].source.values[0])\n        df_id['identification'] = df_id.source.apply(extract_id)\n        dti = pd.Series(np.sort(df_id[df_id.identification.isin(chapters)].pred.values),index=[c for c in chapters if c in df_id.identification.values])\n        df_id['new_pred'] = df_id.identification.map(dti)\n        val_df.loc[val_df['id']==id_str,'pred'] = np.where(~df_id['new_pred'].isna(),df_id['new_pred'],df_id['pred']).copy()\n    return val_df\n\ndef toc_post_process_values(val_df):\n    # Impute values:\n    # 0 for toc\n    # chapters to match toc chapters\n    val_df['is_toc'] = val_df.source.str.lower().str.contains('table of content')\n    val_ids = val_df[val_df['is_toc']].id.unique()\n    for id_str in val_ids:\n        try:\n            df_id = val_df[val_df['id']==id_str].copy()\n            chapters = process_toc(df_id[df_id['is_toc']].source.values[0])\n            df_id['identification'] = df_id.source.apply(extract_id)\n            idx = [c for c in chapters if c in df_id.identification.values]\n            dti = pd.Series(((np.arange(len(idx)+1)+1)/(len(idx)+1))[:len(idx)],index=idx)\n            df_id['new_pred'] = df_id.identification.map(dti)\n            df_id.loc[df_id.is_toc,'new_pred'] = 0\n            val_df.loc[val_df['id']==id_str,'pred'] = np.where(~df_id['new_pred'].isna(),df_id['new_pred'],df_id['pred']).copy()\n        except:\n            print(id_str)\n    return val_df","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:51:01.143790Z","iopub.execute_input":"2022-05-17T11:51:01.144572Z","iopub.status.idle":"2022-05-17T11:51:01.159228Z","shell.execute_reply.started":"2022-05-17T11:51:01.144522Z","shell.execute_reply":"2022-05-17T11:51:01.158504Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"toc_post_process_values(val_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:51:02.089613Z","iopub.execute_input":"2022-05-17T11:51:02.090078Z","iopub.status.idle":"2022-05-17T11:51:03.308426Z","shell.execute_reply.started":"2022-05-17T11:51:02.090043Z","shell.execute_reply":"2022-05-17T11:51:03.307551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_dummy = val_df.sort_values(\"pred\").groupby('id')['cell_id'].apply(list)\nkendall_tau(df_orders.loc[y_dummy.index], y_dummy)","metadata":{"papermill":{"duration":3.230655,"end_time":"2022-05-12T10:33:43.545936","exception":false,"start_time":"2022-05-12T10:33:40.315281","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:03.310117Z","iopub.execute_input":"2022-05-17T11:51:03.310417Z","iopub.status.idle":"2022-05-17T11:51:03.466971Z","shell.execute_reply.started":"2022-05-17T11:51:03.310379Z","shell.execute_reply":"2022-05-17T11:51:03.465951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"paths_test = list((data_dir / 'test').glob('*.json'))\nnotebooks_test = [\n    read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n]\ntest_df = (\n    pd.concat(notebooks_test)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n).reset_index()","metadata":{"papermill":{"duration":3.156008,"end_time":"2022-05-12T10:33:49.98707","exception":false,"start_time":"2022-05-12T10:33:46.831062","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:04.337898Z","iopub.execute_input":"2022-05-17T11:51:04.338284Z","iopub.status.idle":"2022-05-17T11:51:04.383522Z","shell.execute_reply.started":"2022-05-17T11:51:04.338252Z","shell.execute_reply":"2022-05-17T11:51:04.381934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\ntest_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)","metadata":{"papermill":{"duration":3.580422,"end_time":"2022-05-12T10:33:56.648552","exception":false,"start_time":"2022-05-12T10:33:53.06813","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:05.663643Z","iopub.execute_input":"2022-05-17T11:51:05.664203Z","iopub.status.idle":"2022-05-17T11:51:05.674861Z","shell.execute_reply.started":"2022-05-17T11:51:05.664160Z","shell.execute_reply":"2022-05-17T11:51:05.673996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df[\"pct_rank\"] = 0\ntest_ds = MarkdownDataset(test_df[test_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True), max_len=MAX_LEN)\ntest_loader = DataLoader(test_ds, batch_size=BS, shuffle=False, num_workers=NW,\n                          pin_memory=False, drop_last=False)\n\nlen(test_ds), test_ds[0]","metadata":{"papermill":{"duration":3.130783,"end_time":"2022-05-12T10:34:03.120038","exception":false,"start_time":"2022-05-12T10:33:59.989255","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:08.724456Z","iopub.execute_input":"2022-05-17T11:51:08.725038Z","iopub.status.idle":"2022-05-17T11:51:08.781737Z","shell.execute_reply.started":"2022-05-17T11:51:08.725003Z","shell.execute_reply":"2022-05-17T11:51:08.780914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"_, y_test = validate(model, test_loader)","metadata":{"papermill":{"duration":4.00212,"end_time":"2022-05-12T10:34:10.223448","exception":false,"start_time":"2022-05-12T10:34:06.221328","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:09.807414Z","iopub.execute_input":"2022-05-17T11:51:09.807855Z","iopub.status.idle":"2022-05-17T11:51:10.587311Z","shell.execute_reply.started":"2022-05-17T11:51:09.807820Z","shell.execute_reply":"2022-05-17T11:51:10.586486Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_test","metadata":{"papermill":{"duration":3.164567,"end_time":"2022-05-12T10:34:16.415308","exception":false,"start_time":"2022-05-12T10:34:13.250741","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:11.576328Z","iopub.execute_input":"2022-05-17T11:51:11.577132Z","iopub.status.idle":"2022-05-17T11:51:11.583970Z","shell.execute_reply.started":"2022-05-17T11:51:11.577096Z","shell.execute_reply":"2022-05-17T11:51:11.583213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = toc_post_process_values(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-05-17T11:51:12.311614Z","iopub.execute_input":"2022-05-17T11:51:12.312199Z","iopub.status.idle":"2022-05-17T11:51:12.319349Z","shell.execute_reply.started":"2022-05-17T11:51:12.312156Z","shell.execute_reply":"2022-05-17T11:51:12.318573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df = test_df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\nsub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\nsub_df.head()","metadata":{"papermill":{"duration":3.093752,"end_time":"2022-05-12T10:34:22.827853","exception":false,"start_time":"2022-05-12T10:34:19.734101","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:12.865885Z","iopub.execute_input":"2022-05-17T11:51:12.866345Z","iopub.status.idle":"2022-05-17T11:51:12.891895Z","shell.execute_reply.started":"2022-05-17T11:51:12.866308Z","shell.execute_reply":"2022-05-17T11:51:12.889090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df.to_csv(\"submission.csv\", index=False)","metadata":{"papermill":{"duration":3.551878,"end_time":"2022-05-12T10:34:29.528868","exception":false,"start_time":"2022-05-12T10:34:25.97699","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-17T11:51:13.953832Z","iopub.execute_input":"2022-05-17T11:51:13.954377Z","iopub.status.idle":"2022-05-17T11:51:13.962271Z","shell.execute_reply.started":"2022-05-17T11:51:13.954339Z","shell.execute_reply":"2022-05-17T11:51:13.961496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}