{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Pairwise pretrain\nIn this notebook, we demonstrate how to use pairwise model to predict in this competition. Please note that the inference time is much longer than pointwise method or using cosine similarity. \n\n1. **I used a bert-small model pretrained with pairwise-mlm.**\n2. Training with pairwise examples with negative samples randomly sampled.\n3. Inference and predict for all the pairs for test dataset.\n\n* [Pretrain](https://www.kaggle.com/code/yuanzhezhou/ai4code-pairwise-bertsmall-pretrain/notebook)\n* [Training](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-training)\n* [Inference](https://www.kaggle.com/yuanzhezhou/ai4code-pairwise-bertsmall-inference)","metadata":{}},{"cell_type":"code","source":"import json\nfrom pathlib import Path\n\nimport numpy as np\nimport pandas as pd\nfrom scipy import sparse\nfrom tqdm import tqdm\nimport os\n\nos.environ[\"WANDB_DISABLED\"] = \"true\"\n\npd.options.display.width = 180\npd.options.display.max_colwidth = 120\n\nBERT_PATH = \"../input/huggingface-bert-variants/distilbert-base-uncased/distilbert-base-uncased\"\n\ndata_dir = Path('../input/AI4Code')","metadata":{"papermill":{"duration":0.122804,"end_time":"2022-05-12T10:15:14.04297","exception":false,"start_time":"2022-05-12T10:15:13.920166","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:14.530604Z","iopub.execute_input":"2022-05-22T04:24:14.531111Z","iopub.status.idle":"2022-05-22T04:24:14.626974Z","shell.execute_reply.started":"2022-05-22T04:24:14.531022Z","shell.execute_reply":"2022-05-22T04:24:14.626275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"NUM_TRAIN = 200\n\n\ndef read_notebook(path):\n    return (\n        pd.read_json(\n            path,\n            dtype={'cell_type': 'category', 'source': 'str'})\n        .assign(id=path.stem)\n        .rename_axis('cell_id')\n    )\n\n\npaths_train = list((data_dir / 'train').glob('*.json'))[:NUM_TRAIN]\nnotebooks_train = [\n    read_notebook(path) for path in tqdm(paths_train, desc='Train NBs')\n]\ndf = (\n    pd.concat(notebooks_train)\n    .set_index('id', append=True)\n    .swaplevel()\n    .sort_index(level='id', sort_remaining=False)\n)\n\ndf","metadata":{"papermill":{"duration":82.291505,"end_time":"2022-05-12T10:16:36.365197","exception":false,"start_time":"2022-05-12T10:15:14.073692","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:14.628871Z","iopub.execute_input":"2022-05-22T04:24:14.629407Z","iopub.status.idle":"2022-05-22T04:24:19.331744Z","shell.execute_reply.started":"2022-05-22T04:24:14.629369Z","shell.execute_reply":"2022-05-22T04:24:19.331069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get an example notebook\nnb_id = df.index.unique('id')[6]\nprint('Notebook:', nb_id)\n\nprint(\"The disordered notebook:\")\nnb = df.loc[nb_id, :]\ndisplay(nb)\nprint()","metadata":{"papermill":{"duration":0.270693,"end_time":"2022-05-12T10:16:36.882443","exception":false,"start_time":"2022-05-12T10:16:36.61175","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:19.333199Z","iopub.execute_input":"2022-05-22T04:24:19.33351Z","iopub.status.idle":"2022-05-22T04:24:19.350694Z","shell.execute_reply.started":"2022-05-22T04:24:19.333474Z","shell.execute_reply":"2022-05-22T04:24:19.350025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders = pd.read_csv(\n    data_dir / 'train_orders.csv',\n    index_col='id',\n    squeeze=True,\n).str.split()  # Split the string representation of cell_ids into a list\n\ndf_orders","metadata":{"papermill":{"duration":2.835076,"end_time":"2022-05-12T10:16:39.9675","exception":false,"start_time":"2022-05-12T10:16:37.132424","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:19.352493Z","iopub.execute_input":"2022-05-22T04:24:19.352981Z","iopub.status.idle":"2022-05-22T04:24:22.124825Z","shell.execute_reply.started":"2022-05-22T04:24:19.352943Z","shell.execute_reply":"2022-05-22T04:24:22.124083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(df_orders.loc[\"002ba502bdac45\"])","metadata":{"papermill":{"duration":0.257536,"end_time":"2022-05-12T10:16:40.472139","exception":false,"start_time":"2022-05-12T10:16:40.214603","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:22.126047Z","iopub.execute_input":"2022-05-22T04:24:22.126304Z","iopub.status.idle":"2022-05-22T04:24:22.133325Z","shell.execute_reply.started":"2022-05-22T04:24:22.126276Z","shell.execute_reply":"2022-05-22T04:24:22.13267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cell_order = df_orders.loc[nb_id]\n\nprint(\"The ordered notebook:\")\nnb.loc[cell_order, :]","metadata":{"papermill":{"duration":0.265934,"end_time":"2022-05-12T10:16:40.98571","exception":false,"start_time":"2022-05-12T10:16:40.719776","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:22.134734Z","iopub.execute_input":"2022-05-22T04:24:22.135236Z","iopub.status.idle":"2022-05-22T04:24:22.154335Z","shell.execute_reply.started":"2022-05-22T04:24:22.135195Z","shell.execute_reply":"2022-05-22T04:24:22.153479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_ranks(base, derived):\n    return [base.index(d) for d in derived]\n\ncell_ranks = get_ranks(cell_order, list(nb.index))\nnb.insert(0, 'rank', cell_ranks)\n\nnb","metadata":{"papermill":{"duration":0.265625,"end_time":"2022-05-12T10:16:41.501618","exception":false,"start_time":"2022-05-12T10:16:41.235993","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:22.155853Z","iopub.execute_input":"2022-05-22T04:24:22.156478Z","iopub.status.idle":"2022-05-22T04:24:22.171751Z","shell.execute_reply.started":"2022-05-22T04:24:22.156441Z","shell.execute_reply":"2022-05-22T04:24:22.170311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_orders_ = df_orders.to_frame().join(\n    df.reset_index('cell_id').groupby('id')['cell_id'].apply(list),\n    how='right',\n)\n\nranks = {}\nfor id_, cell_order, cell_id in df_orders_.itertuples():\n    ranks[id_] = {'cell_id': cell_id, 'rank': get_ranks(cell_order, cell_id)}\n\ndf_ranks = (\n    pd.DataFrame\n    .from_dict(ranks, orient='index')\n    .rename_axis('id')\n    .apply(pd.Series.explode)\n    .set_index('cell_id', append=True)\n)\n\ndf_ranks","metadata":{"papermill":{"duration":2.967892,"end_time":"2022-05-12T10:16:44.752979","exception":false,"start_time":"2022-05-12T10:16:41.785087","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:22.173267Z","iopub.execute_input":"2022-05-22T04:24:22.173563Z","iopub.status.idle":"2022-05-22T04:24:22.271797Z","shell.execute_reply.started":"2022-05-22T04:24:22.173522Z","shell.execute_reply":"2022-05-22T04:24:22.271038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ancestors = pd.read_csv(data_dir / 'train_ancestors.csv', index_col='id')\ndf_ancestors","metadata":{"papermill":{"duration":0.44203,"end_time":"2022-05-12T10:16:45.446006","exception":false,"start_time":"2022-05-12T10:16:45.003976","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:22.273302Z","iopub.execute_input":"2022-05-22T04:24:22.27379Z","iopub.status.idle":"2022-05-22T04:24:22.49484Z","shell.execute_reply.started":"2022-05-22T04:24:22.273749Z","shell.execute_reply":"2022-05-22T04:24:22.494188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = df.reset_index().merge(df_ranks, on=[\"id\", \"cell_id\"]).merge(df_ancestors, on=[\"id\"])\ndf","metadata":{"papermill":{"duration":1.007951,"end_time":"2022-05-12T10:16:46.70626","exception":false,"start_time":"2022-05-12T10:16:45.698309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:22.497983Z","iopub.execute_input":"2022-05-22T04:24:22.498198Z","iopub.status.idle":"2022-05-22T04:24:22.571385Z","shell.execute_reply.started":"2022-05-22T04:24:22.498171Z","shell.execute_reply":"2022-05-22T04:24:22.570555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df[\"pct_rank\"] = df[\"rank\"] / df.groupby(\"id\")[\"cell_id\"].transform(\"count\")\n\ndf[\"pct_rank\"].hist(bins=10)","metadata":{"papermill":{"duration":0.862186,"end_time":"2022-05-12T10:16:47.820945","exception":false,"start_time":"2022-05-12T10:16:46.958759","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-05-22T04:24:22.572949Z","iopub.execute_input":"2022-05-22T04:24:22.57322Z","iopub.status.idle":"2022-05-22T04:24:22.824713Z","shell.execute_reply.started":"2022-05-22T04:24:22.573184Z","shell.execute_reply":"2022-05-22T04:24:22.824047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dict_cellid_source = dict(zip(df['cell_id'].values, df['source'].values))","metadata":{"execution":{"iopub.status.busy":"2022-05-22T04:24:22.826212Z","iopub.execute_input":"2022-05-22T04:24:22.826477Z","iopub.status.idle":"2022-05-22T04:24:22.833003Z","shell.execute_reply.started":"2022-05-22T04:24:22.826441Z","shell.execute_reply":"2022-05-22T04:24:22.83213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nimport re\n# import fasttext\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics.pairwise import cosine_similarity\nfrom nltk.stem import WordNetLemmatizer\nfrom pathlib import Path\nimport nltk\nnltk.download('wordnet')\n\nstemmer = WordNetLemmatizer()\n\ndef preprocess_text(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        #return document\n\n        # Lemmatization\n        tokens = document.split()\n        tokens = [stemmer.lemmatize(word) for word in tokens]\n        tokens = [word for word in tokens if len(word) > 3]\n\n        preprocessed_text = ' '.join(tokens)\n        return preprocessed_text\n\n    \ndef preprocess_df(df):\n    \"\"\"\n    This function is for processing sorce of notebook\n    returns preprocessed dataframe\n    \"\"\"\n    return [preprocess_text(message) for message in df.source]\n\ndf.source = df.source.apply(preprocess_text)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T04:24:22.834696Z","iopub.execute_input":"2022-05-22T04:24:22.83499Z","iopub.status.idle":"2022-05-22T04:24:28.532678Z","shell.execute_reply.started":"2022-05-22T04:24:22.834949Z","shell.execute_reply":"2022-05-22T04:24:28.531949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nimport sys, os\ntry:\n  from transformers import DistilBertModel, DistilBertTokenizer\nexcept:\n  !pip install transformers\n  from transformers import DistilBertModel, DistilBertTokenizer\nimport torch.nn.functional as F\nimport torch.nn as nn\nimport torch\n\nfrom transformers import BertConfig, BertForMaskedLM, DataCollatorForLanguageModeling\nfrom transformers import AutoModelWithLMHead, AutoTokenizer, AutoModel\n\nif not os.path.exists('text.txt'):\n  with open('text.txt','w') as f:\n    for id, item in tqdm(df.groupby('id')):\n      df_markdown =  item[item['cell_type']=='markdown']\n      for source, rank in df_markdown[['source', 'rank']].values:\n        cell_source = df_markdown[df_markdown['rank']==(rank+1)]\n        if len(cell_source):\n          setence = source + ' [SEP] ' + cell_source.source.values[0]\n          f.write(setence+'\\n')\n      \n\n# Train a tokenizer\nimport tokenizers\nfrom transformers import BertTokenizer, LineByLineTextDataset\n\ntokenizer = AutoTokenizer.from_pretrained('prajjwal1/bert-small')","metadata":{"execution":{"iopub.status.busy":"2022-05-22T04:24:28.533828Z","iopub.execute_input":"2022-05-22T04:24:28.534094Z","iopub.status.idle":"2022-05-22T04:24:48.938708Z","shell.execute_reply.started":"2022-05-22T04:24:28.53406Z","shell.execute_reply":"2022-05-22T04:24:48.93797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = AutoModelWithLMHead.from_pretrained('prajjwal1/bert-small')\n\n\ndata_collator = DataCollatorForLanguageModeling(\n    tokenizer=tokenizer, mlm=True, mlm_probability=0.15\n)\n\nfrom transformers import Trainer, TrainingArguments\n\ndataset= LineByLineTextDataset(\n    tokenizer = tokenizer,\n    file_path = './text.txt',\n    block_size = 128  # maximum sequence length\n)\n\nprint('No. of lines: ', len(dataset)) # No of lines in your datset\n\ntraining_args = TrainingArguments(\n    output_dir='./',\n    overwrite_output_dir=True,\n    num_train_epochs=10,\n    per_device_train_batch_size=64,\n    save_steps=10000,\n)\ntrainer = Trainer(\n    model=model,\n    args=training_args,\n    data_collator=data_collator,\n    train_dataset=dataset,\n)\ntrainer.train()\ntrainer.save_model('./')","metadata":{"execution":{"iopub.status.busy":"2022-05-22T04:24:48.940132Z","iopub.execute_input":"2022-05-22T04:24:48.940397Z","iopub.status.idle":"2022-05-22T04:25:39.017198Z","shell.execute_reply.started":"2022-05-22T04:24:48.940363Z","shell.execute_reply":"2022-05-22T04:25:39.016425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Please upvote if you find it helpful! :D","metadata":{}}]}