{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"MODEL = 'ai4code'\nPIKACHU = 'pikachu3'\nPYTHONPATH = f'PYTHONPATH=.:/kaggle:/kaggle/input/{PIKACHU}/utils:/kaggle/input/{PIKACHU}/third:$PYTHONPATH'\nPYTHONPATH","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:44:43.496428Z","iopub.execute_input":"2022-11-20T01:44:43.496924Z","iopub.status.idle":"2022-11-20T01:44:44.455186Z","shell.execute_reply.started":"2022-11-20T01:44:43.496879Z","shell.execute_reply":"2022-11-20T01:44:44.453869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nimport os  \n\n!rm -rf ./src\n!cp -rf ../input/{MODEL} ./src\nif os.path.exists('/kaggle'):\n  sys.path.append(f'/kaggle/input/{PIKACHU}/utils')\n  sys.path.append(f'/kaggle/input/{PIKACHU}/third')\n  sys.path.append('.')\n!ls ../input","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:58:27.236172Z","iopub.execute_input":"2022-11-20T01:58:27.237057Z","iopub.status.idle":"2022-11-20T01:58:31.05011Z","shell.execute_reply.started":"2022-11-20T01:58:27.237009Z","shell.execute_reply":"2022-11-20T01:58:31.048805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -q icecream --no-index --find-links=file:///kaggle/input/icecream/ \n!pip install -q datasets --no-index --find-links=file:///kaggle/input/hug-datasets/ \n!pip install -q pymp-pypi --no-index --find-links=file:///kaggle/input/pymp-pypi/pymp-pypi-0.4.5/dist\n!wandb disabled","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:58:32.664102Z","iopub.execute_input":"2022-11-20T01:58:32.665601Z","iopub.status.idle":"2022-11-20T01:59:01.914797Z","shell.execute_reply.started":"2022-11-20T01:58:32.665547Z","shell.execute_reply":"2022-11-20T01:59:01.913565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# The following is necessary if you want to use the fast tokenizer for deberta v2 or v3\nimport shutil\nfrom pathlib import Path\n\ntransformers_path = Path(\"/opt/conda/lib/python3.7/site-packages/transformers\")\n\ninput_dir = Path(\"../input/deberta-v2-3-fast-tokenizer\")\n\nconvert_file = input_dir / \"convert_slow_tokenizer.py\"\nconversion_path = transformers_path/convert_file.name\n\nif conversion_path.exists():\n  conversion_path.unlink()\n\nshutil.copy(convert_file, transformers_path)\ndeberta_v2_path = transformers_path / \"models\" / \"deberta_v2\"\n\nfor filename in ['tokenization_deberta_v2.py', 'tokenization_deberta_v2_fast.py']:\n  filepath = deberta_v2_path/filename\n  if filepath.exists():\n    filepath.unlink()\n\n  shutil.copy(input_dir/filename, filepath)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:01.918447Z","iopub.execute_input":"2022-11-20T01:59:01.91878Z","iopub.status.idle":"2022-11-20T01:59:01.935036Z","shell.execute_reply.started":"2022-11-20T01:59:01.91875Z","shell.execute_reply":"2022-11-20T01:59:01.934072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from gezi.common import *\ngezi.beijing_time_str()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:01.93862Z","iopub.execute_input":"2022-11-20T01:59:01.93891Z","iopub.status.idle":"2022-11-20T01:59:01.950011Z","shell.execute_reply.started":"2022-11-20T01:59:01.938884Z","shell.execute_reply":"2022-11-20T01:59:01.948895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from src.config import *\nfrom src.preprocess import *\nfrom src.postprocess import *\nfrom src.eval import *\ngezi.init_flags()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:01.953098Z","iopub.execute_input":"2022-11-20T01:59:01.953622Z","iopub.status.idle":"2022-11-20T01:59:01.960618Z","shell.execute_reply.started":"2022-11-20T01:59:01.953543Z","shell.execute_reply":"2022-11-20T01:59:01.95968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## normal infer for submission\nNUM_IDS = None\n## test hack(DO NOT SUBMIT!) infer 0 for fixed SAMPLE_IDS 383ids\n# NUM_IDS = 0\n# NUM_IDS = 200\n# NUM_IDS = 2000\n# NUM_IDS = 20000","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:01.962352Z","iopub.execute_input":"2022-11-20T01:59:01.962764Z","iopub.status.idle":"2022-11-20T01:59:01.970196Z","shell.execute_reply.started":"2022-11-20T01:59:01.96273Z","shell.execute_reply":"2022-11-20T01:59:01.96925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if NUM_IDS is None:\n  MARK = 'test'\nelse:\n  MARK = 'train'","metadata":{"execution":{"iopub.status.busy":"2022-11-20T02:01:02.666069Z","iopub.execute_input":"2022-11-20T02:01:02.666466Z","iopub.status.idle":"2022-11-20T02:01:02.671306Z","shell.execute_reply.started":"2022-11-20T02:01:02.666431Z","shell.execute_reply":"2022-11-20T02:01:02.670067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%writefile ./src/infer.py \n#!/usr/bin/env python \n# -*- coding: utf-8 -*-\n# ==============================================================================\n#          \\file   infer.py\n#        \\author   chenghuige  \n#          \\date   2022-05-15 07:00:15.332073\n#   \\Description  \n# ==============================================================================\n\n  \nfrom __future__ import absolute_import\nfrom __future__ import division\nfrom __future__ import print_function\n\nimport os, sys\nif os.path.exists('/kaggle'):\n  sys.path.append('/kaggle/input/pikachu3/utils')\n  sys.path.append('/kaggle/input/pikachu3/third')\n  sys.path.append('.')\nelse:\n  sys.path.append('..')\n  sys.path.append('../../../../utils')\n  sys.path.append('../../../../third')\n\nfrom gezi.common import *\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nos.environ[\"NCCL_DEBUG\"] = 'WARNING'\n\n\nfrom src.config import *\nfrom src.preprocess import *\nfrom src.postprocess import *\n\nflags.DEFINE_string('ofile', '../working/x.pkl', '')\n\ndef predict(model):\n  test_ds = get_dataloaders(test_only=True)\n\n  if FLAGS.pairwise_eval:\n    lele.predict(model, test_ds, \n                 out_keys=['id', 'cell_id', 'cid', 'cell_type', 'rank', 'rel_rank', 'n_words', 'n_cell', 'n_code_cell'], \n                 out_hook=out_hook,\n                 amp=FLAGS.amp_infer,\n                 fp16=FLAGS.fp16_infer) \n    x = out_hook_finalize()\n  else:\n    x = lele.predict(model, test_ds, \n                     out_keys=['id', 'cell_id', 'cid', 'code_idxes', 'n_cell', 'n_code_cell'],\n                     amp=FLAGS.amp_infer,\n                     fp16=FLAGS.fp16_infer)\n    if FLAGS.list_infer:\n      x = flattens(x)\n\n  if FLAGS.pairwise and (not FLAGS.two_tower):\n    xs = gezi.batch2list(x)\n    l = []\n    for x_ in xs:\n      l.append(pairwise_cat2pred(x_, infer=True))\n    x_ = gezi.list2batch(l)\n    x.update(x_)\n  \n  return x\n\ndef main(argv):\n  # gezi.init_flags()\n  model_dir = FLAGS.model_dir\n  bs = FLAGS.eval_bs\n  ic(bs)\n  gezi.restore_configs(model_dir, ignores=gezi.get_commandline_flags())\n  ic(model_dir, bs, FLAGS.eval_bs, FLAGS.bs)\n  FLAGS.train_allnew = False\n  FLAGS.grad_acc = 1\n  FLAGS.restore_configs = False\n  FLAGS.bs, FLAGS.eval_bs = bs, bs\n  mt.init()\n  FLAGS.model_dir = model_dir\n  FLAGS.pymp = False\n  FLAGS.num_workers = 1\n  FLAGS.pin_memory = True\n  FLAGS.persistent_workers = True\n  FLAGS.workers = 1\n  FLAGS.fold = 0\n  FLAGS.clear_dfs = True\n\n  if FLAGS.pairwise:\n    if FLAGS.two_tower:\n      FLAGS.pairwise_eval = True\n    FLAGS.n_markdowns = 0\n    FLAGS.list_infer = False\n    FLAGS.n_markdowns = 0\n  else:\n    FLAGS.list_infer = True\n    # FLAGS.max_len = 128\n    # FLAGS.max_context_len = 25\n    assert FLAGS.n_markdowns\n  \n  FLAGS.pairwise_dir2 = None\n  # FLAGS.num_negs = 2 \n  FLAGS.hack_infer = FLAGS.num_ids is not None\n  \n  FLAGS.bs = FLAGS.eval_bs\n  FLAGS.grad_acc = 1\n  ic(FLAGS.model_dir, FLAGS.ofile, FLAGS.amp_infer, FLAGS.fp16_infer)\n  ic(FLAGS.static_inputs_len, FLAGS.hack_infer, \n     FLAGS.num_workers, FLAGS.workers, FLAGS.pin_mem,\n     FLAGS.bs, FLAGS.eval_bs)\n  show()\n  \n  if gezi.in_kaggle():\n    backbone = FLAGS.backbone.split('/')[-1].replace('_', '-')\n    FLAGS.backbone = '../input/' + backbone\n  \n  FLAGS.mode = 'test'\n  FLAGS.work_mode = 'test'\n  ic(FLAGS.backbone, model_dir, os.path.exists(f'{model_dir}/model.pt'))\n  try:\n    display(pd.read_csv(f'{model_dir}/metrics.csv'))\n  except Exception as e:\n    logger.warning(e)\n  from src.torch.model import Model\n  model = Model()\n  ic(gezi.get_mem_gb())\n  gezi.load_weights(model, model_dir)\n  ic(gezi.get_mem_gb())\n  \n  if FLAGS.pairwise and (not FLAGS.two_tower) and FLAGS.rerank_filter:\n    xs = []\n\n    ##  8922 <3.5h\n    FLAGS.rerank_min_thre = 0\n    # FLAGS.rerank_max_thre=0.8\n    # FLAGS.num_negs = 4\n    xs.append(predict(model))\n        \n    x = gezi.merge_array_dicts(xs)\n  else:\n    x = predict(model)\n  \n  ic(gezi.get_mem_gb())\n  gezi.save(x, FLAGS.ofile)\n  \n\nif __name__ == '__main__':\n  app.run(main)  \n  \n","metadata":{"execution":{"iopub.status.busy":"2022-11-20T02:01:03.379798Z","iopub.execute_input":"2022-11-20T02:01:03.38017Z","iopub.status.idle":"2022-11-20T02:01:03.390021Z","shell.execute_reply.started":"2022-11-20T02:01:03.380135Z","shell.execute_reply":"2022-11-20T02:01:03.388843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = {}\ndef show_models(indexes, show=False):\n  for i in indexes:\n    model_dir = f'../input/{MODEL}-model{i}'\n    model_path = open(f'{model_dir}/path.txt').readline().strip()\n    models[i] = model_path\n  ic(models)\n\ndef infer(i, filename, bs=None, args=''):\n  model_dir = f'../input/{MODEL}-model{i}'\n  ofile = f'../working/{filename}.pkl'\n  #amp, fp16 = False, False\n  #   if not 'mpnet' in models[i]:\n  amp, fp16 = True, True\n  command = f'python ./src/infer.py --model_dir={model_dir} --eval_bs={bs} --ofile={ofile} --amp_infer={int(amp)} --fp16_infer={int(fp16)}'\n  if NUM_IDS is not None:\n    command += f' --num_ids={NUM_IDS}'\n  command += f' {args}'\n  ic(i, model_dir, bs, ofile)\n  gezi.system(command)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T02:01:03.663935Z","iopub.execute_input":"2022-11-20T02:01:03.665972Z","iopub.status.idle":"2022-11-20T02:01:03.673002Z","shell.execute_reply.started":"2022-11-20T02:01:03.66594Z","shell.execute_reply":"2022-11-20T02:01:03.671886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## online\nINDEXES = [\n    0, 6, 2\n]\n## offline\n# INDEXES = [\n#     3, 7, 5\n# ]\n## now\n# INDEXES = [\n#     3, 6, 2\n#   ]","metadata":{"execution":{"iopub.status.busy":"2022-11-20T02:01:03.872511Z","iopub.execute_input":"2022-11-20T02:01:03.872844Z","iopub.status.idle":"2022-11-20T02:01:03.878303Z","shell.execute_reply.started":"2022-11-20T02:01:03.872813Z","shell.execute_reply":"2022-11-20T02:01:03.876808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_models(INDEXES)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T02:01:04.071961Z","iopub.execute_input":"2022-11-20T02:01:04.072322Z","iopub.status.idle":"2022-11-20T02:01:04.170911Z","shell.execute_reply.started":"2022-11-20T02:01:04.072285Z","shell.execute_reply":"2022-11-20T02:01:04.170201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infer(INDEXES[0], 'xp', 256)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T02:01:04.267799Z","iopub.execute_input":"2022-11-20T02:01:04.268084Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infer(INDEXES[1], 'xc', 16, args='--pairwise_dir=../working/xp.pkl --max_len=168 --max_context_len=32')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"infer(INDEXES[2], 'xpc', 32, args='--pairwise_dir=../working/xp.pkl --rerank_filter --rerank_max_thre=0.8 --num_negs=2')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xp = gezi.load('../working/xp.pkl')\nxc = gezi.load('../working/xc.pkl')\nxpc = gezi.load('../working/xpc.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gezi.sort_dict_byid_(xp, 'cid')\ngezi.sort_dict_byid_(xc, 'cid')\ngezi.sort_dict_byid_(xpc, 'cid')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xc.keys()","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.005776Z","iopub.status.idle":"2022-11-20T01:59:02.006556Z","shell.execute_reply.started":"2022-11-20T01:59:02.006308Z","shell.execute_reply":"2022-11-20T01:59:02.006331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xc['reg_pred'] = np.asarray(xc['pred'])\nxc['cls_pred_ori'] = np.asarray(xc['cls_pred'])\nxc['cls2_pred'] = np.asarray([cls2pred(xc['cls_pred_ori'][i], method='argmax') for i in range(len(xc['cid']))])\nxc['cls_pred'] = np.asarray([cls2pred(xc['cls_pred_ori'][i], method=FLAGS.cls2pred) for i in range(len(xc['cid']))])\nxc['pred'] = xc['reg_pred'] ","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.007956Z","iopub.status.idle":"2022-11-20T01:59:02.008747Z","shell.execute_reply.started":"2022-11-20T01:59:02.008493Z","shell.execute_reply":"2022-11-20T01:59:02.008517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_negs = xpc['cls_pred_ori'].shape[1] \npc_preds = dict(zip(xpc['cid'], xpc['pred']))\npc_cls_preds = dict(zip(xpc['cid'], xpc['cls_pred']))\npc_probs = dict(zip(xpc['cid'], xpc['probs']))\n\npreds = []\ncls_preds = []\nprobs = []\nmarks = []\nfor cid, pred, cls_pred, prob in zip(xp['cid'], xp['pred'], xp['cls_pred'], xp['probs']):\n  mark = 1 if cid in xpc['cid'] else 0\n  marks.append(mark)\n  preds.append(pc_preds.get(cid, pred))\n  cls_preds.append(pc_cls_preds.get(cid, cls_pred))\n  probs.append(pc_probs.get(cid, prob))\n\nxpc = xp.copy()\nxpc['pred'] = np.asarray(preds)\nxpc['cls_pred'] = np.asarray(cls_preds)\nxpc['probs'] = np.asarray(probs)\nxpc['mark'] = np.asarray(marks)\nic(xpc['mark'].mean())","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.010205Z","iopub.status.idle":"2022-11-20T01:59:02.011027Z","shell.execute_reply.started":"2022-11-20T01:59:02.010736Z","shell.execute_reply":"2022-11-20T01:59:02.01076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"xps = gezi.batch2list(xp)\nxcs = gezi.batch2list(xc)\nxpcs = gezi.batch2list(xpc)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.012455Z","iopub.status.idle":"2022-11-20T01:59:02.013215Z","shell.execute_reply.started":"2022-11-20T01:59:02.012959Z","shell.execute_reply":"2022-11-20T01:59:02.012983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"LABEL_COL = 'rel_rank'\ndef diff_feats(x, l,r, name):\n  if isinstance(l, str):\n    x[f'{name}_diff'] = x[l] - x[r]\n  else:\n    x[f'{name}_diff'] = l - r\n  x[f'abs_{name}_diff'] = abs(x[f'{name}_diff'])\n\ndef gen_pfeat(x_, topn=None):\n  x = {}\n  keys = ['pred', 'cls_pred']\n  x = {k: x_[k] for k in keys}\n  # x['rank_pred'] = x['pred'] * (1 + x_['n_code_cell']) - 0.5\n  # x['cls_rank_pred'] = x['cls_pred'] * (1 + x_['n_code_cell']) - 0.5\n  x['min_prob'] = x_['probs'].min()\n  x['var_prob'] = np.var(x_['probs'])\n  if 'sims' in x_:\n    x['min_sim'] = x_['sims'].min()\n    x['var_sim'] = np.var(x_['sims'])\n  \n  idxes = (-x_['probs']).argsort()\n  # topn = topn or len(idxes)\n  topn = topn or 10\n  for i in range(topn):\n    if i < len(idxes):\n      if i > 0:\n        x[f'top_pred_{i}'] = (idxes[i] + 0.5) / (x_['n_code_cell'] + 1)\n        # x[f'top_rank_{i}'] = idxes[i]\n      x[f'top_prob_{i}'] = x_['probs'][idxes[i]]\n      if 'sims' in x_:\n        x[f'top_sim_{i}'] = x_['sims'][idxes[i]]\n    else:\n      x[f'top_pred_{i}'] = -1\n      # x[f'top_rank_{i}'] = -1\n      x[f'top_prob_{i}'] = -1\n      if 'sims' in x_:\n        x[f'top_sim_{i}'] = -1\n  \n  diff_feats(x, 'cls_pred', 'pred', 'cls')\n  # diff_feats(x, 'cls_rank_pred', 'rank_pred', 'cls_rank')\n  return x\n\ndef gen_cfeat(x_):\n  topn = min(10, len(x_['cls_pred_ori']))\n  x = {}\n  keys = [\n    # 'pred', \n    'cls_pred', \n    'reg_pred', \n    'cls2_pred', \n    'group_pos',\n    'group_rank',\n    ]\n  x = {k: x_[k] for k in keys}\n  # x['cls_rank_pred'] = x['cls_pred'] * FLAGS.num_classes - 0.5\n  # x['cls2_rank_pred'] = x['cls2_pred'] * FLAGS.num_classes - 0.5\n  # x['reg_rank_pred'] = x['reg_pred'] * FLAGS.num_classes - 0.5\n  preds = x_['cls_pred_ori']\n  probs = gezi.softmax(preds)\n  x['min_prob'] = probs.min()\n  x['var_prob'] = np.var(probs)\n  idxes = (-probs).argsort()\n\n  for i in range(topn):\n    if i < len(idxes):\n      if i > 0:\n        x[f'top_pred_{i}'] = (idxes[i] + 0.5) / FLAGS.num_classes\n        # x[f'tpop_rank_{i}']  = idxes[i] \n      x[f'top_prob_{i}'] = probs[idxes[i]]\n    else:\n      x[f'top_pred_{i}'] = -1\n      # x[f'top_rank_{i}'] = -1\n      x[f'top_prob_{i}'] = -1\n  diff_feats(x, 'cls_pred', 'reg_pred', 'cls_reg')\n  diff_feats(x, 'cls_pred', 'cls2_pred', 'cls_cls2')\n  diff_feats(x, 'cls2_pred', 'reg_pred', 'cls2_reg')\n  # diff_feats(x, 'cls_rank_pred', 'reg_rank_pred', 'cls_reg_rank')\n  # diff_feats(x, 'cls_rank_pred', 'cls2_rank_pred', 'cls_cls2_rank')\n  # diff_feats(x, 'cls2_rank_pred', 'reg_rank_pred', 'cls2_reg_rank')\n  # group_id = x_['pred_id']\n  # x['max_pred'] = max_preds[group_id]\n  # x['min_pred'] = min_preds[group_id]\n  # x['pred_ratio'] = (x['pred'] - x['min_pred']) / (x['max_pred'] - x['min_pred']) if (x['max_pred'] - x['min_pred']) else 1\n  # x['max_cls_pred'] = max_cls_preds[group_id]\n  # x['min_cls_pred'] = min_cls_preds[group_id]\n  # x['cls_pred_ratio'] = (x['cls_pred'] - x['min_cls_pred']) / (x['max_cls_pred'] - x['min_cls_pred']) if (x['max_cls_pred'] - x['min_cls_pred']) else 1\n  \n  return x\n\ndef gen_feats(start, end, leave=False):\n  p_feats = [gen_pfeat(x) for x in tqdm(xps[start:end], desc=f'gen_pfeats:xp', leave=leave)]\n  c_feats = [gen_cfeat(x) for x in tqdm(xcs[start:end], desc=f'gen_cfeats:xc', leave=leave)]\n  pc_feats = [gen_pfeat(x, topn=3) for x in tqdm(xpcs[start:end], desc=f'gen_pfeats:xpc', leave=leave)]\n\n  feats = []\n  for i in tqdm(range(len(p_feats)), desc='merge_feats'):\n    fe1 = p_feats[i]\n    fec = c_feats[i]\n    fepc = pc_feats[i]\n    fepc['mark'] = xpc['mark'][i]\n    \n    fe = {}\n    fe['code_ratio'] = xp['n_code_cell'][i] / xp['n_cell'][i]\n    diff_feats(fe, fe1['pred'], fec['reg_pred'], 'pc_reg')\n    diff_feats(fe, fe1['pred'], fec['cls_pred'], 'pc_cls')\n    diff_feats(fe, fe1['pred'], fec['cls2_pred'], 'pc_cls2')\n    diff_feats(fe, fe1['cls_pred'], fec['reg_pred'], 'pc_cls_reg')\n    diff_feats(fe, fe1['cls_pred'], fec['cls_pred'], 'pc_cls_cls')\n    diff_feats(fe, fe1['cls_pred'], fec['cls2_pred'], 'pc_cls_cls2')\n    \n    diff_feats(fe, fepc['pred'], fec['reg_pred'], 'pcc_reg')\n    diff_feats(fe, fepc['pred'], fec['cls_pred'], 'pcc_cls')\n    diff_feats(fe, fepc['pred'], fec['cls2_pred'], 'pcc_cls2')\n    diff_feats(fe, fepc['cls_pred'], fec['reg_pred'], 'pcc_cls_reg')\n    diff_feats(fe, fepc['cls_pred'], fec['cls_pred'], 'pcc_cls_cls')\n    diff_feats(fe, fepc['cls_pred'], fec['cls2_pred'], 'pcc_cls_cls2')\n    \n    # diff_feats(fe, fe1['rank_pred'], fec['reg_rank_pred'], 'pc_cls_rank')\n    # diff_feats(fe, fe1['cls_rank_pred'], fec['reg_rank_pred'], 'pc_cls_reg_rank')\n    # diff_feats(fe, fepc['rank_pred'], fec['reg_rank_pred'], 'pcc_cls_rank')\n    # diff_feats(fe, fepc['cls_rank_pred'], fec['reg_rank_pred'], 'pcc_cls_reg_rank')\n\n    fe1 = gezi.dict_prefix(fe1, 'xp_')\n    fe.update(fe1)\n    fec = gezi.dict_prefix(fec, 'xc_')\n    fe.update(fec)\n    fepc = gezi.dict_prefix(fepc, 'xpc_')\n    fe.update(fepc)\n\n    keys = [\n     'id', 'cell_id', 'cid',\n     'n_words', 'n_code_cell', 'n_cell'\n    ]\n    for key in keys:\n      fe[key] = xp[key][i]\n      \n    #  ic(feat)\n    feats.append(fe)\n  #  break\n  dfeats = pd.DataFrame(feats)\n  return dfeats","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.01468Z","iopub.status.idle":"2022-11-20T01:59:02.015473Z","shell.execute_reply.started":"2022-11-20T01:59:02.015202Z","shell.execute_reply":"2022-11-20T01:59:02.015226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from catboost import CatBoostRegressor\nimport lightgbm as lgb\nn_insts = len(xps)\ncbt_models, lgb_models = [], []\nFOLDS = 1 if NUM_IDS is not None else 5\nic(FOLDS)\n\nfor fold in range(FOLDS):\n  cbt_file = f'../input/ai4code-trees2/{fold}.cbt'\n  cbt_model = CatBoostRegressor()\n  cbt_model.load_model(cbt_file)\n  cbt_models.append(cbt_model)\n    \n  lgb_file = f'../input/ai4code-trees2/{fold}.lgb'\n  lgb_model = mt.load_forest_model(lgb_file)\n  lgb_models.append(lgb_model)\n\nbs = 500 if NUM_IDS is not None else 10000\nic(bs)\nsteps = -((-n_insts) // bs)\nic(steps)\npreds = []\nfor step in tqdm(range(steps), desc='infer'):\n  start = step * bs\n  end = start + bs\n  df = gen_feats(start, end)\n  \n  reg_cols = [x for x in df.columns if x not in ['id', 'cell_id', 'cid', LABEL_COL]]\n  cat_cols = []\n  cols = reg_cols + cat_cols\n  if start == 0:\n    ic(len(cols))\n    \n  pred = np.asarray([0.] * len(df))\n  for fold in tqdm(range(FOLDS), desc='folds', leave=False):\n    pred += cbt_models[fold].predict(df[cols]) * 0.5\n    pred += mt.lgb_predict(df[cols].values, lgb_models[fold], leave=False) * 0.5\n  pred /= FOLDS\n  preds.append(pred)\n\npreds = np.concatenate(preds)\nic(preds.shape)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.016883Z","iopub.status.idle":"2022-11-20T01:59:02.01765Z","shell.execute_reply.started":"2022-11-20T01:59:02.017398Z","shell.execute_reply":"2022-11-20T01:59:02.017422Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x = {'id': xp['id'], 'cell_id': xp['cell_id']}\nx['pred'] = preds","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.019021Z","iopub.status.idle":"2022-11-20T01:59:02.019783Z","shell.execute_reply.started":"2022-11-20T01:59:02.019527Z","shell.execute_reply":"2022-11-20T01:59:02.019551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if NUM_IDS is not None:\n  FLAGS.hack_infer = True\n  FLAGS.num_ids = NUM_IDS\n  ic(calc_metric(xp))\n  ic(calc_metric(xc))\n  ic(calc_metric(xpc))\n  ic(calc_metric(x))","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.021171Z","iopub.status.idle":"2022-11-20T01:59:02.021962Z","shell.execute_reply.started":"2022-11-20T01:59:02.02169Z","shell.execute_reply":"2022-11-20T01:59:02.021713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def to_df(x, key='pred'):\n  df = pd.read_feather(f'../working/{MARK}.fea', ['id', 'cell_id', 'cell_type', 'rel_rank'])\n  x['pred'] = np.asarray(x['pred'])      \n  m = {\n      'id': x['id'],\n      'cell_id': x['cell_id'],\n      'pred': x[key],\n  }\n\n\n  df_pred = pd.DataFrame(m)\n    \n  ids = set(df_pred.id)\n  df = df[df.id.isin(ids)]\n  df = df.merge(df_pred, on=(['id', 'cell_id']), how='left')\n\n  df_pred = order_df(df, 'pred', mark=mark)\n\n  return df_pred","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.023364Z","iopub.status.idle":"2022-11-20T01:59:02.024124Z","shell.execute_reply.started":"2022-11-20T01:59:02.023869Z","shell.execute_reply":"2022-11-20T01:59:02.023893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = to_df(x)\ndf['cell_order'] = df['cell_order'].apply(lambda x: ' '.join(x))\ndisplay(df)\ndf.to_csv('./submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-11-20T01:59:02.025527Z","iopub.status.idle":"2022-11-20T01:59:02.026301Z","shell.execute_reply.started":"2022-11-20T01:59:02.026028Z","shell.execute_reply":"2022-11-20T01:59:02.026052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}