{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%writefile run2.py\nimport pandas as pd\nimport gc\nimport torch\nfrom tqdm.auto import tqdm\n\ntest_df = pd.read_csv('test_df_translated.csv')\nfrom transformers import M2M100ForConditionalGeneration, M2M100Tokenizer\nmodel = M2M100ForConditionalGeneration.from_pretrained(\"../input/m2m100\")\ntokenizer = M2M100Tokenizer.from_pretrained(\"../input/m2m100\")\n\nlangtotrans = test_df['language_label'].unique().tolist()\n#langtotrans = ['fr', 'tr', 'es', 'pt', 'zh', 'ja', 'ru','vi','bg','de','th','ar','hi','el']\n#langtotrans = ['fr', 'tr', 'es', 'pt', 'zh', 'ja', 'ru']\n#langtotrans = ['pt','ja','ru','zh']\n\nfor bnm in langtotrans:\n    try:\n        if len(test_df[test_df['language_label']==f'{bnm}'].source.tolist())==0 or bnm in ['en', 'unknown', 'it']:\n            continue\n        else:\n            print('translation of language... ',bnm)\n            device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n            hi_text = test_df[test_df['language_label']==f'{bnm}'].source.tolist()\n            tokenizer.src_lang = f'{bnm}'\n            batch = 2\n            model = model.to(device)\n            texttrfinal = []\n            for idx in tqdm(range(0,len(hi_text),batch)):\n                idx=idx+batch\n                encoded_hi = tokenizer(hi_text[(idx-batch):idx], return_tensors=\"pt\",padding=True,truncation=True,max_length=512).to(device)\n                generated_tokens = model.generate(**encoded_hi, forced_bos_token_id=tokenizer.get_lang_id(\"en\"))\n                texttr = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)\n                torch.cuda.empty_cache()\n                gc.collect()\n                texttrfinal.extend(texttr)\n            test_df['source'][test_df['language_label']==f'{bnm}'] = texttrfinal\n    except:\n        continue\ntest_df.to_csv('test_df_translated.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:38:43.872524Z","iopub.execute_input":"2022-08-11T16:38:43.873304Z","iopub.status.idle":"2022-08-11T16:38:43.920629Z","shell.execute_reply.started":"2022-08-11T16:38:43.873185Z","shell.execute_reply":"2022-08-11T16:38:43.919971Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#!python ../input/testdftrans/testdftrans.py\n#!python ../input/sub8trans2/testdftrans_2.py\n#!python ../input/sub8trans2/testdftrans_3.py\n#!python ../input/sub8trans2/testdftrans_4.py\n!python ../input/sub8transv6/testdftrans_6.2.py\n#!python ../input/sub8transv4/testdftrans_5.3.py\n#!python ../input/sub8transv3/testdftrans_5.2.py\n#!python ../input/sub8transv7/testdftrans_5.4.py\n!python run2.py\n#!python run3.py\n#!python ../input/sub8trans3/sub8_trans3.py\n#!python ../input/submission8ai4code/submission8.py","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:38:43.968228Z","iopub.execute_input":"2022-08-11T16:38:43.96858Z","iopub.status.idle":"2022-08-11T16:39:48.388219Z","shell.execute_reply.started":"2022-08-11T16:38:43.968541Z","shell.execute_reply":"2022-08-11T16:39:48.387196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#%%writefile run3.py\ndef run3():\n    \n    import numpy as np\n    \n    ckpt_path = [\"../input/codebertbasev3v2/model.bin\",\n                 '../input/codebertbasev5andemb/model.bin']\n    model_path = [\"../input/dataai4codep/codebert-base/codebert-base/\",\n                  '../input/dataai4codep/codebert-base/codebert-base/']\n    tok = [\"../input/codebertbasev3v2/tokenizer\",\n           '../input/codebertbasev5andemb/tokenizer']\n    \n    embedsmodels = ['../input/codeaiembeds1models/',\n                    '../input/codebertbasev5andembmodels/']\n    \n    \n    sumsub = 0\n    wsubs = []\n    nosubs = 0\n    for _ in ckpt_path:\n        nosubs = nosubs+1\n        sumsub = sumsub+(nosubs)\n    print('total sum',sumsub)\n    print('no subs',nosubs)\n\n    for iii in range(nosubs):\n        wsubs.append((nosubs-iii)/sumsub)\n    weights = np.array(wsubs).astype('float128')\n    print(weights)\n    sum(weights)\n\n    # %% [code] {\"papermill\":{\"duration\":0.114595,\"end_time\":\"2022-05-23T03:29:09.832611\",\"exception\":false,\"start_time\":\"2022-05-23T03:29:09.718016\",\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-08-06T21:34:53.955199Z\",\"iopub.execute_input\":\"2022-08-06T21:34:53.955734Z\",\"iopub.status.idle\":\"2022-08-06T21:34:53.983485Z\",\"shell.execute_reply.started\":\"2022-08-06T21:34:53.955638Z\",\"shell.execute_reply\":\"2022-08-06T21:34:53.982628Z\"}}\n    def read_notebook(path):\n        import pandas as pd\n        return (\n            pd.read_json(\n                path,\n                dtype={'cell_type': 'category', 'source': 'str'})\n            .assign(id=path.stem)\n            .rename_axis('cell_id')\n        )\n\n    # %% [code] {\"papermill\":{\"duration\":0.023767,\"end_time\":\"2022-05-23T03:29:09.929422\",\"exception\":false,\"start_time\":\"2022-05-23T03:29:09.905655\",\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-08-06T21:34:53.985303Z\",\"iopub.execute_input\":\"2022-08-06T21:34:53.985577Z\",\"iopub.status.idle\":\"2022-08-06T21:34:53.995271Z\",\"shell.execute_reply.started\":\"2022-08-06T21:34:53.985542Z\",\"shell.execute_reply\":\"2022-08-06T21:34:53.994435Z\"}}\n    def clean_code(cell): return str(cell).replace(\"\\\\n\", \"\\n\")\n\n    def sample_cells(cells, n):\n        import numpy as np\n        cells = [clean_code(cell) for cell in cells]\n        if n >= len(cells): return [cell[:200] for cell in cells]\n        else:\n            results = []\n            step = len(cells) / n\n            idx = 0\n            while int(np.round(idx)) < len(cells):\n                results.append(cells[int(np.round(idx))])\n                idx += step        \n            if cells[-1] not in results: results[-1] = cells[-1]\n            return results\n\n    def get_features(df):\n        from tqdm import tqdm\n        features = dict()\n        df = df.sort_values(\"rank\").reset_index(drop=True)\n        for idx, sub_df in tqdm(df.groupby(\"id\")):\n            features[idx] = dict()\n            total_md = sub_df[sub_df.cell_type == \"markdown\"].shape[0]\n            code_sub_df = sub_df[sub_df.cell_type == \"code\"]\n            total_code = code_sub_df.shape[0]\n            codes = sample_cells(code_sub_df.source.values, 20)\n            features[idx][\"total_code\"] = total_code\n            features[idx][\"total_md\"] = total_md\n            features[idx][\"codes\"] = codes\n        return features\n\n    # %% [code] {\"execution\":{\"iopub.status.busy\":\"2022-08-06T21:34:54.003952Z\",\"iopub.execute_input\":\"2022-08-06T21:34:54.004141Z\",\"iopub.status.idle\":\"2022-08-06T21:35:15.053086Z\",\"shell.execute_reply.started\":\"2022-08-06T21:34:54.004118Z\",\"shell.execute_reply\":\"2022-08-06T21:35:15.052301Z\"}}\n    import re\n    #from nltk.stem import WordNetLemmatizer\n    #import nltk\n\n    #nltk.data.path.append('../input/wordnet/')\n    #nltk.data.path.append('../input/wordnet/wordnet')\n    #nltk.download('wordnet')\n    #stemmer = WordNetLemmatizer()\n\n    def preprocess_text_sc(document):\n        # Remove all the special characters\n        document = re.sub(r'\\W', ' ', str(document))\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # remove all single characters\n        document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n\n        # Remove single characters from the start\n        document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n\n        # Substituting multiple spaces with single space\n        document = re.sub(r'\\s+', ' ', document, flags=re.I)\n\n        # Removing prefixed 'b'\n        document = re.sub(r'^b\\s+', '', document)\n\n        # Converting to Lowercase\n        document = document.lower()\n        return document\n\n\n    # %% [code] {\"papermill\":{\"duration\":0.027037,\"end_time\":\"2022-05-23T03:29:16.100816\",\"exception\":false,\"start_time\":\"2022-05-23T03:29:16.073779\",\"status\":\"completed\"},\"tags\":[],\"execution\":{\"iopub.status.busy\":\"2022-08-06T21:35:15.054695Z\",\"iopub.execute_input\":\"2022-08-06T21:35:15.054897Z\",\"iopub.status.idle\":\"2022-08-06T21:35:15.085561Z\",\"shell.execute_reply.started\":\"2022-08-06T21:35:15.054873Z\",\"shell.execute_reply\":\"2022-08-06T21:35:15.084783Z\"}}\n    def read_data(data): return tuple(d.cuda() for d in data[:-1]), data[-1].cuda()\n\n    def validate(model, val_loader):    \n        import sys\n        import torch    \n        import numpy as np\n        from tqdm import tqdm    \n        model.eval()    \n        tbar = tqdm(val_loader, file=sys.stdout)    \n        preds = []\n        labels = []\n        with torch.no_grad():\n            for idx, data in enumerate(tbar):\n                inputs, target = read_data(data)\n                pred = model(*inputs)\n                preds.append(pred.detach().cpu().numpy().ravel())\n                labels.append(target.detach().cpu().numpy().ravel())    \n        return np.concatenate(labels), np.concatenate(preds)\n\n# '\n#     def get_emb(model, val_loader):\n#         model.eval()\n#         tqdm._instances.clear()\n#         tbar = tqdm(val_loader, file=sys.stdout)\n\n#        # preds = []\n#         labels = []\n#         embeddings = list()\n#         with torch.no_grad():\n#             for idx, data in enumerate(tbar):\n#                 inputs, target = read_data(data)\n\n#                 with torch.cuda.amp.autocast():\n#                     outputs = model(*inputs)\n\n#                # preds.append(pred.detach().cpu().numpy().ravel())\n#                 outputs = outputs.detach().cpu().numpy()#.ravel()\n#                 embeddings.extend(outputs)\n#                 labels.append(target.detach().cpu().numpy().ravel())\n#         tqdm._instances.clear()\n#         return np.concatenate(labels), np.array(embeddings)'\n\n\n    import gc\n    import json\n    import sys, os\n    import numpy as np\n    import pandas as pd\n    from tqdm import tqdm\n    from pathlib import Path\n    from scipy import sparse\n\n    data_dir = Path('../input/AI4Code')\n    paths_test = list((data_dir / 'test').glob('*.json'))\n    notebooks_test = [\n        read_notebook(path) for path in tqdm(paths_test, desc='Test NBs')\n    ]\n    test_df = (\n        pd.concat(notebooks_test)\n        .set_index('id', append=True)\n        .swaplevel()\n        .sort_index(level='id', sort_remaining=False)\n    ).reset_index()\n    test_df[\"rank\"] = test_df.groupby([\"id\", \"cell_type\"]).cumcount()\n    test_df[\"pred\"] = test_df.groupby([\"id\", \"cell_type\"])[\"rank\"].rank(pct=True)\n\n    import torch\n    import torch.nn as nn\n    import torch.nn.functional as F\n    from torch.utils.data import DataLoader, Dataset\n    from transformers import AutoModel, AutoTokenizer\n\n    class MarkdownModel(nn.Module):\n        def __init__(self, model_path):\n            super(MarkdownModel, self).__init__()\n            self.model = AutoModel.from_pretrained(model_path)\n           # self.model.gradient_checkpointing_enable()\n            self.top = nn.Linear(769, 1)\n\n        def forward(self, ids, mask, fts):\n            x = self.model(ids, mask)[0]\n            x = self.top(torch.cat((x[:, 0, :], fts),1))\n            return x\n\n    def get_preds_svm(X_test,idx,RidgeReg=0,nfolds=10,C=8,kernel='rbf', seeed=42):\n\n    #     from cuml import Ridge\n    #     from cuml import SVR\n    #     from sklearn.ensemble import StackingRegressor\n        import pickle\n\n        preds = np.zeros((X_test.shape[0]))\n\n        for i in range(nfolds):\n\n            model = pickle.load(open(f'{embedsmodels[idx]}model_fold_{i}.pkl', \"rb\"))\n            preds += model.predict(X_test)\n            del model\n            torch.cuda.empty_cache()\n            gc.collect()\n\n        return np.array(preds)/nfolds\n\n\n\n#     class MarkdownModel(nn.Module):\n#         def __init__(self, model_path):\n#             super(MarkdownModel, self).__init__()\n#             self.model = AutoModel.from_pretrained(model_path)\n#             self.model.gradient_checkpointing_enable()\n#             self.top = nn.Linear(769, 1)\n\n#         def forward(self, ids, mask, fts):\n#             layer_norm = nn.LayerNorm(768).cuda()\n#             x = self.model(ids, mask)[0]\n#             #x = torch.cat((x[:, 0, :], fts), 1)\n#             x = torch.mean(x, 1)\n#             x = layer_norm(x)\n#            # x = self.top(x)\n#             return x\n\n\n    class MarkdownDataset(Dataset):\n\n        def __init__(self, df, model_name_or_path, total_max_len, md_max_len, fts, ggg):\n            super().__init__()\n            self.df = df.reset_index(drop=True)\n            self.md_max_len = md_max_len\n            self.total_max_len = total_max_len  # maxlen allowed by model config\n            self.tokenizer = AutoTokenizer.from_pretrained(tok[ggg])\n            self.fts = fts\n\n        def __getitem__(self, index):\n            row = self.df.iloc[index]\n\n            inputs = self.tokenizer.encode_plus(\n                row.source,\n                None,\n                add_special_tokens=True,\n                max_length=self.md_max_len,\n                padding=\"max_length\",\n                return_token_type_ids=True,\n                truncation=True\n            )\n            code_inputs = self.tokenizer.batch_encode_plus(\n                [str(x) for x in self.fts[row.id][\"codes\"]],\n                add_special_tokens=True,\n                max_length=23,\n                padding=\"max_length\",\n                truncation=True\n            )\n            n_md = self.fts[row.id][\"total_md\"]\n            n_code = self.fts[row.id][\"total_code\"]\n            if n_md + n_code == 0:\n                fts = torch.FloatTensor([0])\n            else:\n                fts = torch.FloatTensor([n_md / (n_md + n_code)])\n\n            ids = inputs['input_ids']\n            for x in code_inputs['input_ids']:\n                ids.extend(x[:-1])\n            ids = ids[:self.total_max_len]\n            if len(ids) != self.total_max_len:\n                ids = ids + [self.tokenizer.pad_token_id, ] * (self.total_max_len - len(ids))\n            ids = torch.LongTensor(ids)\n\n            mask = inputs['attention_mask']\n            for x in code_inputs['attention_mask']:\n                mask.extend(x[:-1])\n            mask = mask[:self.total_max_len]\n            if len(mask) != self.total_max_len:\n                mask = mask + [self.tokenizer.pad_token_id, ] * (self.total_max_len - len(mask))\n            mask = torch.LongTensor(mask)\n\n            assert len(ids) == self.total_max_len\n\n            return ids, mask, fts, torch.FloatTensor([row.pct_rank])\n\n        def __len__(self):\n            return self.df.shape[0]\n\n    BS = 32\n    NW = 2\n    MAX_LEN = 64\n    test_df[\"pct_rank\"] = 0\n    test_fts = get_features(test_df)\n#     test_df.to_csv('test_df.csv', index=False)\n#     test_fts.to_csv('test_fts.csv', index=False)\n    #test_md = test_df[test_df[\"cell_type\"] == \"markdown\"].reset_index(drop=True)\n    test_md = pd.read_csv('test_df_translated.csv')\n    test_md.source = test_md.source.apply(preprocess_text_sc)\n\n\n    \n    for pp in range(len(model_path)):\n        model = MarkdownModel(model_path[pp])\n        model = model.cuda()\n        model.eval()\n        model.load_state_dict(torch.load(ckpt_path[pp])['model'])\n        test_ds = MarkdownDataset(test_md, md_max_len=64,total_max_len=512, model_name_or_path=model_path, fts=test_fts, ggg=pp)\n        test_loader = DataLoader(test_ds, batch_size=BS, shuffle=False, num_workers=NW,\n                              pin_memory=False, drop_last=False)\n\n       # _, y_emb = get_emb(model, test_loader)\n        _, y_emb = validate(model, test_loader)\n        try:\n   #         y_test += get_preds_svm(y_emb,pp)*weights[pp]\n#             \n            y_test += y_emb*weights[pp]\n        except:\n  #          y_test = get_preds_svm(y_emb,pp)*weights[pp]\n\n            y_test = y_emb*weights[pp]\n        \n        del y_emb, model, test_ds, test_loader\n        torch.cuda.empty_cache() \n        gc.collect()\n\n    test_df.loc[test_df[\"cell_type\"] == \"markdown\", \"pred\"] = y_test\n    sub_df = test_df.sort_values(\"pred\").groupby(\"id\")[\"cell_id\"].apply(lambda x: \" \".join(x)).reset_index()\n    sub_df.rename(columns={\"cell_id\": \"cell_order\"}, inplace=True)\n    sub_df.head()\n    sub_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:46:15.031337Z","iopub.execute_input":"2022-08-11T16:46:15.031624Z","iopub.status.idle":"2022-08-11T16:46:15.07561Z","shell.execute_reply.started":"2022-08-11T16:46:15.031593Z","shell.execute_reply":"2022-08-11T16:46:15.074773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"run3()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:46:20.65047Z","iopub.execute_input":"2022-08-11T16:46:20.651074Z","iopub.status.idle":"2022-08-11T16:46:36.997302Z","shell.execute_reply.started":"2022-08-11T16:46:20.651036Z","shell.execute_reply":"2022-08-11T16:46:36.996555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del run3\nimport gc\ngc.collect()\n# torch.cuda.empty_cache()\n# gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-08-11T16:39:51.407107Z","iopub.status.idle":"2022-08-11T16:39:51.408072Z","shell.execute_reply.started":"2022-08-11T16:39:51.407805Z","shell.execute_reply":"2022-08-11T16:39:51.407829Z"},"trusted":true},"execution_count":null,"outputs":[]}]}