{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Vectorizing code/markdown using CodeBert\nThis notebook contains the code I used to vectorize the code and markdown cells of the notebooks in the training set. The extracted feature vectors can then be used for various downstream tasks.  \nLink to dataset(60% of training data): https://www.kaggle.com/datasets/samratthapa/codebert-ai4code-features\n\nEach file corresponds to a notebook in the training dataset.Each file contains the feature vectors of the topmost 256 cells of the corresponding notebook. ","metadata":{"_uuid":"ed7f5125-5070-40f7-82b2-d5ac365f3ac3","_cell_guid":"2cd11f64-3b35-4dc4-8c24-3b2dfe829d6c","jupyter":{"outputs_hidden":false}}},{"cell_type":"code","source":"# %% [code] {\"jupyter\":{\"outputs_hidden\":false},\"execution\":{\"iopub.status.busy\":\"2022-05-31T09:31:33.409460Z\",\"iopub.execute_input\":\"2022-05-31T09:31:33.410190Z\",\"iopub.status.idle\":\"2022-05-31T09:31:40.576230Z\",\"shell.execute_reply.started\":\"2022-05-31T09:31:33.410092Z\",\"shell.execute_reply\":\"2022-05-31T09:31:40.575454Z\"}}\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom transformers import AutoTokenizer, AutoModel\nimport os\nfrom torch.utils.data import Dataset,DataLoader\nimport glob\nimport torch\nfrom transformers import RobertaTokenizer, RobertaConfig, RobertaModel\nimport fasttext\nfrom tqdm import tqdm\n\ndevice = torch.device(\"cuda\")\nprint(torch.version.cuda)","metadata":{"_uuid":"2c453c03-16d9-4ebc-ba59-4cc8ede6f2e4","_cell_guid":"45c65c4a-8e44-4854-ac4c-5e782f37aa5a","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2022-06-01T13:38:53.996535Z","iopub.execute_input":"2022-06-01T13:38:53.998584Z","iopub.status.idle":"2022-06-01T13:39:01.967163Z","shell.execute_reply.started":"2022-06-01T13:38:53.996938Z","shell.execute_reply":"2022-06-01T13:39:01.966140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_order = pd.read_csv('../input/AI4Code/train_orders.csv')\ntrain_order.index = train_order.id\ntrain_order.drop(columns=['id'])\nt_order = {}\nfor i,order in zip(train_order.id,train_order.cell_order):\n    t_order[i] = order\n\nall_notebooks = {}\n\nfor csv in glob.glob('../input/AI4Code/train/*'):\n    all_notebooks[os.path.split(csv)[-1][:-5]] = csv","metadata":{"execution":{"iopub.status.busy":"2022-06-01T13:39:29.171249Z","iopub.execute_input":"2022-06-01T13:39:29.171626Z","iopub.status.idle":"2022-06-01T13:39:33.817576Z","shell.execute_reply.started":"2022-06-01T13:39:29.171598Z","shell.execute_reply":"2022-06-01T13:39:33.816537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = RobertaTokenizer.from_pretrained(\"microsoft/codebert-base\")\nencoder_model = RobertaModel.from_pretrained(\"microsoft/codebert-base\")\nencoder_model = encoder_model.to(device)","metadata":{"execution":{"iopub.status.busy":"2022-06-01T13:39:43.040987Z","iopub.execute_input":"2022-06-01T13:39:43.041930Z","iopub.status.idle":"2022-06-01T13:40:43.849324Z","shell.execute_reply.started":"2022-06-01T13:39:43.041889Z","shell.execute_reply":"2022-06-01T13:40:43.847889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.makedirs('features',exist_ok=True)\nfor enum,idx in enumerate(tqdm(all_notebooks)):\n    if enum%1000==0:\n        print(enum)\n    notebook_id = os.path.split(all_notebooks[idx])[-1][:-5]\n    notebook = pd.read_json(all_notebooks[idx])\n    cell_order = t_order[notebook_id].split(\" \")\n    notebook = notebook.loc[cell_order].reset_index()\n    notebook = notebook[:256]\n    source_code= notebook.source.to_list()\n    tokens = tokenizer(source_code,padding='longest',truncation='longest_first',return_tensors='pt',max_length=512)\n    # print(tokens['input_ids'].size())\n    with torch.no_grad():\n        _,output = encoder_model(input_ids=tokens['input_ids'].to(device),attention_mask=tokens['attention_mask'].to(device),return_dict=False)\n    torch.save(output,os.path.join(\"./features\",notebook_id+\".cuda\"))    ","metadata":{},"execution_count":null,"outputs":[]}]}