{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!ln -s /kaggle/working/ /workspace","metadata":{"execution":{"iopub.status.busy":"2023-01-10T16:14:08.067947Z","iopub.execute_input":"2023-01-10T16:14:08.068419Z","iopub.status.idle":"2023-01-10T16:14:09.03245Z","shell.execute_reply.started":"2023-01-10T16:14:08.068338Z","shell.execute_reply":"2023-01-10T16:14:09.030857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n%cd /workspace\n!rm -rf proc-ai4code\n%mkdir proc-ai4code\n\nos.environ[\"RAW_DATA_DIR\"] = \"/kaggle/input/AI4Code\"\nos.environ[\"PROCESSED_DATA_DIR\"] = \"/workspace/proc-ai4code\"\nos.environ[\"SEED\"] = \"42\"\nos.environ[\"PICKLE_PROTOCOL\"] = \"4\"\nos.environ[\"NON_MASKED_INDEX\"] = \"-100\"\nos.environ[\"MAX_GRADIENT\"] = \"10.0\"\nos.environ[\"MLM_PROBABILITY\"] = \"0.15\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-10T16:14:09.053004Z","iopub.execute_input":"2023-01-10T16:14:09.056081Z","iopub.status.idle":"2023-01-10T16:14:12.320565Z","shell.execute_reply.started":"2023-01-10T16:14:09.056026Z","shell.execute_reply":"2023-01-10T16:14:12.319031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /workspace\n!rm -rf Model-v5\n\n%cd /workspace\n!git clone https://github.com/nthehai01/Model-v5.git","metadata":{"execution":{"iopub.status.busy":"2023-01-10T16:14:12.32589Z","iopub.execute_input":"2023-01-10T16:14:12.328433Z","iopub.status.idle":"2023-01-10T16:14:15.748916Z","shell.execute_reply.started":"2023-01-10T16:14:12.328383Z","shell.execute_reply":"2023-01-10T16:14:15.74773Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /workspace/Model-v5\n# !pip install -r requirements.txt","metadata":{"execution":{"iopub.status.busy":"2023-01-10T16:14:15.752952Z","iopub.execute_input":"2023-01-10T16:14:15.753283Z","iopub.status.idle":"2023-01-10T16:14:15.757941Z","shell.execute_reply.started":"2023-01-10T16:14:15.75324Z","shell.execute_reply":"2023-01-10T16:14:15.756568Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /workspace\n!rm -rf outputs\n!mkdir outputs","metadata":{"execution":{"iopub.status.busy":"2023-01-10T16:14:15.759734Z","iopub.execute_input":"2023-01-10T16:14:15.760473Z","iopub.status.idle":"2023-01-10T16:14:17.640708Z","shell.execute_reply.started":"2023-01-10T16:14:15.76042Z","shell.execute_reply":"2023-01-10T16:14:17.639303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train MLM","metadata":{}},{"cell_type":"code","source":"# %cd /workspace/Model-v5\n\n# !python train_mlm.py \\\n#     --preprocess_data \\\n#     --train_size 0.5 \\\n#     --start_train_idx 69600 \\\n#     --max_n_md_cells 64 \\\n#     --max_n_code_cells 64 \\\n#     --max_len 32 \\\n#     --epochs 10 \\\n#     --n_epochs_per_save 1 \\\n#     --lr 4e-5 \\\n#     --checkpoint_path /kaggle/input/model-v5-checkpoint/outputs/mlm/notebook_mlm_epoch9.tar \\\n#     --output_dir /workspace/outputs/mlm \\\n#     --test_ids_path /kaggle/input/ai4code-test-dataset/test_ids.pkl \\\n#     --non_en_ids_path /kaggle/input/ai4code-non-english-notebooks/non_en_ids.pkl","metadata":{"execution":{"iopub.status.busy":"2023-01-10T16:14:17.643068Z","iopub.execute_input":"2023-01-10T16:14:17.643522Z","iopub.status.idle":"2023-01-10T16:14:17.649417Z","shell.execute_reply.started":"2023-01-10T16:14:17.643462Z","shell.execute_reply":"2023-01-10T16:14:17.648258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train notebook ordering","metadata":{}},{"cell_type":"code","source":"%cd /workspace/Model-v5\n\n!python train_ordering.py \\\n    --preprocess_data \\\n    --code_pretrained microsoft/codebert-base \\\n    --md_pretrained microsoft/codebert-base \\\n    --train_size 0.35 \\\n    --start_train_idx 97400 \\\n    --max_len 128 \\\n    --batch_size 1 \\\n    --epochs 15 \\\n    --n_epochs_per_save 1 \\\n    --lr 3e-5 \\\n    --checkpoint_path /kaggle/input/codebert-checkpoint/outputs/ordering/notebook_ordering_epoch14.tar \\\n    --no-restore_weights_only \\\n    --output_dir /workspace/outputs/ordering \\\n    --test_ids_path /kaggle/input/ai4code-test-dataset/test_ids.pkl \\\n    --non_en_ids_path /kaggle/input/ai4code-non-english-notebooks/non_en_ids.pkl","metadata":{"execution":{"iopub.status.busy":"2023-01-10T16:14:17.651479Z","iopub.execute_input":"2023-01-10T16:14:17.651949Z","iopub.status.idle":"2023-01-10T16:14:17.659956Z","shell.execute_reply.started":"2023-01-10T16:14:17.651906Z","shell.execute_reply":"2023-01-10T16:14:17.658865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Infer notebook ordering","metadata":{}},{"cell_type":"code","source":"# %cd /workspace/Model-v5\n\n# !python infer_ordering.py \\\n#     --max_len 128 \\\n#     --checkpoint_path /kaggle/input/model-v5-checkpoint/outputs/ordering/notebook_ordering_epoch13.tar \\\n#     --output_dir /workspace/outputs","metadata":{"execution":{"iopub.status.busy":"2023-01-10T16:17:36.428614Z","iopub.execute_input":"2023-01-10T16:17:36.428996Z","iopub.status.idle":"2023-01-10T16:18:01.332207Z","shell.execute_reply.started":"2023-01-10T16:17:36.428962Z","shell.execute_reply":"2023-01-10T16:18:01.330938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}