{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!ln -s /kaggle/working/ /workspace","metadata":{"execution":{"iopub.status.busy":"2022-12-20T11:06:15.879760Z","iopub.execute_input":"2022-12-20T11:06:15.884791Z","iopub.status.idle":"2022-12-20T11:06:17.038249Z","shell.execute_reply.started":"2022-12-20T11:06:15.884747Z","shell.execute_reply":"2022-12-20T11:06:17.036064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\n\n%cd /workspace\n!rm -rf proc-ai4code\n%mkdir proc-ai4code\n\nos.environ[\"RAW_DATA_DIR\"] = \"/kaggle/input/AI4Code\"\nos.environ[\"PROCESSED_DATA_DIR\"] = \"/workspace/proc-ai4code\"\nos.environ[\"SEED\"] = \"42\"\nos.environ[\"PICKLE_PROTOCOL\"] = \"4\"\nos.environ[\"NON_MASKED_INDEX\"] = \"-100\"\nos.environ[\"MAX_GRADIENT\"] = \"10.0\"\nos.environ[\"MLM_PROBABILITY\"] = \"0.15\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-20T11:06:17.044834Z","iopub.execute_input":"2022-12-20T11:06:17.045681Z","iopub.status.idle":"2022-12-20T11:06:19.316027Z","shell.execute_reply.started":"2022-12-20T11:06:17.045602Z","shell.execute_reply":"2022-12-20T11:06:19.314290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /workspace\n!rm -rf Model-v5\n\n%cd /workspace\n!git clone https://github.com/nthehai01/Model-v5.git","metadata":{"execution":{"iopub.status.busy":"2022-12-20T11:13:32.111144Z","iopub.execute_input":"2022-12-20T11:13:32.111542Z","iopub.status.idle":"2022-12-20T11:13:34.764419Z","shell.execute_reply.started":"2022-12-20T11:13:32.111505Z","shell.execute_reply":"2022-12-20T11:13:34.763290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# %cd /workspace/Model-v5\n# !pip install -r requirements.txt","metadata":{"execution":{"iopub.status.busy":"2022-12-20T10:55:05.208788Z","iopub.execute_input":"2022-12-20T10:55:05.211298Z","iopub.status.idle":"2022-12-20T10:55:05.218410Z","shell.execute_reply.started":"2022-12-20T10:55:05.211252Z","shell.execute_reply":"2022-12-20T10:55:05.216886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%cd /workspace\n!rm -rf outputs\n!mkdir outputs","metadata":{"execution":{"iopub.status.busy":"2022-12-20T11:06:22.569207Z","iopub.execute_input":"2022-12-20T11:06:22.571545Z","iopub.status.idle":"2022-12-20T11:06:24.640159Z","shell.execute_reply.started":"2022-12-20T11:06:22.571500Z","shell.execute_reply":"2022-12-20T11:06:24.638862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train MLM","metadata":{}},{"cell_type":"code","source":"%cd /workspace/Model-v5\n\n!python train_mlm.py \\\n    --preprocess_data \\\n    --train_size 0.5 \\\n    --val_size 0.001 \\\n    --start_train_idx 69600 \\\n    --max_n_md_cells 64 \\\n    --max_n_code_cells 64 \\\n    --max_len 32 \\\n    --epochs 10 \\\n    --n_epochs_per_save 1 \\\n    --lr 4e-5 \\\n    --checkpoint_path /kaggle/input/model-v5-checkpoint/outputs/mlm/notebook_mlm_epoch9.tar \\\n    --no-restore_weights_only \\\n    --output_dir /workspace/outputs/mlm \\\n    --test_ids_path /kaggle/input/ai4code-test-dataset/test_ids.pkl \\\n    --non_en_ids_path /kaggle/input/ai4code-non-english-notebooks/non_en_ids.pkl","metadata":{"execution":{"iopub.status.busy":"2022-12-20T11:13:35.030812Z","iopub.execute_input":"2022-12-20T11:13:35.031168Z","iopub.status.idle":"2022-12-20T11:16:33.906958Z","shell.execute_reply.started":"2022-12-20T11:13:35.031133Z","shell.execute_reply":"2022-12-20T11:16:33.905774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train notebook ordering","metadata":{}},{"cell_type":"code","source":"# %cd /workspace/Model-v5\n\n# !python train_ordering.py \\\n#     --no-freeze_pretrained \\\n#     --preprocess_data \\\n#     --train_size 0.5 \\\n#     --start_train_idx 0 \\\n#     --epochs 1 \\\n#     --start_epoch 1 \\\n#     --n_epochs_per_save 2 \\\n#     --lr 4.5e-5 \\\n#     --checkpoint_path /kaggle/input/model-v5-checkpoint/outputs/model_epoch2.tar \\\n#     --restore_weights_only \\\n#     --output_dir /workspace/outputs/ordering \\\n#     --test_ids_path /kaggle/input/ai4code-test-dataset/test_ids.pkl \\\n#     --non_en_ids_path /kaggle/input/ai4code-non-english-notebooks/non_en_ids.pkl","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}