{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":30894,"databundleVersionId":3147400,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2024-04-21T13:58:43.554533Z","iopub.execute_input":"2024-04-21T13:58:43.554912Z","iopub.status.idle":"2024-04-21T13:58:43.565835Z","shell.execute_reply.started":"2024-04-21T13:58:43.554862Z","shell.execute_reply":"2024-04-21T13:58:43.565147Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install peft\n!pip install accelerate\n!pip install -i https://pypi.org/simple/ bitsandbytes","metadata":{"execution":{"iopub.status.busy":"2024-04-21T13:58:43.567620Z","iopub.execute_input":"2024-04-21T13:58:43.568028Z","iopub.status.idle":"2024-04-21T13:59:55.837095Z","shell.execute_reply.started":"2024-04-21T13:58:43.567998Z","shell.execute_reply":"2024-04-21T13:59:55.836095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import datetime\nimport os\nimport sys\nimport peft\n\nimport torch\nfrom peft import (\n    LoraConfig,\n    get_peft_model,\n    get_peft_model_state_dict,\n    prepare_model_for_kbit_training,\n    set_peft_model_state_dict,\n)\nfrom transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForSeq2Seq","metadata":{"execution":{"iopub.status.busy":"2024-04-21T13:59:55.838445Z","iopub.execute_input":"2024-04-21T13:59:55.838737Z","iopub.status.idle":"2024-04-21T14:00:26.675802Z","shell.execute_reply.started":"2024-04-21T13:59:55.838709Z","shell.execute_reply":"2024-04-21T14:00:26.675055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datasets import load_dataset\n\n# 加载数据集，先取出总数据的 11%\ntemporary_dataset = load_dataset('csv', data_files='/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv', split='train[:1%]')\n\n# 从这 11% 中进一步分割出大约 90.9% 作为训练集（实际上是总数据的 10%），剩下的作为测试集（实际上是总数据的 1%）\nsplit_dataset = temporary_dataset.train_test_split(test_size=0.1)  # 0.091 * 11% ≈ 1%\ntrain_dataset = split_dataset['train']\neval_dataset = split_dataset['test']\n\n# train_dataset = load_dataset('csv', data_files='/kaggle/input/g-research-crypto-forecasting/train.csv', split='train')\n# eval_dataset = load_dataset('csv', data_files='/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv', split='train')\n\nbase_model = \"Qwen/Qwen1.5-7B-Chat\"\n# \"codellama/CodeLlama-7b-hf\"\nmodel = AutoModelForCausalLM.from_pretrained(\n    base_model,\n    load_in_8bit=True,\n    torch_dtype=torch.float16,\n    device_map=\"auto\",\n)\ntokenizer = AutoTokenizer.from_pretrained(base_model) #use_fast=False\n\ntokenizer.add_eos_token = True\ntokenizer.pad_token_id = 0\ntokenizer.padding_side = \"left\"\n\ndef tokenize(prompt):\n    result = tokenizer(\n        prompt,\n        truncation=True,\n        max_length=200,\n        padding=False,\n        return_tensors=None,\n    )\n\n    # \"self-supervised learning\" means the labels are also the inputs:\n    result[\"labels\"] = result[\"input_ids\"].copy()\n\n    return result\n\ndef generate_and_tokenize_prompt(data_point):\n    prompt = f\"\"\"[INST]\n    <<SYS>>You are a cryptocurrency market prediction model. Here is the data of a crypto asset in the past one minute. You need to give your prediction for the Target: residual log-returns for the asset over a 15 minute horizon. <</SYS>>\n    timestamp: {data_point['timestamp']}\n    Asset_ID: {data_point['Asset_ID']}\n    Count: {data_point['Count']}\n    Open: {data_point['Open']}\n    High: {data_point['High']}\n    Low: {data_point['Low']}\n    Close: {data_point['Close']}\n    Volume: {data_point['Volume']}\n    VWAP: {data_point['VWAP']}\n    Directly give the Target value here:\n    [/INST] {data_point['Target']}\n    \"\"\"\n    return tokenize(prompt)\n\ntokenized_train_dataset = train_dataset.map(generate_and_tokenize_prompt)\ntokenized_val_dataset = eval_dataset.map(generate_and_tokenize_prompt)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T14:00:26.676883Z","iopub.execute_input":"2024-04-21T14:00:26.677434Z","iopub.status.idle":"2024-04-21T14:03:49.237720Z","shell.execute_reply.started":"2024-04-21T14:00:26.677401Z","shell.execute_reply":"2024-04-21T14:03:49.236709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.train() # put model back into training mode\nmodel = prepare_model_for_kbit_training(model)\nos.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:64'\n\nconfig = LoraConfig(\n    r=2,\n    lora_alpha=2,\n    target_modules=[\n    \"q_proj\",\n    \"k_proj\",\n    \"v_proj\",\n    \"o_proj\",\n],\n    lora_dropout=0.05,\n    bias=\"none\",\n    task_type=\"CAUSAL_LM\",\n)\nmodel = get_peft_model(model, config)\n\nbatch_size = 12\nper_device_train_batch_size = 6\ngradient_accumulation_steps = batch_size // per_device_train_batch_size\noutput_dir = \"llama2_test_1\"\n\ntraining_args = TrainingArguments(\n        per_device_train_batch_size=per_device_train_batch_size,\n        gradient_accumulation_steps=gradient_accumulation_steps,\n        warmup_steps=40,\n        max_steps=200,\n        learning_rate=1e-4,\n        fp16=True,\n        logging_steps=50,\n        optim=\"adamw_torch\",\n        evaluation_strategy=\"steps\", # if val_set_size > 0 else \"no\",\n        save_strategy=\"steps\",\n        eval_steps=50,\n        save_steps=50,\n        output_dir=output_dir,\n        load_best_model_at_end=True,\n        group_by_length=True, # group sequences of roughly the same length together to speed up training\n        report_to=\"none\", # if use_wandb else \"none\",\n        run_name=\"none\", # if use_wandb else None,\n    )\n\ntrainer = Trainer(\n    model=model,\n    train_dataset=tokenized_train_dataset,\n    eval_dataset=tokenized_val_dataset,\n    args=training_args,\n    data_collator=DataCollatorForSeq2Seq(\n        tokenizer, pad_to_multiple_of=8, return_tensors=\"pt\", padding=True\n    ),\n)\n\nmodel.config.use_cache = False","metadata":{"execution":{"iopub.status.busy":"2024-04-21T14:03:49.240583Z","iopub.execute_input":"2024-04-21T14:03:49.241312Z","iopub.status.idle":"2024-04-21T14:03:49.534848Z","shell.execute_reply.started":"2024-04-21T14:03:49.241271Z","shell.execute_reply":"2024-04-21T14:03:49.534057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with torch.autocast(\"cuda\"): \n    trainer.train() # resume_from_checkpoint=True","metadata":{"execution":{"iopub.status.busy":"2024-04-21T14:03:49.535975Z","iopub.execute_input":"2024-04-21T14:03:49.536309Z","iopub.status.idle":"2024-04-21T15:07:35.807877Z","shell.execute_reply.started":"2024-04-21T14:03:49.536283Z","shell.execute_reply":"2024-04-21T15:07:35.807082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.save_pretrained(output_dir)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T15:07:35.809166Z","iopub.execute_input":"2024-04-21T15:07:35.809749Z","iopub.status.idle":"2024-04-21T15:07:36.122242Z","shell.execute_reply.started":"2024-04-21T15:07:35.809722Z","shell.execute_reply":"2024-04-21T15:07:36.121345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_prompt = \"\"\"[INST]<<SYS>>You are a cryptocurrency market prediction model. Here is the data of a crypto asset in the past one minute. You need to give your prediction for the Target: residual log-returns for the asset over a 15 minute horizon. <</SYS>>\ntimestamp: 1632182640\nAsset_ID: 5\nCount: 362.0\nOpen: 4.197323908333333\nHigh: 4.1998\nLow: 4.178999999999999\nClose: 4.182849999999998\nVolume: 55306.26469690001\nVWAP: 4.191202370279445\nDirectly give the Target value here:\n[/INST] \n\"\"\"\ntokenizer = AutoTokenizer.from_pretrained(base_model, padding_side='left')\nmodel_input = tokenizer(eval_prompt, return_tensors=\"pt\").to(\"cuda\")\n\n# 进行预测\nmodel.eval()\nwith torch.no_grad():\n    output = model.generate(**model_input, max_new_tokens=200)[0]\n    print(tokenizer.decode(output, skip_special_tokens=True))","metadata":{"execution":{"iopub.status.busy":"2024-04-21T15:07:36.124392Z","iopub.execute_input":"2024-04-21T15:07:36.125170Z","iopub.status.idle":"2024-04-21T15:07:44.817033Z","shell.execute_reply.started":"2024-04-21T15:07:36.125136Z","shell.execute_reply":"2024-04-21T15:07:44.816158Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_prompt = \"\"\"[INST]<<SYS>>You are a cryptocurrency market prediction model. Here is the data of a crypto asset in the past one minute. You need to give your prediction for the Target: residual log-returns for the asset over a 15 minute horizon. <</SYS>>\ntimestamp: 1632182640\nAsset_ID: 5\nCount: 362.0\nOpen: 4.197323908333333\nHigh: 4.1998\nLow: 4.178999999999999\nClose: 4.182849999999998\nVolume: 55306.26469690001\nVWAP: 4.191202370279445\nDirectly give the Target value here:\n[/INST] \n\"\"\"\ntokenizer = AutoTokenizer.from_pretrained(base_model, padding_side='left')\nmodel_input = tokenizer(eval_prompt, return_tensors=\"pt\").to(\"cuda\")\n\n# 进行预测\nmodel.eval()\nwith torch.no_grad():\n    output = model.generate(**model_input, max_new_tokens=200)[0]\n    print(tokenizer.decode(output, skip_special_tokens=True))","metadata":{"execution":{"iopub.status.busy":"2024-04-21T15:08:42.988020Z","iopub.execute_input":"2024-04-21T15:08:42.988736Z","iopub.status.idle":"2024-04-21T15:08:49.865799Z","shell.execute_reply.started":"2024-04-21T15:08:42.988706Z","shell.execute_reply":"2024-04-21T15:08:49.864663Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eval_prompt = \"\"\"[INST]<<SYS>>You are a cryptocurrency market prediction model. Here is the data of a crypto asset in the past one minute. You need to give your prediction for the Target: residual log-returns for the asset over a 15 minute horizon. <</SYS>>\ntimestamp: 1632182640\nAsset_ID: 5\nCount: 362.0\nOpen: 4.197323908333333\nHigh: 4.1998\nLow: 4.178999999999999\nClose: 4.182849999999998\nVolume: 55306.26469690001\nVWAP: 4.191202370279445\nDirectly give the Target value here:\n[/INST] \n\"\"\"\ntokenizer = AutoTokenizer.from_pretrained(base_model, padding_side='left')\nmodel_input = tokenizer(eval_prompt, return_tensors=\"pt\").to(\"cuda\")\n\n# 进行预测\nmodel.eval()\nwith torch.no_grad():\n    output = model.generate(**model_input, max_new_tokens=200)[0]\n    print(tokenizer.decode(output, skip_special_tokens=True))","metadata":{"execution":{"iopub.status.busy":"2024-04-21T15:09:01.880236Z","iopub.execute_input":"2024-04-21T15:09:01.880710Z","iopub.status.idle":"2024-04-21T15:09:08.683524Z","shell.execute_reply.started":"2024-04-21T15:09:01.880682Z","shell.execute_reply":"2024-04-21T15:09:08.682576Z"},"trusted":true},"execution_count":null,"outputs":[]}]}