{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Original notebook:\n- https://www.kaggle.com/code/bestfitting/en-3models-vit-blip-convnext-notebook/notebook\n\nI want to understand how it works and adapt to be able to use Diffusion Model and generate images from existing images.\n\n# Objectives:\n- understanding the code and annotating it\n\n## Prepare the Environment by installing required packages and setting up pretrained models for later use\n\n- Installing Packages\n- Creating a Directory named \"pkgs\" used to store some files.\n- Copying Files (related to the \"sentence-transformers\" package, a pretrained model called \"blip2-opt-2.7b\" and another pretrained model called \"CLIP-convnext_xxlarge-laion2B-s34B-b82K-augreg-soup\")\n- Extracting and Cleaning Files: The cat and tar commands are used to extract and clean up some compressed files. In the case of \"blip2-opt-2.7b,\" the cat command concatenates multiple parts of a compressed file into a single file, which is then extracted using the tar -xzvf command. The extracted file is a binary file with the extension .bin.\n- Cleaning Up: rm -rf commands remove the unnecessary compressed files that were extracted in the previous steps, freeing up disk space.\n","metadata":{}},{"cell_type":"code","source":"pkgs = [\n    '/kaggle/input/diffusion2023-package/transformers-4.27.4-py3-none-any.whl',\n    '/kaggle/input/diffusion2023-package/ftfy-6.1.1-py3-none-any.whl',\n    '/kaggle/input/diffusion2023-package/timm-0.8.21.dev0-py3-none-any.whl',\n    '/kaggle/input/diffusion2023-package/open_clip_torch-2.19.0-py3-none-any.whl',\n]\n!pip install -qq --no-deps --no-python-version-warning --no-warn-conflicts --no-warn-script-location {' '.join(pkgs)} \n\n!mkdir -p pkgs\n!cp -r /kaggle/input/diffusion2023-package/sentence-transformers-2.2.2 ./pkgs\n!cd ./pkgs/sentence-transformers-2.2.2; pip install -qq .\n\n!cp -r /kaggle/input/diffusion2023-pretrained/blip2-opt-2.7b ./\n!cat /kaggle/working/blip2-opt-2.7b/pytorch_model-00001-of-00002.bin.tar.gz* > /kaggle/working/blip2-opt-2.7b/pytorch_model-00001-of-00002.bin.tar.gz\n!cd /kaggle/working/blip2-opt-2.7b/; tar -xzvf pytorch_model-00001-of-00002.bin.tar.gz\n!rm -rf /kaggle/working/blip2-opt-2.7b/pytorch_model-00001-of-00002.bin.tar.gz*\n\n!cp -r /kaggle/input/diffusion2023-pretrained/CLIP-convnext_xxlarge-laion2B-s34B-b82K-augreg-soup ./\n!cd /kaggle/working/CLIP-convnext_xxlarge-laion2B-s34B-b82K-augreg-soup/;cat open_clip_pytorch_model.bin.tar* | tar -xv\n!rm -rf /kaggle/working/CLIP-convnext_xxlarge-laion2B-s34B-b82K-augreg-soup/open_clip_pytorch_model.bin.tar*\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:52:05.535552Z","iopub.execute_input":"2023-06-15T20:52:05.536026Z","iopub.status.idle":"2023-06-15T20:56:37.975467Z","shell.execute_reply.started":"2023-06-15T20:52:05.535995Z","shell.execute_reply":"2023-06-15T20:56:37.974091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Reference the original files stored in specific directories without having to duplicate the files in your current working directory\n\n- Creating a Directory: the directory named \"raw\" will be used to store some raw data files.\n- Creating Symbolic Links: The ln -s command creates symbolic links to files or directories (from the files in the \"/kaggle/input/stable-diffusion-image-to-prompts/\" directory to the \"raw\" directory in the current working directory. This allows you to access the files in the \"raw\" directory without copying them, saving disk space).\n- Creating Result Directories: the directory named \"result/models\" will be used to store some result files.\n- Creating Symbolic Links to Model Files: from the files in the \"/kaggle/input/diffusion2023-model/\" directory to the \"result/models\" directory in the current working directory.","metadata":{}},{"cell_type":"code","source":"!mkdir -p raw\n!ln -s /kaggle/input/stable-diffusion-image-to-prompts/* ./raw/\n\n!mkdir -p ./result/models/\n!ln -s /kaggle/input/diffusion2023-model/* ./result/models/","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:56:37.979179Z","iopub.execute_input":"2023-06-15T20:56:37.979869Z","iopub.status.idle":"2023-06-15T20:56:41.984475Z","shell.execute_reply.started":"2023-06-15T20:56:37.979812Z","shell.execute_reply":"2023-06-15T20:56:41.980172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport sys\nimport argparse\nimport time\nimport gc\nimport shutil\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport open_clip\n\nfrom tqdm import tqdm\nfrom argparse import Namespace\nfrom PIL import Image\nfrom scipy import spatial\nfrom typing import Optional, Tuple\nfrom multiprocessing import Process\nfrom torch.nn import DataParallel\nfrom torch.nn.parallel import DistributedDataParallel\nfrom torch.cuda.amp import autocast\nfrom torch.autograd import Variable\nfrom torch.utils.data import Dataset, DataLoader, SequentialSampler\nfrom transformers import CLIPProcessor\nfrom transformers import CLIPModel\nfrom transformers.models.clip import CLIPConfig\nfrom transformers import Blip2ForConditionalGeneration\n\nopj = os.path.join\nope = os.path.exists","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:57:13.439899Z","iopub.execute_input":"2023-06-15T20:57:13.440287Z","iopub.status.idle":"2023-06-15T20:57:25.543066Z","shell.execute_reply.started":"2023-06-15T20:57:13.440252Z","shell.execute_reply":"2023-06-15T20:57:25.542101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Config: store paths, column names, and constants that will be used throughout the code\n\n- TRANSFORMERS: a dictionary that maps pretrained model names to their respective paths (keys are model names and the values are the corresponding paths).","metadata":{}},{"cell_type":"code","source":"sample_submission = pd.read_csv('/kaggle/input/stable-diffusion-image-to-prompts/sample_submission.csv')\nIS_KAGGLE_SUBMIT = len(sample_submission) != 7 * 384 # determine if the code is running in a Kaggle submit environment or not\nRESULT_DIR = '/kaggle/working/result'\nDATA_DIR = '/kaggle/working'\nTRANSFORMERS = {\n    'openai/clip-vit-large-patch14-336': '/kaggle/input/diffusion2023-pretrained/clip-vit-large-patch14-336',\n    'Salesforce/blip2-opt-2.7b': '/kaggle/working/blip2-opt-2.7b',\n    'laion/CLIP-convnext_xxlarge-laion2B-s34B-b82K-augreg-soup': \"/kaggle/working/CLIP-convnext_xxlarge-laion2B-s34B-b82K-augreg-soup\",\n}\nPRETRAINED_MODEL_NAME_OR_PATH = lambda model_name: TRANSFORMERS.get(model_name, model_name)\n\nEMBEDDING_LENGTH = 384 # length of the embeddings generated by the models\nID = 'imgId_eId' # column name in the dataset that contains the image ID and the embedding ID\nTARGET = 'val' # column name in the dataset that contains the target variable\nIMG_ID = 'imgId' # column name in the dataset that contains the image ID\nPROMPT = 'prompt' # column name in the dataset that contains the prompt associated with an image","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-06-15T20:57:25.544780Z","iopub.execute_input":"2023-06-15T20:57:25.545177Z","iopub.status.idle":"2023-06-15T20:57:25.577718Z","shell.execute_reply.started":"2023-06-15T20:57:25.545133Z","shell.execute_reply":"2023-06-15T20:57:25.576867Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset:\n- allows you to load and access individual data samples from the dataset, applying transformations if needed","metadata":{}},{"cell_type":"code","source":"class DiffusionDataset(Dataset): # inherits from the torch.utils.data.Dataset class, used to load and process data for the diffusion model\n    def __init__(self, args, dataset='valid', transform=None):\n        self.args = args\n        self.dataset = dataset\n        self.transform = transform\n        self.df, self.targets = self.load_data()\n\n    def load_data(self):\n        self.img_dir = f'{DATA_DIR}/raw/images'\n        test_df = pd.read_csv(f'{DATA_DIR}/raw/sample_submission.csv')\n        test_df[[IMG_ID, 'eId']] = test_df['imgId_eId'].str.split(\"_\", expand=True) # splits the 'imgId_eId' column into separate 'imgId' and 'eId' columns\n        df = test_df[[IMG_ID]].drop_duplicates().reset_index(drop=True) # dataframe df containing only the unique image IDs and an empty prompt column\n        targets = test_df['val'].values.reshape(-1, EMBEDDING_LENGTH) # reshaped into a 2D array\n        df[PROMPT] = ''\n        return df, targets\n\n    def __getitem__(self, index):\n        row = self.df.iloc[index] # retrieves the row at the given index from the dataframe\n        img_id = row[IMG_ID] # extracts the image ID\n        img_file = f'{self.img_dir}/{img_id}.png' # constructs the image file path\n        image = Image.open(img_file)\n        inputs = self.transform(image)\n        inputs['target'] = torch.tensor(self.targets[index])\n        return inputs\n\n    def __len__(self):\n        return len(self.df)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:57:29.857790Z","iopub.execute_input":"2023-06-15T20:57:29.858156Z","iopub.status.idle":"2023-06-15T20:57:29.868904Z","shell.execute_reply.started":"2023-06-15T20:57:29.858125Z","shell.execute_reply":"2023-06-15T20:57:29.867948Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Predictor:\n- utility functions useful for loading pretrained model state dictionaries into PyTorch models, allowing for flexibility in key replacement and handling of size mismatches.","metadata":{}},{"cell_type":"code","source":"def state_dict_replace(load_state_dict, replace_key='transformer.'):\n    new_load_state_dict = dict()\n    for key in load_state_dict.keys():\n        if key.count(replace_key) > 0:\n            dst_key = key.replace(replace_key, '')\n        else:\n            dst_key = key\n        new_load_state_dict[dst_key] = load_state_dict[key]\n    load_state_dict = new_load_state_dict\n    return load_state_dict\n\n\ndef load_pretrained(net, pretrained_file, strict=False, can_print=True):\n    if can_print:\n        print(f'load pretrained file: {pretrained_file}')\n    load_state_dict = torch.load(pretrained_file, map_location=torch.device('cpu'))\n    net = load_pretrained_state_dict(net, load_state_dict, strict=strict, can_print=can_print)\n    return net\n\n\ndef load_pretrained_state_dict(net, load_state_dict, strict=False, can_print=True):\n    if 'epoch' in load_state_dict and can_print:\n        epoch = load_state_dict['epoch']\n        print(f'load epoch:{epoch:.2f}')\n    if 'state_dict' in load_state_dict:\n        load_state_dict = load_state_dict['state_dict']\n    if type(net) == DataParallel or type(net) == DistributedDataParallel:\n        state_dict = net.module.state_dict()\n    else:\n        state_dict = net.state_dict()\n    load_state_dict = state_dict_replace(load_state_dict, '_orig_mod.')\n    for key in list(load_state_dict.keys()):\n        if key not in state_dict:\n            if strict:\n                raise Exception(f'not in {key}')\n            if can_print:\n                print('not in', key)\n            continue\n        if load_state_dict[key].size() != state_dict[key].size():\n            if strict:\n                raise Exception(f'size not the same {key}')\n            if can_print:\n                print('size not the same', key)\n            continue\n        state_dict[key] = load_state_dict[key]\n    if type(net) == DataParallel or type(net) == DistributedDataParallel:\n        net.module.load_state_dict(state_dict)\n    else:\n        net.load_state_dict(state_dict)\n    return net","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:05.762285Z","iopub.execute_input":"2023-06-15T20:59:05.762737Z","iopub.status.idle":"2023-06-15T20:59:05.781027Z","shell.execute_reply.started":"2023-06-15T20:59:05.762698Z","shell.execute_reply":"2023-06-15T20:59:05.779919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Blip2ValidTransformer, CLIPValidTransformer:\n- used for transforming input data into appropriate formats for the Blip2 and CLIP models during validation\n\n### Blip2ValidTransformer:\n- The image and text inputs are passed to the Blip2 processor, which performs tokenization, truncation, padding, etc.\n- The resulting data dictionary is processed to remove the batch dimension by 'squeezing' the tensors.\n- The transformed data dictionary is returned\n\n### CLIPValidTransformer:\n- The image is passed to the CLIP processor, which performs image preprocessing (resizing and center cropping)\n- The resulting image data is extracted from the processor's output and returned as a dictionary with the key 'image'.","metadata":{}},{"cell_type":"code","source":"class Blip2ValidTransformer:\n    def __init__(self, args):\n        self.args = args\n        from transformers import AutoProcessor\n        self.blip2_processor = AutoProcessor.from_pretrained(PRETRAINED_MODEL_NAME_OR_PATH(F\"Salesforce/{args.model_name}\"))\n        self.blip2_processor.image_processor.size = {'height': self.args.image_size, 'width': self.args.image_size}\n\n    def __call__(self, image, text=None, return_tensors=\"pt\", truncation=True, max_length=None, padding='max_length'):\n        data = self.blip2_processor(\n            image,\n            text=text,\n            return_tensors=return_tensors,\n            truncation=truncation,\n            max_length=max_length,\n            padding=padding\n        )\n        for key in data.keys():\n            data[key] = data[key].squeeze(0)\n        return data\n\n\nclass CLIPValidTransformer:\n    def __init__(self, args):\n        self.args = args\n        self.clip_processor = CLIPProcessor.from_pretrained(PRETRAINED_MODEL_NAME_OR_PATH(args.model_name))\n        self.clip_processor.image_processor.size['shortest_edge'] = args.image_size\n        self.clip_processor.image_processor.do_center_crop = False\n\n    def __call__(self, image):\n        data = self.clip_processor(images=image)['pixel_values'][0]\n        return {\n            'image': data\n        }\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:07.450592Z","iopub.execute_input":"2023-06-15T20:59:07.451020Z","iopub.status.idle":"2023-06-15T20:59:07.466592Z","shell.execute_reply.started":"2023-06-15T20:59:07.450985Z","shell.execute_reply":"2023-06-15T20:59:07.465478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Generating predictions, saving embeddings, and evaluating the performance of models\nclass Predictor:\n    def __init__(self, args):\n        self.args = args\n        self.device = 'cuda'\n        self.submit_dir = f'{RESULT_DIR}/submissions/{args.out_dir}/fold{args.fold}'\n        os.makedirs(self.submit_dir, exist_ok=True)\n\n    def get_dataloader(self):\n        args = self.args\n        # set transform\n        if args.is_clip_model:\n            transform = CLIPValidTransformer(args)\n        elif args.model_name.startswith('blip'):\n            transform = Blip2ValidTransformer(args)\n        # create dataset, dataloader\n        test_dataset = DiffusionDataset(args, dataset=args.dataset, transform=transform)\n        test_sampler = SequentialSampler(test_dataset)\n        test_dataloader = DataLoader(\n            test_dataset,\n            sampler=test_sampler,\n            drop_last=False,\n            pin_memory=True,\n            batch_size=args.test_batch_size,\n            num_workers=args.num_workers,\n        )\n        return test_dataloader\n\n    def load_model(self, epoch):\n        args = self.args\n        model = eval(args.model)(args)\n        model_path = f'{RESULT_DIR}/models/{args.out_dir}/fold{args.fold}/{epoch}.pth'\n        load_pretrained(model, model_path, strict=False, can_print=True)\n        model.cuda()\n        model.eval()\n        return model\n\n    def do_predict(self, dataloader, model):\n        model.eval()\n        tbar = tqdm(dataloader, file=sys.stdout)\n        N = len(dataloader.dataset)\n        print(f'dataset len: {N}')\n        preds = []\n        with torch.no_grad():\n            with autocast():\n                for idx, data in enumerate(tbar):\n                    logits = get_outputs(data, model)\n                    preds.append(logits.detach().cpu().numpy())\n        prompt_embeddings = np.vstack(preds)\n        imgIds = dataloader.dataset.df[IMG_ID].values\n        return prompt_embeddings, imgIds\n\n    def save_embeddings(self, embeddings, ids):\n        file = F\"{self.submit_dir}/embeddings_{self.args.dataset}_epoch{self.args.epoch}.npz\"\n        np.savez_compressed(file, embeddings=embeddings, ids=ids)\n\n        \n# argument parser object used to parse command-line arguments and access their values\ndef make_parser():\n    parser = argparse.ArgumentParser(description='generate_submission')\n    parser.add_argument('-f', type=str, default=None)\n    parser.add_argument('--HistoryManager.hist_file', type=str, default='')\n    # model config\n    parser.add_argument('--module', '-m', type=str, default='basic_net', help='module')\n    parser.add_argument('--model', type=str, default=None, help='model')\n    parser.add_argument('--model_name', type=str, default='vit_base_patch16_224', help='model_name')\n    parser.add_argument('--embedding_dim', default=384, type=int)\n    parser.add_argument(\"--out_dir\", type=str, default=None, required=False)\n    parser.add_argument(\"--epoch\", type=str, default='best_ema', required=False)  \n    # dataset config\n    parser.add_argument('--dataset', default='valid', type=str)  \n    parser.add_argument('--split_type', default='random', type=str)\n    parser.add_argument('--folds_num', default=1, type=int)\n    parser.add_argument('--fold', default=0, type=int)\n    parser.add_argument('--image_size', default=224, type=int)\n    # predict param\n    parser.add_argument('--test_batch_size', default=16, type=int)\n    parser.add_argument(\"--gpus\", type=str, default=\"0\", required=False) # CUDA_VISIBLE_DEVICES \n    parser.add_argument('--num_workers', default=2, type=int) # number of subprocesses to use for data loading\n    parser.add_argument('--save_embeddings', default=1, type=int, help='') \n    parser.add_argument('--update', default=1, type=int, help='') \n    return parser\n\n\n# passes keys from inpput data to the model for forward propagation\ndef get_outputs(data, model):\n    keys = ['image', 'pixel_values', 'target', 'input_ids', 'attention_mask']\n    for key in keys:\n        if key in data:\n            data[key] = Variable(data[key].cuda())\n    return model(data)\n\n\n# pandas DataFrame containing the image ID, embedding ID, and corresponding embedding value for each prediction\ndef generate_submission(embeddings, ids):\n    imgId_eId = [\n        '_'.join(map(str, i)) for i in zip(\n            np.repeat(ids, EMBEDDING_LENGTH),\n            np.tile(range(EMBEDDING_LENGTH), len(ids))\n        )\n    ]\n    submission = pd.DataFrame({\n        'imgId_eId': imgId_eId,\n        'val': embeddings.flatten(),\n    })\n    return submission\n\n\n# cosine similarity between the true embeddings and predicted embeddings\ndef cosine_similarity(y_trues, y_preds):\n    scores = [\n        1 - spatial.distance.cosine(y_true, y_pred)\n        for y_true, y_pred in zip(y_trues, y_preds)\n    ]\n    return np.mean(scores)\n   \n    \ndef _do_predict(params):\n    parser = make_parser()\n    args = parser.parse_args(namespace=Namespace(**params))\n    args.can_print = True\n    args.is_clip_model = args.model_name.startswith('openai/clip') or args.model_name.startswith('laion/CLIP')\n    os.environ['CUDA_VISIBLE_DEVICES'] = args.gpus\n\n    predictor = Predictor(args)\n    pred_path_suffix = f'{args.dataset}_epoch{args.epoch}'\n    pred_path = f'{predictor.submit_dir}/{pred_path_suffix}_pred.csv'\n\n    dataloader = predictor.get_dataloader()\n    if not ope(pred_path) or args.update == 1:\n        model = predictor.load_model(args.epoch)\n        if len(args.gpus.split(',')) > 1:\n            model = DataParallel(model)\n        embeddings, ids = predictor.do_predict(dataloader, model)\n        pred_df = generate_submission(embeddings, ids)\n        if args.save_embeddings:\n            predictor.save_embeddings(embeddings, ids)\n        pred_df.to_csv(pred_path, index=False)\n        print(f'save pred to: {pred_path}')\n    else:\n        pred_df = pd.read_csv(pred_path)\n        print(f'load pred from: {pred_path}')\n    print(pred_df.head())\n\n    check_df = pred_df['imgId_eId'].str.split(\"_\", expand=True)\n    check_df.columns = [IMG_ID, 'eId']\n    indexs = check_df['eId'].values.reshape(-1, EMBEDDING_LENGTH).astype(np.int16)\n    assert np.all(indexs == range(EMBEDDING_LENGTH))\n\n    if not IS_KAGGLE_SUBMIT:\n        pred_embeddings = pred_df['val'].values.reshape(-1, EMBEDDING_LENGTH)\n        pred_ids = check_df[IMG_ID].drop_duplicates().values\n        truth_embeddings = dataloader.dataset.targets\n        truth_ids = dataloader.dataset.df[IMG_ID].values\n        assert np.all(truth_ids == pred_ids)\n        score = cosine_similarity(truth_embeddings, pred_embeddings)\n        print(f'score: {score:.5f}')\n\n# wrapper function that creates a new process \ndef do_predict(params):\n    p = Process(target=_do_predict, args=(params,))\n    p.start()\n    p.join()","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:08.732644Z","iopub.execute_input":"2023-06-15T20:59:08.733131Z","iopub.status.idle":"2023-06-15T20:59:08.781104Z","shell.execute_reply.started":"2023-06-15T20:59:08.733086Z","shell.execute_reply":"2023-06-15T20:59:08.779883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Linear Layer and Linear transformations using the LoRA (Low-Rank Approximation):\n- used to incorporate the LoRA technique into linear transformations within a neural network architecture","metadata":{}},{"cell_type":"code","source":"class LoRALinearLayer(nn.Module):\n    def __init__(self, in_features, out_features, rank=4):\n        super().__init__()\n\n        if rank > min(in_features, out_features):\n            raise ValueError(f\"LoRA rank {rank} must be less or equal than {min(in_features, out_features)}\")\n\n        self.down = nn.Linear(in_features, rank, bias=False)\n        self.up = nn.Linear(rank, out_features, bias=False)\n\n        nn.init.normal_(self.down.weight, std=1 / rank)\n        nn.init.zeros_(self.up.weight)\n    \n    # applying the linear layers sequentially\n    def forward(self, hidden_states):\n        orig_dtype = hidden_states.dtype\n        dtype = self.down.weight.dtype\n\n        down_hidden_states = self.down(hidden_states.to(dtype))\n        up_hidden_states = self.up(down_hidden_states)\n\n        return up_hidden_states.to(orig_dtype)\n\n\n# combines an attention mechanism and a LoRA linear transformation\nclass LoRALinear(nn.Module):\n    def __init__(self, attn, rank=4, lora_scale=1):\n        super().__init__()\n        self.attn = attn\n        self.rank = rank\n        self.lora_scale = lora_scale\n\n        in_features = attn.in_features\n        out_features = attn.out_features\n        _rank = min(in_features // rank, out_features // rank)\n        self.fc_lora = LoRALinearLayer(in_features, out_features, _rank)\n\n    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:\n        hidden_states = self.attn(hidden_states) + self.lora_scale * self.fc_lora(hidden_states)\n        return hidden_states","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:10.190114Z","iopub.execute_input":"2023-06-15T20:59:10.190536Z","iopub.status.idle":"2023-06-15T20:59:10.207884Z","shell.execute_reply.started":"2023-06-15T20:59:10.190500Z","shell.execute_reply":"2023-06-15T20:59:10.207016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## BLIP-2:\n- processes self-attention using the LoRA, designed as a modification of the attention mechanism used in the CLIP model","metadata":{}},{"cell_type":"code","source":"class Blip2LoRAAttnProcessor(nn.Module):\n    def __init__(self, attn, rank=4, lora_scale=1):\n        super().__init__()\n\n        self.attn = attn\n        config = attn.config\n        embed_dim = config.hidden_size\n\n        # small tweak here compared to CLIP, no bias here\n        self.qkv_lora = LoRALinearLayer(embed_dim, 3 * embed_dim, embed_dim // rank)\n        self.projection_lora = LoRALinearLayer(embed_dim, embed_dim, embed_dim // rank)\n        self.rank = rank\n        self.lora_scale = lora_scale\n\n    # LoRA-based self-attention computation    \n    def forward(\n            self,\n            hidden_states: torch.Tensor,\n            head_mask: Optional[torch.Tensor] = None,\n            output_attentions: Optional[bool] = False,\n    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:\n        \"\"\"Input shape: Batch x Time x Channel\"\"\"\n\n        bsz, tgt_len, embed_dim = hidden_states.size()\n\n        mixed_qkv = self.attn.qkv(hidden_states) + self.lora_scale * self.qkv_lora(hidden_states)\n\n        mixed_qkv = mixed_qkv.reshape(bsz, tgt_len, 3, self.attn.num_heads, embed_dim // self.attn.num_heads).permute(\n            2, 0, 3, 1, 4\n        )\n        query_states, key_states, value_states = (\n            mixed_qkv[0],\n            mixed_qkv[1],\n            mixed_qkv[2],\n        )\n\n        # Take the dot product between \"query\" and \"key\" to get the raw attention scores.\n        attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2))\n\n        attention_scores = attention_scores * self.attn.scale\n\n        # Normalize the attention scores to probabilities.\n        attention_probs = nn.functional.softmax(attention_scores, dim=-1)\n\n        # This is actually dropping out entire tokens to attend to, which might\n        # seem a bit unusual, but is taken from the original Transformer paper.\n        attention_probs = self.attn.dropout(attention_probs)\n\n        # Mask heads if we want to\n        if head_mask is not None:\n            attention_probs = attention_probs * head_mask\n\n        context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3)\n\n        new_context_layer_shape = context_layer.size()[:-2] + (self.attn.embed_dim,)\n        context_layer = context_layer.reshape(new_context_layer_shape)\n\n        output = self.attn.projection(context_layer) + self.lora_scale * self.projection_lora(context_layer)\n\n        outputs = (output, attention_probs) if output_attentions else (output, None)\n\n        return outputs","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:11.774188Z","iopub.execute_input":"2023-06-15T20:59:11.774612Z","iopub.status.idle":"2023-06-15T20:59:11.796212Z","shell.execute_reply.started":"2023-06-15T20:59:11.774580Z","shell.execute_reply":"2023-06-15T20:59:11.795378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# replaces the self-attention and linear layers in those layers with LoRA-based versions\ndef lora_blip2_model(model, lora_scale, lora_rate):\n    layer_num = len(model.vision_model.encoder.layers)\n    lora_num = int(layer_num * lora_rate)\n    for _i in range(layer_num - lora_num, layer_num):\n        attn = model.vision_model.encoder.layers[_i].self_attn\n        new_attn = Blip2LoRAAttnProcessor(attn, rank=4, lora_scale=lora_scale)\n        model.vision_model.encoder.layers[_i].self_attn = new_attn\n        model.vision_model.encoder.layers[_i].mlp.fc1 = LoRALinear(\n            model.vision_model.encoder.layers[_i].mlp.fc1, rank=4, lora_scale=lora_scale)\n        model.vision_model.encoder.layers[_i].mlp.fc2 = LoRALinear(\n            model.vision_model.encoder.layers[_i].mlp.fc2, rank=4, lora_scale=lora_scale)","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:13.079232Z","iopub.execute_input":"2023-06-15T20:59:13.079691Z","iopub.status.idle":"2023-06-15T20:59:13.095880Z","shell.execute_reply.started":"2023-06-15T20:59:13.079654Z","shell.execute_reply":"2023-06-15T20:59:13.092567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n\nclass BasicNet(torch.nn.Module):\n    def __init__(self, args, lora_scale=1, lora_rate=0.4):\n        super().__init__()\n        self.args = args\n        self.model = Blip2ForConditionalGeneration.from_pretrained(\n            TRANSFORMERS.get(F\"Salesforce/{args.model_name}\"),\n            torch_dtype=torch.float16,\n            low_cpu_mem_usage=True\n        )\n        del self.model.language_model\n        del self.model.language_projection\n        self.model.vision_model.requires_grad_(False)\n        self.model.feat_proj = torch.nn.Linear(self.model.qformer.config.hidden_size * 32, self.args.embedding_dim)\n        lora_blip2_model(self.model, lora_scale, lora_rate)\n\n    def forward(self, samples):\n        pixel_values = samples['pixel_values']\n        pixel_values = pixel_values.half()\n        image_embeds = self.model.vision_model(pixel_values, return_dict=True).last_hidden_state\n        image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long, device=image_embeds.device)\n\n        query_tokens = self.model.query_tokens.expand(image_embeds.shape[0], -1, -1)\n        query_outputs = self.model.qformer(\n            query_embeds=query_tokens,\n            encoder_hidden_states=image_embeds,\n            encoder_attention_mask=image_attention_mask,\n            return_dict=True,\n        )\n        query_output = query_outputs.last_hidden_state\n\n        feat_pred = self.model.feat_proj(query_output.view(len(query_output), -1))\n        return feat_pred\n\n\ndef blip2_model_lora04(args):\n    model = BasicNet(args, lora_scale=1, lora_rate=0.4)\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:15.032277Z","iopub.execute_input":"2023-06-15T20:59:15.032642Z","iopub.status.idle":"2023-06-15T20:59:15.044219Z","shell.execute_reply.started":"2023-06-15T20:59:15.032615Z","shell.execute_reply":"2023-06-15T20:59:15.043141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"do_predict({\n    'module': 'blip2_net', \n    'model': 'blip2_model_lora04', \n    'model_name': 'blip2-opt-2.7b',  \n    'out_dir': 'train2m_pretrain6-6m_lora04_blip2_opt_2_7b_224x224_1folds', \n    'epoch': 'en3_ema', \n    'fold': 0, \n    'test_batch_size': 256, \n    'dataset': 'test', \n    'gpus': '0',\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-15T20:59:17.643239Z","iopub.execute_input":"2023-06-15T20:59:17.643582Z","iopub.status.idle":"2023-06-15T21:00:11.939253Z","shell.execute_reply.started":"2023-06-15T20:59:17.643556Z","shell.execute_reply":"2023-06-15T21:00:11.937713Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ConvNeXt-XXLarge","metadata":{}},{"cell_type":"code","source":"def lora_convnext_model(model, args, lora_scale, lora_rate, nfz_rate=0.4, rank=4):\n    model.trunk.stem.requires_grad_(False)\n    model.trunk.stages[0].requires_grad_(False)\n    model.trunk.stages[1].requires_grad_(False)\n    model.trunk.stages[2].requires_grad_(False)\n    layer_num = len(model.trunk.stages[2].blocks)\n    lora_num = int(layer_num * lora_rate)\n    nfz_num = int(layer_num * nfz_rate)\n\n    for _idx in range(layer_num - lora_num - nfz_num, layer_num):\n        if (layer_num-_idx) <= nfz_num:\n            model.trunk.stages[2].blocks[_idx].requires_grad_(True)\n            continue\n        model.trunk.stages[2].blocks[_idx].mlp.fc1 = LoRALinear(model.trunk.stages[2].blocks[_idx].mlp.fc1, rank=rank, lora_scale=lora_scale)\n        model.trunk.stages[2].blocks[_idx].mlp.fc2 = LoRALinear(model.trunk.stages[2].blocks[_idx].mlp.fc2, rank=rank, lora_scale=lora_scale)\n    if args.can_print: print(f'lora clip_model, scale:{lora_scale} rate:{lora_rate} num:{lora_num}')\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T21:00:29.532603Z","iopub.execute_input":"2023-06-15T21:00:29.532983Z","iopub.status.idle":"2023-06-15T21:00:29.544736Z","shell.execute_reply.started":"2023-06-15T21:00:29.532949Z","shell.execute_reply":"2023-06-15T21:00:29.543622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BaseClipNet(torch.nn.Module):\n    def __init__(self, args, lora_rate=0.0, nfz_rate=0.0, lora_scale=1.0, rank=4, model_type='ViT-L-14', ebd_dim=768):\n        super().__init__()\n        self.args = args\n        model, _, preprocess = open_clip.create_model_and_transforms(model_type,\n                                                                     pretrained=f'{PRETRAINED_MODEL_NAME_OR_PATH(args.model_name)}/open_clip_pytorch_model.bin')\n        self.vision_model = model.visual\n        fc_dim = 16 * 1024\n        self.head = nn.Sequential(\n            nn.Linear(ebd_dim, fc_dim),\n            nn.BatchNorm1d(fc_dim),\n            nn.ReLU(),\n            nn.Linear(fc_dim, args.embedding_dim),\n        )\n        lora_convnext_model(self.vision_model, args, lora_scale, lora_rate, nfz_rate, rank)\n\n    def forward(self, data):\n        out = self.vision_model(data['image'])\n        logits = self.head(out)\n        return logits\n\n\ndef clipnet_convnext_xxlarge_256_lora04_nfz02_rank8(args):\n    model = BaseClipNet(args, lora_rate=0.4, nfz_rate=0.2, model_type='convnext_xxlarge', rank=8, ebd_dim=1024)\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T21:00:32.905249Z","iopub.execute_input":"2023-06-15T21:00:32.905611Z","iopub.status.idle":"2023-06-15T21:00:32.915909Z","shell.execute_reply.started":"2023-06-15T21:00:32.905581Z","shell.execute_reply":"2023-06-15T21:00:32.914912Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"do_predict({\n    'out_dir': 'train2m_pretrain6-6m_lora04_laion_CLIP_convnext_xxlarge_laion2B_s34B_b82K_augreg_soup_256x256_1folds',\n    'module': 'clip_convnext_net',\n    'model': 'clipnet_convnext_xxlarge_256_lora04_nfz02_rank8',\n    'model_name': 'laion/CLIP-convnext_xxlarge-laion2B-s34B-b82K-augreg-soup',\n    'image_size': 256,\n    'gpus': '0',\n    'epoch': 'en3_ema',\n    'dataset': 'test',\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-15T21:00:34.725824Z","iopub.execute_input":"2023-06-15T21:00:34.726227Z","iopub.status.idle":"2023-06-15T21:01:41.752333Z","shell.execute_reply.started":"2023-06-15T21:00:34.726195Z","shell.execute_reply":"2023-06-15T21:01:41.750817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ViT-Large","metadata":{}},{"cell_type":"code","source":"class LoRACLIPAttention(nn.Module):\n    def __init__(self, attn, rank=4, lora_scale=1):\n        super().__init__()\n        self.attn = attn\n        self.rank = rank\n        self.lora_scale = lora_scale\n\n        config = attn.config\n        embed_dim = config.hidden_size\n        self.k_proj_lora = LoRALinearLayer(embed_dim, embed_dim, embed_dim // rank)\n        self.v_proj_lora = LoRALinearLayer(embed_dim, embed_dim, embed_dim // rank)\n        self.q_proj_lora = LoRALinearLayer(embed_dim, embed_dim, embed_dim // rank)\n        self.out_proj_lora = LoRALinearLayer(embed_dim, embed_dim, embed_dim // rank)\n\n    def forward(\n            self,\n            hidden_states: torch.Tensor,\n            attention_mask: Optional[torch.Tensor] = None,\n            causal_attention_mask: Optional[torch.Tensor] = None,\n            output_attentions: Optional[bool] = False,\n    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:\n        \"\"\"Input shape: Batch x Time x Channel\"\"\"\n\n        bsz, tgt_len, embed_dim = hidden_states.size()\n\n        # get query proj\n        query_states = (self.attn.q_proj(hidden_states) + self.lora_scale * self.q_proj_lora(\n            hidden_states)) * self.attn.scale\n        key_states = self.attn._shape(\n            self.attn.k_proj(hidden_states) + self.lora_scale * self.k_proj_lora(hidden_states), -1, bsz)\n        value_states = self.attn._shape(\n            self.attn.v_proj(hidden_states) + self.lora_scale * self.v_proj_lora(hidden_states), -1, bsz)\n\n        proj_shape = (bsz * self.attn.num_heads, -1, self.attn.head_dim)\n        query_states = self.attn._shape(query_states, tgt_len, bsz).view(*proj_shape)\n        key_states = key_states.view(*proj_shape)\n        value_states = value_states.view(*proj_shape)\n\n        src_len = key_states.size(1)\n        attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))\n\n        if attn_weights.size() != (bsz * self.attn.num_heads, tgt_len, src_len):\n            raise ValueError(\n                f\"Attention weights should be of size {(bsz * self.attn.num_heads, tgt_len, src_len)}, but is\"\n                f\" {attn_weights.size()}\"\n            )\n\n        # apply the causal_attention_mask first\n        if causal_attention_mask is not None:\n            if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len):\n                raise ValueError(\n                    f\"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is\"\n                    f\" {causal_attention_mask.size()}\"\n                )\n            attn_weights = attn_weights.view(bsz, self.attn.num_heads, tgt_len, src_len) + causal_attention_mask\n            attn_weights = attn_weights.view(bsz * self.attn.num_heads, tgt_len, src_len)\n\n        if attention_mask is not None:\n            if attention_mask.size() != (bsz, 1, tgt_len, src_len):\n                raise ValueError(\n                    f\"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}\"\n                )\n            attn_weights = attn_weights.view(bsz, self.attn.num_heads, tgt_len, src_len) + attention_mask\n            attn_weights = attn_weights.view(bsz * self.attn.num_heads, tgt_len, src_len)\n\n        attn_weights = nn.functional.softmax(attn_weights, dim=-1)\n\n        if output_attentions:\n            # this operation is a bit akward, but it's required to\n            # make sure that attn_weights keeps its gradient.\n            # In order to do so, attn_weights have to reshaped\n            # twice and have to be reused in the following\n            attn_weights_reshaped = attn_weights.view(bsz, self.attn.num_heads, tgt_len, src_len)\n            attn_weights = attn_weights_reshaped.view(bsz * self.attn.num_heads, tgt_len, src_len)\n        else:\n            attn_weights_reshaped = None\n\n        attn_probs = nn.functional.dropout(attn_weights, p=self.attn.dropout, training=self.training)\n\n        attn_output = torch.bmm(attn_probs, value_states)\n\n        if attn_output.size() != (bsz * self.attn.num_heads, tgt_len, self.attn.head_dim):\n            raise ValueError(\n                f\"`attn_output` should be of size {(bsz, self.attn.num_heads, tgt_len, self.attn.head_dim)}, but is\"\n                f\" {attn_output.size()}\"\n            )\n\n        attn_output = attn_output.view(bsz, self.attn.num_heads, tgt_len, self.attn.head_dim)\n        attn_output = attn_output.transpose(1, 2)\n        attn_output = attn_output.reshape(bsz, tgt_len, embed_dim)\n\n        attn_output = self.attn.out_proj(attn_output) + self.lora_scale * self.out_proj_lora(attn_output)\n\n        return attn_output, attn_weights_reshaped\n    \ndef lora_clip_model(model, args, lora_scale, lora_rate, nfz_rate=0.4, rank=4):\n    model.embeddings.requires_grad_(False)\n    model.pre_layrnorm.requires_grad_(False)\n    model.encoder.layers.requires_grad_(False)\n    layer_num = len(model.encoder.layers)\n    lora_num = int(layer_num * lora_rate)\n    nfz_num = int(layer_num * nfz_rate)\n    for _i in range(layer_num - lora_num - nfz_num, layer_num):\n        if (layer_num - _i) <= nfz_num:\n            model.encoder.layers[_i].requires_grad_(True)\n            continue\n        attn = model.encoder.layers[_i].self_attn\n        new_attn = LoRACLIPAttention(attn, rank=rank, lora_scale=lora_scale)\n        model.encoder.layers[_i].self_attn = new_attn\n    if args.can_print: print(f'lora clip_model, scale:{lora_scale} rate:{lora_rate} num:{lora_num}')\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T21:01:41.754860Z","iopub.execute_input":"2023-06-15T21:01:41.755197Z","iopub.status.idle":"2023-06-15T21:01:41.785458Z","shell.execute_reply.started":"2023-06-15T21:01:41.755167Z","shell.execute_reply":"2023-06-15T21:01:41.784576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class BaseClipNet(torch.nn.Module):\n    def __init__(self, args, lora_rate=0.0, nfz_rate=0.0, lora_scale=1.0, rank=4):\n        super().__init__()\n        self.args = args\n        config = CLIPConfig.from_pretrained(PRETRAINED_MODEL_NAME_OR_PATH(args.model_name))\n        config.vision_config.image_size = args.image_size\n        clip = CLIPModel(config)\n\n        pretrained_file = f'{PRETRAINED_MODEL_NAME_OR_PATH(args.model_name)}/pytorch_model.pt'\n        load_pretrained(clip, pretrained_file, strict=False, can_print=True)\n\n        self.vision_model = clip.vision_model\n        fc_dim = 16 * 1024\n        ebd_dim = self.vision_model.embeddings.position_embedding.embedding_dim\n        self.head = nn.Sequential(\n            nn.Linear(ebd_dim, fc_dim),\n            nn.BatchNorm1d(fc_dim),\n            nn.ReLU(),\n            nn.Linear(fc_dim, args.embedding_dim),\n        )\n        lora_clip_model(self.vision_model, args, lora_scale, lora_rate, nfz_rate, rank)\n\n    def forward(self, data):\n        out = self.vision_model(data['image'])\n        logits = self.head(out['pooler_output'])\n        return logits\n\ndef clipnet_lora06(args):\n    model = BaseClipNet(args, lora_rate=0.6, nfz_rate=0.4)\n    return model\n","metadata":{"execution":{"iopub.status.busy":"2023-06-15T21:01:41.786730Z","iopub.execute_input":"2023-06-15T21:01:41.787098Z","iopub.status.idle":"2023-06-15T21:01:41.797837Z","shell.execute_reply.started":"2023-06-15T21:01:41.787063Z","shell.execute_reply":"2023-06-15T21:01:41.796747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"do_predict({\n    'out_dir': 'train2m_pretrain6-6m_lora06_hd_openai_clip_vit_large_patch14_336_336x336_1folds',\n    'model': 'clipnet_lora06',\n    'module': 'clip_net',\n    'model_name': 'openai/clip-vit-large-patch14-336',\n    'image_size': 336,\n    'epoch': 'en3_ema',\n    'dataset': 'test',\n    'gpus': '0',\n})","metadata":{"execution":{"iopub.status.busy":"2023-06-15T21:01:41.799969Z","iopub.execute_input":"2023-06-15T21:01:41.800380Z","iopub.status.idle":"2023-06-15T21:02:11.866378Z","shell.execute_reply.started":"2023-06-15T21:01:41.800348Z","shell.execute_reply":"2023-06-15T21:02:11.864840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"en_config = [\n    {\n        'out_dir': 'train2m_pretrain6-6m_lora06_hd_openai_clip_vit_large_patch14_336_336x336_1folds', \n        'weight': 0.25\n    }, \n    {\n        'out_dir': 'train2m_pretrain6-6m_lora04_blip2_opt_2_7b_224x224_1folds', \n        'weight': 0.35\n    }, \n    {\n        'out_dir': 'train2m_pretrain6-6m_lora04_laion_CLIP_convnext_xxlarge_laion2B_s34B_b82K_augreg_soup_256x256_1folds', \n        'weight': 0.4\n    }\n]","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:16:51.982264Z","iopub.execute_input":"2023-06-02T21:16:51.982657Z","iopub.status.idle":"2023-06-02T21:16:51.989189Z","shell.execute_reply.started":"2023-06-02T21:16:51.982621Z","shell.execute_reply":"2023-06-02T21:16:51.988207Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = 'test'\nmeta_df = pd.read_csv(f'{DATA_DIR}/raw/sample_submission.csv')\nvalues_sum = 0.\nweight_sum = 0.\nfor idx, cfg in enumerate(en_config):\n    out_dir = cfg['out_dir']\n    weight = cfg['weight']\n    fold = 0\n    epoch = 'en3_ema'\n    pred_path_suffix = f'{dataset}_epoch{epoch}'\n    pred_df = pd.read_csv(f\"{RESULT_DIR}/submissions/{out_dir}/fold{fold}/{pred_path_suffix}_pred.csv\")\n    values = meta_df[[ID]].merge(pred_df, on=ID, how='left')[TARGET].values\n    values = values.reshape(-1, EMBEDDING_LENGTH)\n    mean = values.mean(axis=1, keepdims=True)\n    std = values.std(axis=1, keepdims=True)\n    values = (values - mean) / (std + 1e-8)\n    values = values.reshape(-1)\n    values_sum += values * weight\n    weight_sum += weight\nvalues = values_sum / weight_sum\npred_df = pd.DataFrame({\n    ID: meta_df[ID].values,\n    TARGET: values,\n})\nif not IS_KAGGLE_SUBMIT:\n    pred_embeddings = pred_df[TARGET].values.reshape(-1, EMBEDDING_LENGTH)\n    truth_embeddings = meta_df[TARGET].values.reshape(-1, EMBEDDING_LENGTH)\n    score = cosine_similarity(truth_embeddings, pred_embeddings)\n    print(f'score: {score:.5f}')\npred_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:16:53.660749Z","iopub.execute_input":"2023-06-02T21:16:53.661160Z","iopub.status.idle":"2023-06-02T21:16:53.737234Z","shell.execute_reply.started":"2023-06-02T21:16:53.661129Z","shell.execute_reply":"2023-06-02T21:16:53.736150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dir_list = os.listdir('/kaggle/working')\nfor file in dir_list:\n    if os.path.isdir(file):\n        shutil.rmtree(file)\n    elif file != 'submission.csv':\n        os.remove(file)","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:16:56.339688Z","iopub.execute_input":"2023-06-02T21:16:56.340332Z","iopub.status.idle":"2023-06-02T21:16:56.751323Z","shell.execute_reply.started":"2023-06-02T21:16:56.340297Z","shell.execute_reply":"2023-06-02T21:16:56.750322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submit_df = pd.read_csv('./submission.csv')\nprint(f'submission: {submit_df.shape}')\nprint(submit_df.head())","metadata":{"execution":{"iopub.status.busy":"2023-06-02T21:17:08.645393Z","iopub.execute_input":"2023-06-02T21:17:08.645776Z","iopub.status.idle":"2023-06-02T21:17:08.663253Z","shell.execute_reply.started":"2023-06-02T21:17:08.645744Z","shell.execute_reply":"2023-06-02T21:17:08.661997Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}