{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":43873,"databundleVersionId":5585780,"sourceType":"competition"},{"sourceId":5857376,"sourceType":"datasetVersion","datasetId":3259992},{"sourceId":10075894,"sourceType":"datasetVersion","datasetId":6209817},{"sourceId":10442039,"sourceType":"datasetVersion","datasetId":6463135}],"dockerImageVersionId":30823,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from transformers import AutoProcessor, Pix2StructForConditionalGeneration, Seq2SeqTrainer, Seq2SeqTrainingArguments\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom datasets import Dataset as HFDataset\nfrom PIL import Image\nimport json\nimport os\nfrom tqdm import tqdm\nimport torch\nimport os\n\n# Set CUDA environment variable to avoid memory fragmentation\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\nclass ChartDataset(Dataset):\n    def __init__(self, images_dir, annotations_dir, processor):\n        self.images_dir = images_dir\n        self.annotations_dir = annotations_dir\n        self.processor = processor\n        self.image_files = [f for f in os.listdir(images_dir) if f.endswith('.jpg')]\n\n    def __len__(self):\n        return min(1000,len(self.image_files))\n\n    def __getitem__(self, idx):\n        image_file = self.image_files[idx]\n        annotation_file = image_file.replace('.jpg', '.json')\n\n        # Load image\n        image_path = os.path.join(self.images_dir, image_file)\n        image = Image.open(image_path).convert(\"RGB\")\n\n        # Load annotation\n        annotation_path = os.path.join(self.annotations_dir, annotation_file)\n        with open(annotation_path, 'r') as f:\n            annotation = json.load(f)\n\n        # Extract data-series from annotation\n        data_series = annotation.get(\"data-series\", [])\n        for entry in data_series:\n            entry[\"y\"] = round(float(entry[\"y\"]), 8)  # Ensure y has 8 decimal places\n\n            \n        # Preprocess the image with the text \"Generate data series\"\n        inputs = self.processor(images=image, text=\"Generate data series:\", return_tensors=\"pt\")\n\n        return {\n            \"flattened_patches\": inputs[\"flattened_patches\"].squeeze(),\n            \"labels\": torch.tensor(self.processor.tokenizer(json.dumps(data_series), padding=\"max_length\", max_length=512, truncation=True).input_ids)  # Reduced max_length\n        }\n\n# Custom data collator\ndef custom_data_collator(features):\n    flattened_patches = torch.stack([feature[\"flattened_patches\"] for feature in features])\n    labels = torch.nn.utils.rnn.pad_sequence([feature[\"labels\"] for feature in features], batch_first=True, padding_value=-100)\n    return {\"flattened_patches\": flattened_patches, \"labels\": labels}\n\n# Paths to your data directories\nimages_dir = \"/kaggle/input/sortrd-chart-data/sorted_data/sorted_data/line/images\"\nannotations_dir = \"/kaggle/input/sortrd-chart-data/sorted_data/sorted_data/line/annotations\"\n\n# Load processor and model\nmodel = Pix2StructForConditionalGeneration.from_pretrained(\"google/deplot\")\nprocessor = AutoProcessor.from_pretrained(\"google/deplot\")\n\n# Prepare dataset\ndataset = ChartDataset(images_dir, annotations_dir, processor)\n\n# Split dataset into training and evaluation\ntrain_size = int(0.8 * len(dataset))\neval_size = len(dataset) - train_size\ntrain_dataset, eval_dataset = random_split(dataset, [train_size, eval_size])\n\n# Use DataLoader for lazy loading and batching\ndata_loader = DataLoader(dataset, batch_size=4, shuffle=True, num_workers=4)\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-14T10:15:19.425626Z","iopub.execute_input":"2025-02-14T10:15:19.425977Z","iopub.status.idle":"2025-02-14T10:15:27.753180Z","shell.execute_reply.started":"2025-02-14T10:15:19.425920Z","shell.execute_reply":"2025-02-14T10:15:27.752361Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Training arguments\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"/kaggle/working/deplot_finetuned\",\n    eval_strategy=\"epoch\",\n    learning_rate=5e-5,\n    per_device_train_batch_size=1,  # Further reduced batch size\n    per_device_eval_batch_size=1,  # Further reduced batch size\n    num_train_epochs=5,\n    save_strategy=\"epoch\",\n    save_total_limit=2,\n    predict_with_generate=True,\n    fp16=True,\n    logging_dir=\"/kaggle/working/logs\",\n    logging_steps=200,\n    push_to_hub=False,\n    dataloader_num_workers=2,  # Reduced workers\n    ddp_find_unused_parameters=False,\n    report_to=[\"none\"],\n    gradient_accumulation_steps=2,  # Reduced gradient accumulation\n    no_cuda=False,\n    load_best_model_at_end=True\n)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-14T10:16:07.412032Z","iopub.execute_input":"2025-02-14T10:16:07.412689Z","iopub.status.idle":"2025-02-14T10:16:07.544785Z","shell.execute_reply.started":"2025-02-14T10:16:07.412661Z","shell.execute_reply":"2025-02-14T10:16:07.543804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Initialize trainer\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,  # Training dataset\n    eval_dataset=eval_dataset,    # Evaluation dataset\n    tokenizer=processor.tokenizer,\n    data_collator=custom_data_collator\n)\n\n# Train model\ntrainer.train()\n\n# Save the model\nmodel.save_pretrained(\"/kaggle/working/deplot_finetuned\")\nprocessor.save_pretrained(\"/kaggle/working/deplot_finetuned\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-14T10:16:09.821175Z","iopub.execute_input":"2025-02-14T10:16:09.821544Z","iopub.status.idle":"2025-02-14T11:11:02.419804Z","shell.execute_reply.started":"2025-02-14T10:16:09.821515Z","shell.execute_reply":"2025-02-14T11:11:02.418724Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"model saved\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoProcessor, Pix2StructForConditionalGeneration\nfrom PIL import Image\n\n# Load the fine-tuned model and processor\n#model = Pix2StructForConditionalGeneration.from_pretrained(\"/kaggle/input/deplot-finetuned-line-version-2\")\n#processor = AutoProcessor.from_pretrained(\"/kaggle/input/deplot-finetuned-line-version-2\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-14T11:31:07.353239Z","iopub.execute_input":"2025-02-14T11:31:07.353619Z","iopub.status.idle":"2025-02-14T11:31:07.629925Z","shell.execute_reply.started":"2025-02-14T11:31:07.353590Z","shell.execute_reply":"2025-02-14T11:31:07.628991Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, random_split\nimport pandas as pd\nimport json\nimport os\nimport torch\nfrom transformers import AutoProcessor, Pix2StructForConditionalGeneration\nfrom PIL import Image\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Load the fine-tuned model and processor\nmodel = Pix2StructForConditionalGeneration.from_pretrained(\"/kaggle/working/deplot_finetuned\").to(device)\nprocessor = AutoProcessor.from_pretrained(\"/kaggle/working/deplot_finetuned\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-14T11:31:50.017277Z","iopub.execute_input":"2025-02-14T11:31:50.017576Z","iopub.status.idle":"2025-02-14T11:31:50.684172Z","shell.execute_reply.started":"2025-02-14T11:31:50.017544Z","shell.execute_reply":"2025-02-14T11:31:50.683439Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load an example image for prediction\nimage_path = \"/kaggle/input/benetech-making-graphs-accessible/test/images/000b92c3b098.jpg\"\nimage = Image.open(image_path)\ndisplay(image)\n\ninputs = processor(images=image, text=\"Generate data series:\", return_tensors=\"pt\")\n\n# Move inputs to GPU\ninputs = {key: value.to(device) for key, value in inputs.items()}\n\npredictions = model.generate(**inputs, max_new_tokens=2048)\nprint(processor.decode(predictions[0], skip_special_tokens=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-14T11:31:52.625705Z","iopub.execute_input":"2025-02-14T11:31:52.626016Z","iopub.status.idle":"2025-02-14T11:31:57.899694Z","shell.execute_reply.started":"2025-02-14T11:31:52.625991Z","shell.execute_reply":"2025-02-14T11:31:57.898579Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load an example image for prediction\nimage_path = \"/kaggle/input/benetech-making-graphs-accessible/test/images/007a18eb4e09.jpg\"\nimage = Image.open(image_path)\ndisplay(image)\n\ninputs = processor(images=image, text=\"Generate data series:\", return_tensors=\"pt\")\n\n# Move inputs to GPU\ninputs = {key: value.to(device) for key, value in inputs.items()}\n\npredictions = model.generate(**inputs, max_new_tokens=2048)\nprint(processor.decode(predictions[0], skip_special_tokens=True))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-02-14T11:32:25.062036Z","iopub.execute_input":"2025-02-14T11:32:25.062360Z","iopub.status.idle":"2025-02-14T11:32:33.776643Z","shell.execute_reply.started":"2025-02-14T11:32:25.062333Z","shell.execute_reply":"2025-02-14T11:32:33.775925Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from transformers import AutoProcessor, Pix2StructForConditionalGeneration, Seq2SeqTrainer, Seq2SeqTrainingArguments\nfrom torch.utils.data import Dataset, DataLoader, random_split\nfrom PIL import Image\nimport pandas as pd\nimport json\nimport os\nimport torch\n\n# Set CUDA environment variable to avoid memory fragmentation\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\nclass CSVChartDataset(Dataset):\n    def __init__(self, images_dir, annotations_csv, processor):\n        self.images_dir = images_dir\n        self.annotations = pd.read_csv(annotations_csv)\n        self.processor = processor\n\n    def __len__(self):\n        return min(1000, len(self.annotations))  # Use only 1000 images for fine-tuning\n\n    def __getitem__(self, idx):\n        idx = idx + 210000\n        row = self.annotations.iloc[idx]\n        image_path = os.path.join(self.images_dir, os.path.basename(row[\"file_name\"]))\n        data_series = row[\"text\"]\n\n        # Convert the data-series to JSON-like format\n        data_series_json = []\n        for pair in data_series.split(\"<0x0A>\"):\n            if \"|\" in pair:\n                x, y = pair.split(\"|\")\n                data_series_json.append({\"x\": x.strip(), \"y\": round(float(y.strip()), 4)})\n\n        # Load image\n        image = Image.open(image_path).convert(\"RGB\")\n\n        # Preprocess the image and data series\n        inputs = self.processor(images=image, text=\"Generate data series:\", return_tensors=\"pt\")\n\n        return {\n            \"flattened_patches\": inputs[\"flattened_patches\"].squeeze(),\n            \"labels\": torch.tensor(\n                self.processor.tokenizer(\n                    json.dumps(data_series_json, ensure_ascii=False),\n                    padding=\"max_length\",\n                    max_length=512,\n                    truncation=True\n                ).input_ids\n            )\n        }\n\n# Custom data collator\ndef custom_data_collator(features):\n    flattened_patches = torch.stack([feature[\"flattened_patches\"] for feature in features])\n    labels = torch.nn.utils.rnn.pad_sequence([feature[\"labels\"] for feature in features], batch_first=True, padding_value=-100)\n    return {\"flattened_patches\": flattened_patches, \"labels\": labels}\n\n# Paths to your data directories\nimages_dir = \"/kaggle/input/benetech-extra-generated-data/graphs_l\"\nannotations_csv = \"/kaggle/input/benetech-extra-generated-data/metadata.csv\"\n\n# Load processor and model\n#model = Pix2StructForConditionalGeneration.from_pretrained(\"/kaggle/input/deplot-finetuned-dot/deplot_finetuned\")  # Load previously fine-tuned model\n#processor = AutoProcessor.from_pretrained(\"/kaggle/input/deplot-finetuned-dot/deplot_finetuned\")\n\n# Prepare dataset\ndataset = CSVChartDataset(images_dir, annotations_csv, processor)\n\n# Split dataset into training and evaluation\ntrain_size = int(0.8 * len(dataset))\neval_size = len(dataset) - train_size\ntrain_dataset, eval_dataset = random_split(dataset, [train_size, eval_size])\n\n# Training arguments\ntraining_args = Seq2SeqTrainingArguments(\n    output_dir=\"/kaggle/working/deplot_finetuned_v2\",\n    eval_strategy=\"epoch\",\n    learning_rate=5e-5,\n    per_device_train_batch_size=1,\n    per_device_eval_batch_size=1,\n    num_train_epochs=5,\n    save_strategy=\"epoch\",\n    save_total_limit=2,\n    predict_with_generate=True,\n    fp16=True,\n    logging_dir=\"/kaggle/working/logs_v2\",\n    logging_steps=100,\n    push_to_hub=False,\n    dataloader_num_workers=2,\n    ddp_find_unused_parameters=False,\n    report_to=[\"none\"],\n    gradient_accumulation_steps=2,\n    no_cuda=False,\n    load_best_model_at_end=True\n)\n\n# Initialize trainer\ntrainer = Seq2SeqTrainer(\n    model=model,\n    args=training_args,\n    train_dataset=train_dataset,\n    eval_dataset=eval_dataset,\n    tokenizer=processor.tokenizer,\n    data_collator=custom_data_collator\n)\n\n# Train model\ntrainer.train()\n\n# Save the model\nmodel.save_pretrained(\"/kaggle/working/deplot_finetuned_v2\")\nprocessor.save_pretrained(\"/kaggle/working/deplot_finetuned_v2\")\n","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from torch.utils.data import Dataset, DataLoader, random_split\nimport pandas as pd\nimport json\nimport os\nimport torch\nfrom transformers import AutoProcessor, Pix2StructForConditionalGeneration\nfrom PIL import Image\n\n\ndevice = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n\n# Load the fine-tuned model and processor\nmodel = Pix2StructForConditionalGeneration.from_pretrained(\"/kaggle/input/deplot-finetuned-line-version-2\").to(device)\nprocessor = AutoProcessor.from_pretrained(\"/kaggle/input/deplot-finetuned-line-version-2\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load an example image for prediction\nimage_path = \"/kaggle/input/sortrd-chart-data/sorted_data/sorted_data/line/images/0003a50817cf.jpg\"\nimage = Image.open(image_path)\ndisplay(image)\n\ninputs = processor(images=image, text=\"Generate data series:\", return_tensors=\"pt\")\n\n# Move inputs to GPU\ninputs = {key: value.to(device) for key, value in inputs.items()}\n\npredictions = model.generate(**inputs, max_new_tokens=2048)\nprint(processor.decode(predictions[0], skip_special_tokens=True))","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"\\n\\n\\nstart loop:\")\nfor i in range(84360, 84401):\n    image_path = f\"/kaggle/input/benetech-extra-generated-data/graphs_l/{i}.jpg\"\n    try:\n        image = Image.open(image_path)\n        display(image)\n        inputs = processor(images=image, text=\"Generate data series:\", return_tensors=\"pt\")\n        \n        # Move inputs to GPU\n        inputs = {key: value.to(device) for key, value in inputs.items()}\n        \n        predictions = model.generate(**inputs, max_new_tokens=2048)\n        print(processor.decode(predictions[0], skip_special_tokens=True))\n    except FileNotFoundError:\n        print(f\"Image not found: {image_path}\")\n    except Exception as e:\n        print(f\"Error processing {image_path}: {e}\")","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nfrom PIL import Image\nimport torch\n\n# Folder path\nfolder_path = \"/kaggle/input/sortrd-chart-data/sorted_data/sorted_data/vertical_bar/images\"\ncnt = 0\n\n# Iterate over the images\nfor image_file in os.listdir(folder_path):\n    # Stop after processing 40 images\n    if cnt >= 40:\n        break\n    \n    image_path = os.path.join(folder_path, image_file)\n    \n    # Ensure the file is an image\n    if not image_file.lower().endswith(('.png', '.jpg', '.jpeg')):\n        continue\n    \n    cnt += 1\n    image = Image.open(image_path)\n    display(image)\n\n    # Prepare inputs\n    inputs = processor(images=image, text=\"Generate data series:\", return_tensors=\"pt\")\n\n    # Move inputs to GPU\n    inputs = {key: value.to(device) for key, value in inputs.items()}\n\n    # Generate predictions\n    predictions = model.generate(**inputs, max_new_tokens=2048)\n\n    # Decode and print predictions\n    print(f\"Predictions for {image_file}:\")\n    print(processor.decode(predictions[0], skip_special_tokens=True))\n","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}