{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":85540,"databundleVersionId":9725394,"sourceType":"competition"},{"sourceId":119837,"sourceType":"modelInstanceVersion","modelInstanceId":100788,"modelId":124969}],"dockerImageVersionId":30776,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Requiremenet to use Qwen2VLForConditionalGeneration (see https://github.com/QwenLM/Qwen2-VL?tab=readme-ov-file#quickstart)\n!pip install git+https://github.com/huggingface/transformers@21fac7abba2a37fae86106f87fcf9974fd1e3830 accelerate ","metadata":{"execution":{"iopub.status.busy":"2024-09-24T10:48:28.881056Z","iopub.execute_input":"2024-09-24T10:48:28.881386Z","iopub.status.idle":"2024-09-24T10:49:05.19005Z","shell.execute_reply.started":"2024-09-24T10:48:28.881354Z","shell.execute_reply":"2024-09-24T10:49:05.188818Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import json\nimport numpy as np\nimport pandas as pd\nfrom PIL import Image, ImageOps\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\n\nimport torch\nfrom transformers import Qwen2VLForConditionalGeneration, AutoProcessor","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-24T10:49:05.195408Z","iopub.execute_input":"2024-09-24T10:49:05.196058Z","iopub.status.idle":"2024-09-24T10:49:12.129251Z","shell.execute_reply.started":"2024-09-24T10:49:05.19601Z","shell.execute_reply":"2024-09-24T10:49:12.128439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load model\n\nckpt = \"/kaggle/input/qwen2-vl/transformers/qwen2-vl-7b-instruct/1/\"\nmodel = Qwen2VLForConditionalGeneration.from_pretrained(ckpt, torch_dtype=torch.bfloat16, device_map=\"auto\")\nprocessor = AutoProcessor.from_pretrained(ckpt)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T10:49:12.24634Z","iopub.execute_input":"2024-09-24T10:49:12.24668Z","iopub.status.idle":"2024-09-24T10:49:19.668571Z","shell.execute_reply.started":"2024-09-24T10:49:12.246646Z","shell.execute_reply":"2024-09-24T10:49:19.667557Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 1. Describing a puzzle","metadata":{}},{"cell_type":"code","source":"def describe_puzzle(image, prompt):\n    \n    # Create prompt\n    messages = [\n        {\n            \"role\": \"user\",\n            \"content\": [\n                {\"type\": \"image\"},\n                {\"type\": \"text\", \"text\": prompt},\n            ],\n        },\n    ]\n\n    text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n    inputs = processor(text=[text], images=[image], return_tensors=\"pt\")\n    inputs = inputs.to(model.device)\n\n    # Run inference\n    generated_ids = model.generate(**inputs, max_new_tokens=128)\n    generated_ids = generated_ids[0, inputs.input_ids.shape[1]:]\n    generated_text = processor.decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)\n    return generated_text","metadata":{"execution":{"iopub.status.busy":"2024-09-24T10:49:19.669762Z","iopub.execute_input":"2024-09-24T10:49:19.670059Z","iopub.status.idle":"2024-09-24T10:49:19.677415Z","shell.execute_reply.started":"2024-09-24T10:49:19.670026Z","shell.execute_reply":"2024-09-24T10:49:19.676384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sanity check with a local image file\n\n# Specify the path to the local image file\nlocal_image_path = \"/kaggle/input/applications-of-deep-learning-wustl-fall-2024/SimPictures/SimPictures/Home-00003_Split-Train.png\"  # replace with the actual file path\nimage = Image.open(local_image_path)\ndisplay(image)\n\n#prompt = \"Describe the building and its surroundings\"\nprompt = \"Describe the architectural style, quality of construction, and condition of the building, noting distinctive features such as materials used, roof design, and window placements. Detail the building’s interior and exterior, including any unique elements like balconies, courtyards, or historical touches. Additionally, describe the surrounding area, mentioning proximity to parks, schools, shops, public transportation, and any major roads or highways. Include any specific neighborhood features, landmarks, or environmental factors that might enhance or detract from the location’s appeal, such as scenic views, noise levels, or nearby industrial sites.\"\n#prompt = \"Describe the mood of the picture\"\nprint(f\"Prompt: {prompt}\")\nprint(f\"Answer: {describe_puzzle(image, prompt)}\")\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T10:49:19.678588Z","iopub.execute_input":"2024-09-24T10:49:19.678865Z","iopub.status.idle":"2024-09-24T10:49:27.709036Z","shell.execute_reply.started":"2024-09-24T10:49:19.678834Z","shell.execute_reply":"2024-09-24T10:49:27.708075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom tqdm import tqdm  # For progress bar\n\n# Set the path to the directory containing the images\nimage_dir = \"/kaggle/input/applications-of-deep-learning-wustl-fall-2024/SimPictures/SimPictures/\"\n\n# Prepare a list to hold the data\ndata = []\n\n# Iterate over each image file in the directory\nfor image_file in tqdm(os.listdir(image_dir)):\n    if image_file.endswith(\".png\"):  # Only process PNG files\n        # Extract the Home ID from the filename\n        home_id = image_file.split('_')[0]  # e.g., \"Home-00003\"\n        \n        # Load the image\n        image_path = os.path.join(image_dir, image_file)\n        image = Image.open(image_path)\n        \n        # Generate the description using the model\n        #prompt = \"Describe the mood of the picture\"\n        prompt = \"Describe the architectural style, quality of construction, and condition of the building, noting distinctive features such as materials used, roof design, and window placements. Detail the building’s interior and exterior, including any unique elements like balconies, courtyards, or historical touches. Additionally, describe the surrounding area, mentioning proximity to parks, schools, shops, public transportation, and any major roads or highways. Include any specific neighborhood features, landmarks, or environmental factors that might enhance or detract from the location’s appeal, such as scenic views, noise levels, or nearby industrial sites.\"\n        description = describe_puzzle(image, prompt)\n        \n        # Append the result to the data list\n        data.append({\"homeid\": home_id, \"prompt\": description})\n\n# Convert to DataFrame\ndf = pd.DataFrame(data)\n\n# Save to CSV\noutput_csv_path = \"/kaggle/working/home_descriptions.csv\"\ndf.to_csv(output_csv_path, index=False)\n\n# Display the DataFrame\nprint(\"Data saved to CSV:\", output_csv_path)\ndf.head()\n","metadata":{},"execution_count":null,"outputs":[]}]}