BLIP-2: Modelo Multimodal Visión-Lenguaje

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!$ conda create -n blip2 python=3.9
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!$ conda activate blip2
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!$ conda install pytorch torchvision torchaudio pytorch-cuda=11.7 -c pytorch -c nvidia
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!$ conda install -c anaconda pillow
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!$ conda install -y -c anaconda requests
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!$ conda install -y -c anaconda jupyter
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!$ pip install salesforce-lavis
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
import torch
from PIL import Image
import requests
from lavis.models import load_model_and_preprocess
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
img_url = 'https://upload.wikimedia.org/wikipedia/commons/thumb/4/4d/12_-_The_Mystical_King_Cobra_and_Coffee_Forests.jpg/800px-12_-_The_Mystical_King_Cobra_and_Coffee_Forests.jpg'
raw_image = Image.open(requests.get(img_url, stream=True).raw).convert('RGB')
display(raw_image.resize((500, 500)))
	
	
Copied

			

				
<PIL.Image.Image image mode=RGB size=500x500>
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
device = torch.device("cuda" if torch.cuda.is_available() else 'cpu')
device
	
	
Copied

			

				
device(type='cuda')
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
# name = "blip2_opt"; model_type = "pretrain_opt2.7b"           # ok
# name = "blip2_opt"; model_type = "caption_coco_opt2.7b"       # FAIL VRAM
# name = "blip2_opt"; model_type = "pretrain_opt6.7b"           # FAIL RAM
# name = "blip2_opt"; model_type = "caption_coco_opt6.7b"       # FAIL RAM
 
# name = "blip2"; model_type = "pretrain"                       # FAIL type error
# name = "blip2"; model_type = "coco"                           # ok
 
name = "blip2_t5"; model_type = "pretrain_flant5xl" # ok
# name = "blip2_t5"; model_type = "caption_coco_flant5xl"       # FAIL VRAM
# name = "blip2_t5"; model_type = "pretrain_flant5xxl"          # FAIL
 
model, vis_processors, _ = load_model_and_preprocess(
    name=name, model_type=model_type, is_eval=True, device=device
)
 
vis_processors.keys()
	
	
Copied

			

				
Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
			
		

			

				
dict_keys(['train', 'eval'])
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
image = vis_processors["eval"](raw_image).unsqueeze(0).to(device)
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
model.generate({"image": image})
	
	
Copied

			

				
['a black and white snake']
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
prompt = None
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
def prepare_prompt(prompt, question):
    if prompt is None:
        prompt = question + " Answer:"
    else:
        prompt = prompt + " " + question + " Answer:"
    return prompt
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
def get_answer(prompt, question, model):
    prompt = prepare_prompt(prompt, question)
    answer = model.generate(
        {
            "image": image,
            "prompt": prompt
        }
    )
    answer = answer[0]
    prompt = prompt + " " + answer + "."
    return prompt, answer
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
question = "What's in the picture?"
prompt, answer = get_answer(prompt, question, model)
print(f"Question: {question}")
print(f"Answer: {answer}")
	
	
Copied

			

				
Question: What's in the picture?
Answer: a snake
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
question = "What kind of snake?"
prompt, answer = get_answer(prompt, question, model)
print(f"Question: {question}")
print(f"Answer: {answer}")
	
	
Copied

			

				
Question: What kind of snake?
Answer: cobra
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
question = "Is it poisonous?"
prompt, answer = get_answer(prompt, question, model)
print(f"Question: {question}")
print(f"Answer: {answer}")
	
	
Copied

			

				
Question: Is it poisonous?
Answer: yes
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
question = "If it bites me, can I die?"
prompt, answer = get_answer(prompt, question, model)
print(f"Question: {question}")
print(f"Answer: {answer}")
	
	
Copied

			

				
Question: If it bites me, can I die?
Answer: yes