Accelerate: Salvamento, Precisão Mista e Inferência (2/2)

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%writefile accelerate_scripts/10_accelerate_save_and_load_checkpoints.py
 
import torch
from torch.utils.data import DataLoader
from torch.optim import Adam
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import evaluate
import tqdm
 
# Importamos e inicializamos Accelerator
from accelerate import Accelerator
accelerator = Accelerator()
 
dataset = load_dataset("tweet_eval", "emoji")
num_classes = len(dataset["train"].info.features["label"].names)
max_len = 130
 
checkpoints = "cardiffnlp/twitter-roberta-base-irony"
tokenizer = AutoTokenizer.from_pretrained(checkpoints)
 
def tokenize_function(dataset):
    return tokenizer(dataset["text"], max_length=max_len, padding="max_length", truncation=True, return_tensors="pt")
tokenized_dataset = {
    "train": dataset["train"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "validation": dataset["validation"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "test": dataset["test"].map(tokenize_function, batched=True, remove_columns=["text"]),
}
tokenized_dataset["train"].set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])
tokenized_dataset["validation"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
tokenized_dataset["test"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
 
BS = 128
dataloader = {
    "train": DataLoader(tokenized_dataset["train"], batch_size=BS, shuffle=True),
    "validation": DataLoader(tokenized_dataset["validation"], batch_size=BS, shuffle=True),
    "test": DataLoader(tokenized_dataset["test"], batch_size=BS, shuffle=True),
}
 
model = AutoModelForSequenceClassification.from_pretrained(checkpoints)
model.classifier.out_proj = torch.nn.Linear(in_features=model.classifier.out_proj.in_features, out_features=num_classes, bias=True)
 
loss_function = torch.nn.CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=5e-4)
metric = evaluate.load("accuracy")
 
@accelerator.on_local_main_process
def print_something(something):
    print(something)
 
EPOCHS = 1
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = accelerator.device
 
# model.to(device)
model, optimizer, dataloader["train"], dataloader["validation"] = accelerator.prepare(model, optimizer, dataloader["train"], dataloader["validation"])
 
for i in range(EPOCHS):
    model.train()
    progress_bar_train = tqdm.tqdm(dataloader["train"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_train:
        optimizer.zero_grad()
 
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        loss = loss_function(outputs['logits'], labels)
 
        # loss.backward()
        accelerator.backward(loss)
        optimizer.step()
 
    model.eval()
    progress_bar_validation = tqdm.tqdm(dataloader["validation"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_validation:
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        with torch.no_grad():
            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs['logits'], axis=-1)
        # Recopilamos las predicciones de todos los dispositivos
        predictions = accelerator.gather_for_metrics(predictions)
        labels = accelerator.gather_for_metrics(labels)
 
        accuracy = metric.add_batch(predictions=predictions, references=labels)
    accuracy = metric.compute()
 
    # Guardamos los pesos
    accelerator.save_state("accelerate_scripts/checkpoints")
 
print_something(f"Accuracy = {accuracy['accuracy']}")
 
# Cargamos los pesos
accelerator.load_state("accelerate_scripts/checkpoints")
	
	
Copied

			

				
Overwriting accelerate_scripts/09_accelerate_save_and_load_checkpoints.py
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!accelerate launch accelerate_scripts/10_accelerate_save_and_load_checkpoints.py
	
	
Copied

			

				
100%|█████████████████████████████████████████| 176/176 [01:58<00:00,  1.48it/s]
100%|███████████████████████████████████████████| 20/20 [00:05<00:00,  3.40it/s]
Accuracy = 0.2142
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%writefile accelerate_scripts/11_accelerate_save_model.py
 
import torch
from torch.utils.data import DataLoader
from torch.optim import Adam
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import evaluate
import tqdm
 
# Importamos e inicializamos Accelerator
from accelerate import Accelerator
accelerator = Accelerator()
 
dataset = load_dataset("tweet_eval", "emoji")
num_classes = len(dataset["train"].info.features["label"].names)
max_len = 130
 
checkpoints = "cardiffnlp/twitter-roberta-base-irony"
tokenizer = AutoTokenizer.from_pretrained(checkpoints)
 
def tokenize_function(dataset):
    return tokenizer(dataset["text"], max_length=max_len, padding="max_length", truncation=True, return_tensors="pt")
tokenized_dataset = {
    "train": dataset["train"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "validation": dataset["validation"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "test": dataset["test"].map(tokenize_function, batched=True, remove_columns=["text"]),
}
tokenized_dataset["train"].set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])
tokenized_dataset["validation"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
tokenized_dataset["test"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
 
BS = 128
dataloader = {
    "train": DataLoader(tokenized_dataset["train"], batch_size=BS, shuffle=True),
    "validation": DataLoader(tokenized_dataset["validation"], batch_size=BS, shuffle=True),
    "test": DataLoader(tokenized_dataset["test"], batch_size=BS, shuffle=True),
}
 
model = AutoModelForSequenceClassification.from_pretrained(checkpoints)
model.classifier.out_proj = torch.nn.Linear(in_features=model.classifier.out_proj.in_features, out_features=num_classes, bias=True)
 
loss_function = torch.nn.CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=5e-4)
metric = evaluate.load("accuracy")
 
@accelerator.on_local_main_process
def print_something(something):
    print(something)
 
EPOCHS = 1
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = accelerator.device
 
# model.to(device)
model, optimizer, dataloader["train"], dataloader["validation"] = accelerator.prepare(model, optimizer, dataloader["train"], dataloader["validation"])
 
for i in range(EPOCHS):
    model.train()
    progress_bar_train = tqdm.tqdm(dataloader["train"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_train:
        optimizer.zero_grad()
 
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        loss = loss_function(outputs['logits'], labels)
 
        # loss.backward()
        accelerator.backward(loss)
        optimizer.step()
 
    model.eval()
    progress_bar_validation = tqdm.tqdm(dataloader["validation"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_validation:
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        with torch.no_grad():
            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs['logits'], axis=-1)
        # Recopilamos las predicciones de todos los dispositivos
        predictions = accelerator.gather_for_metrics(predictions)
        labels = accelerator.gather_for_metrics(labels)
 
        accuracy = metric.add_batch(predictions=predictions, references=labels)
    accuracy = metric.compute()
 
    # Guardamos el modelo
    accelerator.wait_for_everyone()
    accelerator.save_model(model, "accelerate_scripts/model", safe_serialization=True)
 
print_something(f"Accuracy = {accuracy['accuracy']}")
	
	
Copied

			

				
Writing accelerate_scripts/11_accelerate_save_model.py
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!accelerate launch accelerate_scripts/11_accelerate_save_model.py
	
	
Copied

			

				
100%|█████████████████████████████████████████| 176/176 [01:58<00:00,  1.48it/s]
100%|███████████████████████████████████████████| 20/20 [00:05<00:00,  3.35it/s]
Accuracy = 0.214
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%writefile accelerate_scripts/12_accelerate_save_pretrained.py
 
import torch
from torch.utils.data import DataLoader
from torch.optim import Adam
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import evaluate
import tqdm
 
# Importamos e inicializamos Accelerator
from accelerate import Accelerator
accelerator = Accelerator()
 
dataset = load_dataset("tweet_eval", "emoji")
num_classes = len(dataset["train"].info.features["label"].names)
max_len = 130
 
checkpoints = "cardiffnlp/twitter-roberta-base-irony"
tokenizer = AutoTokenizer.from_pretrained(checkpoints)
 
def tokenize_function(dataset):
    return tokenizer(dataset["text"], max_length=max_len, padding="max_length", truncation=True, return_tensors="pt")
tokenized_dataset = {
    "train": dataset["train"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "validation": dataset["validation"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "test": dataset["test"].map(tokenize_function, batched=True, remove_columns=["text"]),
}
tokenized_dataset["train"].set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])
tokenized_dataset["validation"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
tokenized_dataset["test"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
 
BS = 128
dataloader = {
    "train": DataLoader(tokenized_dataset["train"], batch_size=BS, shuffle=True),
    "validation": DataLoader(tokenized_dataset["validation"], batch_size=BS, shuffle=True),
    "test": DataLoader(tokenized_dataset["test"], batch_size=BS, shuffle=True),
}
 
model = AutoModelForSequenceClassification.from_pretrained(checkpoints)
model.classifier.out_proj = torch.nn.Linear(in_features=model.classifier.out_proj.in_features, out_features=num_classes, bias=True)
 
loss_function = torch.nn.CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=5e-4)
metric = evaluate.load("accuracy")
 
@accelerator.on_local_main_process
def print_something(something):
    print(something)
 
EPOCHS = 1
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = accelerator.device
 
# model.to(device)
model, optimizer, dataloader["train"], dataloader["validation"] = accelerator.prepare(model, optimizer, dataloader["train"], dataloader["validation"])
 
for i in range(EPOCHS):
    model.train()
    progress_bar_train = tqdm.tqdm(dataloader["train"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_train:
        optimizer.zero_grad()
 
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        loss = loss_function(outputs['logits'], labels)
 
        # loss.backward()
        accelerator.backward(loss)
        optimizer.step()
 
    model.eval()
    progress_bar_validation = tqdm.tqdm(dataloader["validation"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_validation:
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        with torch.no_grad():
            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs['logits'], axis=-1)
        # Recopilamos las predicciones de todos los dispositivos
        predictions = accelerator.gather_for_metrics(predictions)
        labels = accelerator.gather_for_metrics(labels)
 
        accuracy = metric.add_batch(predictions=predictions, references=labels)
    accuracy = metric.compute()
 
    # Guardamos el modelo pretrained
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained(
        "accelerate_scripts/model_pretrained",
        is_main_process=accelerator.is_main_process,
        save_function=accelerator.save,
    )
 
print_something(f"Accuracy = {accuracy['accuracy']}")
	
	
Copied

			

				
Writing accelerate_scripts/11_accelerate_save_pretrained.py
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!accelerate launch accelerate_scripts/12_accelerate_save_pretrained.py
	
	
Copied

			

				
Map: 100%|██████████████████████| 45000/45000 [00:02<00:00, 15152.47 examples/s]
Map: 100%|██████████████████████| 45000/45000 [00:02<00:00, 15119.13 examples/s]
Map: 100%|████████████████████████| 5000/5000 [00:00<00:00, 12724.70 examples/s]
Map: 100%|████████████████████████| 5000/5000 [00:00<00:00, 12397.49 examples/s]
Map: 100%|██████████████████████| 50000/50000 [00:03<00:00, 15247.21 examples/s]
Map: 100%|██████████████████████| 50000/50000 [00:03<00:00, 15138.03 examples/s]
100%|█████████████████████████████████████████| 176/176 [01:59<00:00,  1.48it/s]
100%|███████████████████████████████████████████| 20/20 [00:05<00:00,  3.37it/s]
Accuracy = 0.21
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
from transformers import AutoModel
 
checkpoints = "accelerate_scripts/model_pretrained"
tokenizer = AutoModel.from_pretrained(checkpoints)
	
	
Copied

			

				
Some weights of RobertaModel were not initialized from the model checkpoint at accelerate_scripts/model_pretrained and are newly initialized: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
import torch
from torch.utils.data import DataLoader
from torch.optim import Adam
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import evaluate
import tqdm
import time
# from accelerate import Accelerator
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
def train_code(batch_size: int = 64):
    from accelerate import Accelerator
    accelerator = Accelerator()
 
    dataset = load_dataset("tweet_eval", "emoji")
    num_classes = len(dataset["train"].info.features["label"].names)
    max_len = 130
 
    checkpoints = "cardiffnlp/twitter-roberta-base-irony"
    tokenizer = AutoTokenizer.from_pretrained(checkpoints)
 
    def tokenize_function(dataset):
        return tokenizer(dataset["text"], max_length=max_len, padding="max_length", truncation=True, return_tensors="pt")
    tokenized_dataset = {
        "train": dataset["train"].map(tokenize_function, batched=True, remove_columns=["text"]),
        "validation": dataset["validation"].map(tokenize_function, batched=True, remove_columns=["text"]),
        "test": dataset["test"].map(tokenize_function, batched=True, remove_columns=["text"]),
    }
    tokenized_dataset["train"].set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])
    tokenized_dataset["validation"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
    tokenized_dataset["test"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
 
    BS = batch_size
    dataloader = {
        "train": DataLoader(tokenized_dataset["train"], batch_size=BS, shuffle=True),
        "validation": DataLoader(tokenized_dataset["validation"], batch_size=BS, shuffle=True),
        "test": DataLoader(tokenized_dataset["test"], batch_size=BS, shuffle=True),
    }
 
    model = AutoModelForSequenceClassification.from_pretrained(checkpoints)
    model.classifier.out_proj = torch.nn.Linear(in_features=model.classifier.out_proj.in_features, out_features=num_classes, bias=True)
 
    loss_function = torch.nn.CrossEntropyLoss()
    optimizer = Adam(model.parameters(), lr=5e-4)
    metric = evaluate.load("accuracy")
 
    EPOCHS = 1
    # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    device = accelerator.device
 
    # model.to(device)
    model, optimizer, dataloader["train"], dataloader["validation"] = accelerator.prepare(model, optimizer, dataloader["train"], dataloader["validation"])
 
    for i in range(EPOCHS):
        model.train()
        progress_bar_train = tqdm.tqdm(dataloader["train"], disable=not accelerator.is_local_main_process)
        for batch in progress_bar_train:
            optimizer.zero_grad()
 
            input_ids = batch["input_ids"]#.to(device)
            attention_mask = batch["attention_mask"]#.to(device)
            labels = batch["label"]#.to(device)
 
            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
            loss = loss_function(outputs['logits'], labels)
 
            # loss.backward()
            accelerator.backward(loss)
            optimizer.step()
 
        model.eval()
        progress_bar_validation = tqdm.tqdm(dataloader["validation"], disable=not accelerator.is_local_main_process)
        for batch in progress_bar_validation:
            input_ids = batch["input_ids"]#.to(device)
            attention_mask = batch["attention_mask"]#.to(device)
            labels = batch["label"]#.to(device)
 
            with torch.no_grad():
                outputs = model(input_ids=input_ids, attention_mask=attention_mask)
            predictions = torch.argmax(outputs['logits'], axis=-1)
            # Recopilamos las predicciones de todos los dispositivos
            predictions = accelerator.gather_for_metrics(predictions)
            labels = accelerator.gather_for_metrics(labels)
 
            accuracy = metric.add_batch(predictions=predictions, references=labels)
        accuracy = metric.compute()
        
    accelerator.print(f"Accuracy = {accuracy['accuracy']}")
	
	
Copied

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
from accelerate import notebook_launcher
 
args = (128,)
notebook_launcher(train_code, args, num_processes=2)
	
	
Copied

			

				
Launching training on 2 GPUs.
			
		

			

				
100%|██████████| 176/176 [02:01<00:00,  1.45it/s]
100%|██████████| 20/20 [00:06<00:00,  3.31it/s]
			
		

			

				
Accuracy = 0.2112
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!accelerate config
	
	
Copied

			

				
--------------------------------------------------------------------------------
In which compute environment are you running?
This machine
--------------------------------------------------------------------------------
multi-GPU
How many different machines will you use (use more than 1 for multi-node training)? [1]: 1
Should distributed operations be checked while running for errors? This can avoid timeout issues but will be slower. [yes/NO]: no
Do you wish to optimize your script with torch dynamo?[yes/NO]:no
Do you want to use DeepSpeed? [yes/NO]: no
Do you want to use FullyShardedDataParallel? [yes/NO]: no
Do you want to use Megatron-LM ? [yes/NO]: no
How many GPU(s) should be used for distributed training? [1]:2
What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]:0,1
--------------------------------------------------------------------------------
Do you wish to use FP16 or BF16 (mixed precision)?
fp16
accelerate configuration saved at ~/.cache/huggingface/accelerate/default_config.yaml
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%writefile accelerate_scripts/13_accelerate_base_code_fp16_bs128.py
 
import torch
from torch.utils.data import DataLoader
from torch.optim import Adam
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import evaluate
import tqdm
 
# Importamos e inicializamos Accelerator
from accelerate import Accelerator
accelerator = Accelerator()
 
dataset = load_dataset("tweet_eval", "emoji")
num_classes = len(dataset["train"].info.features["label"].names)
max_len = 130
 
checkpoints = "cardiffnlp/twitter-roberta-base-irony"
tokenizer = AutoTokenizer.from_pretrained(checkpoints)
 
def tokenize_function(dataset):
    return tokenizer(dataset["text"], max_length=max_len, padding="max_length", truncation=True, return_tensors="pt")
tokenized_dataset = {
    "train": dataset["train"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "validation": dataset["validation"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "test": dataset["test"].map(tokenize_function, batched=True, remove_columns=["text"]),
}
tokenized_dataset["train"].set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])
tokenized_dataset["validation"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
tokenized_dataset["test"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
 
BS = 128
dataloader = {
    "train": DataLoader(tokenized_dataset["train"], batch_size=BS, shuffle=True),
    "validation": DataLoader(tokenized_dataset["validation"], batch_size=BS, shuffle=True),
    "test": DataLoader(tokenized_dataset["test"], batch_size=BS, shuffle=True),
}
 
model = AutoModelForSequenceClassification.from_pretrained(checkpoints)
model.classifier.out_proj = torch.nn.Linear(in_features=model.classifier.out_proj.in_features, out_features=num_classes, bias=True)
 
loss_function = torch.nn.CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=5e-4)
metric = evaluate.load("accuracy")
 
EPOCHS = 1
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = accelerator.device
 
# model.to(device)
model, optimizer, dataloader["train"], dataloader["validation"] = accelerator.prepare(model, optimizer, dataloader["train"], dataloader["validation"])
 
for i in range(EPOCHS):
    model.train()
    progress_bar_train = tqdm.tqdm(dataloader["train"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_train:
        optimizer.zero_grad()
 
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        loss = loss_function(outputs['logits'], labels)
 
        # loss.backward()
        accelerator.backward(loss)
        optimizer.step()
 
    model.eval()
    progress_bar_validation = tqdm.tqdm(dataloader["validation"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_validation:
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        with torch.no_grad():
            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs['logits'], axis=-1)
        # Recopilamos las predicciones de todos los dispositivos
        predictions = accelerator.gather_for_metrics(predictions)
        labels = accelerator.gather_for_metrics(labels)
 
        accuracy = metric.add_batch(predictions=predictions, references=labels)
    accuracy = metric.compute()
    
accelerator.print(f"Accuracy = {accuracy['accuracy']}")
	
	
Copied

			

				
Overwriting accelerate_scripts/12_accelerate_base_code_fp16_bs128.py
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%time
 
!accelerate launch accelerate_scripts/13_accelerate_base_code_fp16_bs128.py
	
	
Copied

			

				
Map: 100%|████████████████████████| 5000/5000 [00:00<00:00, 14983.76 examples/s]
Map: 100%|██████████████████████| 50000/50000 [00:03<00:00, 14315.47 examples/s]
100%|█████████████████████████████████████████| 176/176 [01:01<00:00,  2.88it/s]
100%|███████████████████████████████████████████| 20/20 [00:02<00:00,  6.84it/s]
Accuracy = 0.2094
CPU times: user 812 ms, sys: 163 ms, total: 976 ms
Wall time: 1min 27s
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%writefile accelerate_scripts/14_accelerate_base_code_fp16_bs256.py
 
import torch
from torch.utils.data import DataLoader
from torch.optim import Adam
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import evaluate
import tqdm
 
# Importamos e inicializamos Accelerator
from accelerate import Accelerator
accelerator = Accelerator()
 
dataset = load_dataset("tweet_eval", "emoji")
num_classes = len(dataset["train"].info.features["label"].names)
max_len = 130
 
checkpoints = "cardiffnlp/twitter-roberta-base-irony"
tokenizer = AutoTokenizer.from_pretrained(checkpoints)
 
def tokenize_function(dataset):
    return tokenizer(dataset["text"], max_length=max_len, padding="max_length", truncation=True, return_tensors="pt")
tokenized_dataset = {
    "train": dataset["train"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "validation": dataset["validation"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "test": dataset["test"].map(tokenize_function, batched=True, remove_columns=["text"]),
}
tokenized_dataset["train"].set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])
tokenized_dataset["validation"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
tokenized_dataset["test"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
 
BS = 256
dataloader = {
    "train": DataLoader(tokenized_dataset["train"], batch_size=BS, shuffle=True),
    "validation": DataLoader(tokenized_dataset["validation"], batch_size=BS, shuffle=True),
    "test": DataLoader(tokenized_dataset["test"], batch_size=BS, shuffle=True),
}
 
model = AutoModelForSequenceClassification.from_pretrained(checkpoints)
model.classifier.out_proj = torch.nn.Linear(in_features=model.classifier.out_proj.in_features, out_features=num_classes, bias=True)
 
loss_function = torch.nn.CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=5e-4)
metric = evaluate.load("accuracy")
 
EPOCHS = 1
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = accelerator.device
 
# model.to(device)
model, optimizer, dataloader["train"], dataloader["validation"] = accelerator.prepare(model, optimizer, dataloader["train"], dataloader["validation"])
 
for i in range(EPOCHS):
    model.train()
    progress_bar_train = tqdm.tqdm(dataloader["train"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_train:
        optimizer.zero_grad()
 
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        loss = loss_function(outputs['logits'], labels)
 
        # loss.backward()
        accelerator.backward(loss)
        optimizer.step()
 
    model.eval()
    progress_bar_validation = tqdm.tqdm(dataloader["validation"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_validation:
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        with torch.no_grad():
            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs['logits'], axis=-1)
        # Recopilamos las predicciones de todos los dispositivos
        predictions = accelerator.gather_for_metrics(predictions)
        labels = accelerator.gather_for_metrics(labels)
 
        accuracy = metric.add_batch(predictions=predictions, references=labels)
    accuracy = metric.compute()
    
accelerator.print(f"Accuracy = {accuracy['accuracy']}")
	
	
Copied

			

				
Overwriting accelerate_scripts/13_accelerate_base_code_fp16_bs256.py
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%time
 
!accelerate launch accelerate_scripts/14_accelerate_base_code_fp16_bs256.py
	
	
Copied

			

				
Map: 100%|████████████████████████| 5000/5000 [00:00<00:00, 15390.30 examples/s]
Map: 100%|████████████████████████| 5000/5000 [00:00<00:00, 14990.92 examples/s]
100%|███████████████████████████████████████████| 88/88 [00:54<00:00,  1.62it/s]
100%|███████████████████████████████████████████| 10/10 [00:02<00:00,  3.45it/s]
Accuracy = 0.2236
CPU times: user 670 ms, sys: 91.6 ms, total: 761 ms
Wall time: 1min 12s
			
		

Anteriormente treinamos em FP16 e agora vamos fazer isso em BF16, qual é a diferença?

Como podemos ver na imagem, enquanto o FP16, em comparação com o FP32, tem menos bits na mantissa e no expoente, o que faz com que seu intervalo seja muito menor, o BF16, em comparação com o FP32, tem o mesmo número de bits no expoente, mas menos na mantissa, o que faz com que o BF16 tenha o mesmo intervalo de números que o FP32, mas seja menos preciso

Isso é benéfico porque em FP16 alguns cálculos poderiam resultar em números muito altos, que no formato FP16 não poderiam ser representados. Além disso, há certos dispositivos de HW que estão otimizados para esse formato

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!accelerate config
	
	
Copied

			

				
--------------------------------------------------------------------------------
In which compute environment are you running?
This machine
--------------------------------------------------------------------------------
multi-GPU
How many different machines will you use (use more than 1 for multi-node training)? [1]: 1
Should distributed operations be checked while running for errors? This can avoid timeout issues but will be slower. [yes/NO]: no
Do you wish to optimize your script with torch dynamo?[yes/NO]:no
Do you want to use DeepSpeed? [yes/NO]: no
Do you want to use FullyShardedDataParallel? [yes/NO]: no
Do you want to use Megatron-LM ? [yes/NO]: no
How many GPU(s) should be used for distributed training? [1]:2
What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]:0,1
--------------------------------------------------------------------------------
Do you wish to use FP16 or BF16 (mixed precision)?
bf16
accelerate configuration saved at ~/.cache/huggingface/accelerate/default_config.yaml
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%time
 
!accelerate launch accelerate_scripts/14_accelerate_base_code_fp16_bs256.py
	
	
Copied

			

				
Map: 100%|██████████████████████| 50000/50000 [00:03<00:00, 14814.95 examples/s]
Map: 100%|██████████████████████| 50000/50000 [00:03<00:00, 14506.83 examples/s]
100%|███████████████████████████████████████████| 88/88 [00:51<00:00,  1.70it/s]
100%|███████████████████████████████████████████| 10/10 [00:03<00:00,  3.21it/s]
Accuracy = 0.2112
CPU times: user 688 ms, sys: 144 ms, total: 832 ms
Wall time: 1min 17s
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
!accelerate config
	
	
Copied

			

				
--------------------------------------------------------------------------------
In which compute environment are you running?
This machine
--------------------------------------------------------------------------------
multi-GPU
How many different machines will you use (use more than 1 for multi-node training)? [1]: 1
Should distributed operations be checked while running for errors? This can avoid timeout issues but will be slower. [yes/NO]: no
Do you wish to optimize your script with torch dynamo?[yes/NO]:no
Do you want to use DeepSpeed? [yes/NO]: no
Do you want to use FullyShardedDataParallel? [yes/NO]: no
Do you want to use Megatron-LM ? [yes/NO]: no
How many GPU(s) should be used for distributed training? [1]:2
What GPU(s) (by id) should be used for training on this machine as a comma-seperated list? [all]:0,1
--------------------------------------------------------------------------------
Do you wish to use FP16 or BF16 (mixed precision)?
fp8
accelerate configuration saved at ~/.cache/huggingface/accelerate/default_config.yaml
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%time
 
!accelerate launch accelerate_scripts/14_accelerate_base_code_fp16_bs256.py
	
	
Copied

			

				
Traceback (most recent call last):
  File "/home/wallabot/Documentos/web/portafolio/posts/accelerate_scripts/13_accelerate_base_code_fp16_bs256.py", line 12, in <module>
    accelerator = Accelerator()
                  ^^^^^^^^^^^^^
  File "/home/wallabot/miniconda3/envs/nlp/lib/python3.11/site-packages/accelerate/accelerator.py", line 371, in __init__
    self.state = AcceleratorState(
                 ^^^^^^^^^^^^^^^^^
  File "/home/wallabot/miniconda3/envs/nlp/lib/python3.11/site-packages/accelerate/state.py", line 790, in __init__
    raise ValueError(
ValueError: Using `fp8` precision requires `transformer_engine` or `MS-AMP` to be installed.
Traceback (most recent call last):
  File "/home/wallabot/Documentos/web/portafolio/posts/accelerate_scripts/13_accelerate_base_code_fp16_bs256.py", line 12, in <module>
    accelerator = Accelerator()
                  ^^^^^^^^^^^^^
  File "/home/wallabot/miniconda3/envs/nlp/lib/python3.11/site-packages/accelerate/accelerator.py", line 371, in __init__
    self.state = AcceleratorState(
                 ^^^^^^^^^^^^^^^^^
  File "/home/wallabot/miniconda3/envs/nlp/lib/python3.11/site-packages/accelerate/state.py", line 790, in __init__
    raise ValueError(
ValueError: Using `fp8` precision requires `transformer_engine` or `MS-AMP` to be installed.
...
[0]:
  time      : 2024-05-13_21:40:56
  host      : wallabot
  rank      : 0 (local_rank: 0)
  exitcode  : 1 (pid: 501480)
  error_file: <N/A>
  traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
============================================================
CPU times: user 65.1 ms, sys: 14.5 ms, total: 79.6 ms
Wall time: 7.24 s
			
		

	

		

			< >
			Input
		
		

			Python
			
				
				Copy
			
		
	
	

		
%%writefile accelerate_scripts/15_accelerate_base_code_fp8_bs512.py
 
import torch
from torch.utils.data import DataLoader
from torch.optim import Adam
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import evaluate
import tqdm
 
# Importamos e inicializamos Accelerator
from accelerate import Accelerator
accelerator = Accelerator()
 
dataset = load_dataset("tweet_eval", "emoji")
num_classes = len(dataset["train"].info.features["label"].names)
max_len = 130
 
checkpoints = "cardiffnlp/twitter-roberta-base-irony"
tokenizer = AutoTokenizer.from_pretrained(checkpoints)
 
def tokenize_function(dataset):
    return tokenizer(dataset["text"], max_length=max_len, padding="max_length", truncation=True, return_tensors="pt")
tokenized_dataset = {
    "train": dataset["train"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "validation": dataset["validation"].map(tokenize_function, batched=True, remove_columns=["text"]),
    "test": dataset["test"].map(tokenize_function, batched=True, remove_columns=["text"]),
}
tokenized_dataset["train"].set_format(type="torch", columns=['input_ids', 'attention_mask', 'label'])
tokenized_dataset["validation"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
tokenized_dataset["test"].set_format(type="torch", columns=['label', 'input_ids', 'attention_mask'])
 
BS = 512
dataloader = {
    "train": DataLoader(tokenized_dataset["train"], batch_size=BS, shuffle=True),
    "validation": DataLoader(tokenized_dataset["validation"], batch_size=BS, shuffle=True),
    "test": DataLoader(tokenized_dataset["test"], batch_size=BS, shuffle=True),
}
 
model = AutoModelForSequenceClassification.from_pretrained(checkpoints)
model.classifier.out_proj = torch.nn.Linear(in_features=model.classifier.out_proj.in_features, out_features=num_classes, bias=True)
 
loss_function = torch.nn.CrossEntropyLoss()
optimizer = Adam(model.parameters(), lr=5e-4)
metric = evaluate.load("accuracy")
 
EPOCHS = 1
# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = accelerator.device
 
# model.to(device)
model, optimizer, dataloader["train"], dataloader["validation"] = accelerator.prepare(model, optimizer, dataloader["train"], dataloader["validation"])
 
for i in range(EPOCHS):
    model.train()
    progress_bar_train = tqdm.tqdm(dataloader["train"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_train:
        optimizer.zero_grad()
 
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        loss = loss_function(outputs['logits'], labels)
 
        # loss.backward()
        accelerator.backward(loss)
        optimizer.step()
 
    model.eval()
    progress_bar_validation = tqdm.tqdm(dataloader["validation"], disable=not accelerator.is_local_main_process)
    for batch in progress_bar_validation:
        input_ids = batch["input_ids"]#.to(device)
        attention_mask = batch["attention_mask"]#.to(device)
        labels = batch["label"]#.to(device)
 
        with torch.no_grad():
            outputs = model(input_ids=input_ids, attention_mask=attention_mask)
        predictions = torch.argmax(outputs['logits'], axis=-1)
        # Recopilamos las predicciones de todos los dispositivos
        predictions = accelerator.gather_for_metrics(predictions)
        labels = accelerator.gather_for_metrics(labels)
 
        accuracy = metric.add_batch(predictions=predictions, references=labels)
    accuracy = metric.compute()
    
accelerator.print(f"Accuracy = {accuracy['accuracy']}")
	
	
Copied