rkumar70900
/

Llama-3.2-3B-Instruct-GPTQ-INT4

Text Generation

text-generation-inference

4-bit precision

Model card Files Files and versions

rkumar70900 commited on Jul 12

Commit

a504f50

·

verified ·

1 Parent(s): 506e559

updating readme with cuda device

Files changed (1) hide show

README.md +2 -2

README.md CHANGED Viewed

@@ -50,12 +50,12 @@ import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
 model_id = "rkumar70900/Llama-3.2-3B-Instruct-GPTQ-INT4"
-tokenizer = AutoTokenizer.from_pretrained(model_id)
 model = AutoModelForCausalLM.from_pretrained(
   model_id,
   torch_dtype=torch.float16,
   low_cpu_mem_usage=True,
-  device_map="auto",
 )
 prompt = [

 from transformers import AutoModelForCausalLM, AutoTokenizer
 model_id = "rkumar70900/Llama-3.2-3B-Instruct-GPTQ-INT4"
+tokenizer = AutoTokenizer.from_pretrained(model_id, device_map="cuda")
 model = AutoModelForCausalLM.from_pretrained(
   model_id,
   torch_dtype=torch.float16,
   low_cpu_mem_usage=True,
+  device_map="cuda",
 )
 prompt = [