Update README.md

README.md

@@ -49,13 +49,18 @@ import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 model_id = "hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoModelForCausalLM.from_pretrained(
+  model_id,
+  torch_dtype=torch.float16,
+  low_cpu_mem_usage=True,
+  device_map="auto",
+)
+
 prompt = [
   {"role": "system", "content": "You are a helpful assistant, that responds as a pirate."},
   {"role": "user", "content": "What's Deep Learning?"},
 ]
-
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-
 inputs = tokenizer.apply_chat_template(
   prompt,
   tokenize=True,
@@ -64,13 +69,6 @@ inputs = tokenizer.apply_chat_template(
   return_dict=True,
 ).to("cuda")
 
-model = AutoModelForCausalLM.from_pretrained(
-  model_id,
-  torch_dtype=torch.float16,
-  low_cpu_mem_usage=True,
-  device_map="auto",
-)
-
 outputs = model.generate(**inputs, do_sample=True, max_new_tokens=256)
 print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
 ```
@@ -92,13 +90,18 @@ from auto_gptq import AutoGPTQForCausalLM
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 model_id = "hugging-quants/Meta-Llama-3.1-70B-Instruct-GPTQ-INT4"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+model = AutoGPTQForCausalLM.from_pretrained(
+  model_id,
+  torch_dtype=torch.float16,
+  low_cpu_mem_usage=True,
+  device_map="auto",
+)
+
 prompt = [
   {"role": "system", "content": "You are a helpful assistant, that responds as a pirate."},
   {"role": "user", "content": "What's Deep Learning?"},
 ]
-
-tokenizer = AutoTokenizer.from_pretrained(model_id)
-
 inputs = tokenizer.apply_chat_template(
   prompt,
   tokenize=True,
@@ -107,13 +110,6 @@ inputs = tokenizer.apply_chat_template(
   return_dict=True,
 ).to("cuda")
 
-model = AutoGPTQForCausalLM.from_pretrained(
-  model_id,
-  torch_dtype=torch.float16,
-  low_cpu_mem_usage=True,
-  device_map="auto",
-)
-
 outputs = model.generate(**inputs, do_sample=True, max_new_tokens=256)
 print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
 ```