Code to generate.

README.md

@@ -1,3 +1,34 @@
----
-license: apache-2.0
----
+---
+license: apache-2.0
+---
+
+Generated from:
+
+```python
+import torch
+from llmcompressor import oneshot
+from llmcompressor.modifiers.quantization import QuantizationModifier
+from transformers import (
+    AutoModelForCausalLM,  # type:ignore[reportPrivateImportUsage]
+    AutoTokenizer,  # type:ignore[reportPrivateImportUsage]
+)
+
+MODEL_ID = "tiiuae/falcon-7b-instruct"
+
+# Copied from <https://github.com/vllm-project/llm-compressor/blob/9d8a46418f517dd6399e2e9c179805247a7be584/examples/quantization_w8a8_fp8/README.md>
+model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.bfloat16)
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+
+# Configure the simple PTQ quantization
+recipe = QuantizationModifier(
+    targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"]
+)
+
+# Apply the quantization algorithm.
+oneshot(model=model, recipe=recipe)  # type:ignore[arg-type]
+
+# Save the model.
+SAVE_DIR = f"data/models/{MODEL_ID.split('/')[-1]}-FP8-Dynamic"
+model.save_pretrained(SAVE_DIR)
+tokenizer.save_pretrained(SAVE_DIR)
+```