inference.py

from llama_cpp import Llama

# Path to the downloaded GGUF file (update this to match your folder and file)
model_path = "models/oss_20b_gguf/gpt-oss-20b-Q2_K_L.gguf"  

# Load the model
llm = Llama(
    model_path=model_path,
    n_ctx=2048,        # Context size
    n_threads=8        # Adjust based on your CPU cores
)

# Generate text
prompt = "Who is the iron man"
output = llm(
    prompt,
    max_tokens=200,
    temperature=0.7,
    stop=["</s>"]
)

print(output["choices"][0]["text"])