Update inference.py

inference.py

@@ -40,7 +40,7 @@ def model_fn(model_dir, context=None):
     model = LLM(
         model=model_dir,
         trust_remote_code=True,
-        tensor_parallel_size=2,  # Use 2 GPUs for parallelism
+        tensor_parallel_size=8,  # Use 8 GPUs for parallelism
         gpu_memory_utilization=0.9  # Optimal GPU usage
     )
     return model