Update README.md

README.md

@@ -308,230 +308,467 @@ guidellm --model neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16 --tar
       <th></th>
       <th></th>
       <th></th>
-      <th style="text-align: center;" colspan="2" >Instruction Following<br>prefill: 256 tokens<br>decode: 128 tokens</th>
-      <th style="text-align: center;" colspan="2" >Multi-turn Chat<br>prefill: 512 tokens<br>decode: 256 tokens</th>
-      <th style="text-align: center;" colspan="2" >Docstring Generation<br>prefill: 768 tokens<br>decode: 128 tokens</th>
-      <th style="text-align: center;" colspan="2" >RAG<br>prefill: 1024 tokens<br>decode: 128 tokens</th>
-      <th style="text-align: center;" colspan="2" >Code Completion<br>prefill: 256 tokens<br>decode: 1024 tokens</th>
-      <th style="text-align: center;" colspan="2" >Code Fixing<br>prefill: 1024 tokens<br>decode: 1024 tokens</th>
-      <th style="text-align: center;" colspan="2" >Large Summarization<br>prefill: 4096 tokens<br>decode: 512 tokens</th>
-      <th style="text-align: center;" colspan="2" >Large RAG<br>prefill: 10240 tokens<br>decode: 1536 tokens</th>
+      <th style="text-align: center;" colspan="2" >Instruction Following<br>256 / 128</th>
+      <th style="text-align: center;" colspan="2" >Multi-turn Chat<br>512 / 256</th>
+      <th style="text-align: center;" colspan="2" >Docstring Generation<br>768 / 128</th>
+      <th style="text-align: center;" colspan="2" >RAG<br>1024 / 128</th>
+      <th style="text-align: center;" colspan="2" >Code Completion<br>256 / 1024</th>
+      <th style="text-align: center;" colspan="2" >Code Fixing<br>1024 / 1024</th>
+      <th style="text-align: center;" colspan="2" >Large Summarization<br>4096 / 512</th>
+      <th style="text-align: center;" colspan="2" >Large RAG<br>10240 / 1536</th>
     </tr>
     <tr>
       <th>GPU class</th>
       <th>Number of GPUs</th>
       <th>Model</th>
-      <th>Cost reduction</th>
+      <th>Average cost reduction</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
       <th>Latency (s)</th>
-      <th>QPS</th>
+      <th>QPD</th>
     </tr>
   </thead>
-  <tbody>
+  <tbody style="text-align: center" >
     <tr>
       <th rowspan="3" valign="top">A6000</th>
       <td>4</td>
       <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
-      <td></td>
+      <td>---</td>
       <td>7.4</td>
-      <td>151.6</td>
+      <td>152</td>
       <td>14.9</td>
-      <td>75.6</td>
+      <td>76</td>
       <td>7.5</td>
-      <td>149.1</td>
+      <td>149</td>
       <td>7.7</td>
-      <td>146.4</td>
+      <td>146</td>
       <td>57.2</td>
-      <td>19.7</td>
+      <td>20</td>
       <td>58.9</td>
-      <td>19.1</td>
+      <td>19</td>
       <td>31.9</td>
-      <td>35.2</td>
+      <td>35</td>
       <td>98.4</td>
-      <td>11.4</td>
+      <td>11</td>
     </tr>
     <tr>
       <td>2</td>
       <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8</th>
       <td>1.93</td>
       <td>7.7</td>
-      <td>291.8</td>
+      <td>292</td>
       <td>15.2</td>
-      <td>147.7</td>
+      <td>148</td>
       <td>7.8</td>
-      <td>287.5</td>
+      <td>287</td>
       <td>8.0</td>
-      <td>282.5</td>
+      <td>282</td>
       <td>60.7</td>
-      <td>37.1</td>
+      <td>37</td>
       <td>60.2</td>
-      <td>37.4</td>
+      <td>37</td>
       <td>32.3</td>
-      <td>69.6</td>
+      <td>70</td>
       <td>104.0</td>
-      <td>21.6</td>
+      <td>22</td>
     </tr>
     <tr>
       <td>2</td>
       <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
       <td>2.83</td>
       <td>4.9</td>
-      <td>457.1</td>
+      <td>457</td>
       <td>10.0</td>
-      <td>225.1</td>
+      <td>225</td>
       <td>5.5</td>
-      <td>411.5</td>
+      <td>411</td>
       <td>5.8</td>
-      <td>388.9</td>
+      <td>389</td>
       <td>38.9</td>
-      <td>57.8</td>
+      <td>58</td>
       <td>39.2</td>
-      <td>57.4</td>
+      <td>57</td>
       <td>23.7</td>
-      <td>94.8</td>
+      <td>95</td>
       <td>76.6</td>
-      <td>29.4</td>
+      <td>29</td>
     </tr>
     <tr>
       <th rowspan="3" valign="top">A100</th>
       <td>2</td>
       <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
-      <td></td>
+      <td>---</td>
       <td>6.4</td>
-      <td>156.9</td>
+      <td>157</td>
       <td>12.8</td>
-      <td>78.8</td>
+      <td>79</td>
       <td>6.6</td>
-      <td>152.9</td>
+      <td>153</td>
       <td>6.7</td>
-      <td>151.2</td>
+      <td>151</td>
       <td>50.4</td>
-      <td>19.9</td>
+      <td>20</td>
       <td>50.8</td>
-      <td>19.8</td>
+      <td>20</td>
       <td>27.0</td>
-      <td>37.3</td>
+      <td>37</td>
       <td>85.4</td>
-      <td>11.8</td>
+      <td>12</td>
     </tr>
     <tr>
       <td>2</td>
       <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8</th>
       <td>1.48</td>
       <td>4.1</td>
-      <td>245.2</td>
+      <td>245</td>
       <td>8.2</td>
-      <td>122.6</td>
+      <td>123</td>
       <td>4.2</td>
-      <td>238.0</td>
+      <td>238</td>
       <td>4.3</td>
-      <td>234.9</td>
+      <td>235</td>
       <td>32.4</td>
-      <td>31.0</td>
+      <td>31</td>
       <td>32.8</td>
-      <td>30.7</td>
+      <td>31</td>
       <td>17.6</td>
-      <td>57.1</td>
+      <td>57</td>
       <td>90.8</td>
-      <td>11.1</td>
+      <td>11</td>
     </tr>
     <tr>
       <td>1</td>
       <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
       <td>2.69</td>
       <td>4.6</td>
-      <td>439.9</td>
+      <td>440</td>
       <td>9.2</td>
-      <td>219.7</td>
+      <td>220</td>
       <td>4.9</td>
-      <td>406.5</td>
+      <td>407</td>
       <td>5.2</td>
-      <td>388.5</td>
+      <td>389</td>
       <td>35.3</td>
-      <td>57.0</td>
+      <td>57</td>
       <td>36.3</td>
-      <td>55.5</td>
+      <td>55</td>
       <td>21.2</td>
-      <td>94.9</td>
+      <td>95</td>
       <td>68.1</td>
-      <td>29.5</td>
+      <td>30</td>
     </tr>
     <tr>
       <th rowspan="3" valign="top">H100</th>
       <td>2</td>
       <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
-      <td></td>
+      <td>---</td>
       <td>3.8</td>
-      <td>149.0</td>
+      <td>149</td>
       <td>7.6</td>
-      <td>74.2</td>
+      <td>74</td>
       <td>3.9</td>
-      <td>145.9</td>
+      <td>146</td>
       <td>3.9</td>
-      <td>143.9</td>
+      <td>144</td>
       <td>30.0</td>
-      <td>18.8</td>
+      <td>19</td>
       <td>30.4</td>
-      <td>18.6</td>
+      <td>19</td>
       <td>16.1</td>
-      <td>34.9</td>
+      <td>35</td>
       <td>56.5</td>
-      <td>10.0</td>
+      <td>10</td>
     </tr>
     <tr>
       <td>2</td>
       <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic</th>
       <td>1.39</td>
       <td>2.7</td>
-      <td>209.9</td>
+      <td>210</td>
       <td>5.3</td>
-      <td>105.6</td>
+      <td>106</td>
       <td>2.7</td>
-      <td>206.7</td>
+      <td>207</td>
       <td>2.8</td>
-      <td>203.2</td>
+      <td>203</td>
       <td>21.1</td>
-      <td>26.7</td>
+      <td>27</td>
       <td>21.4</td>
-      <td>26.4</td>
+      <td>26</td>
       <td>11.5</td>
-      <td>48.9</td>
+      <td>49</td>
       <td>47.2</td>
-      <td>12.0</td>
+      <td>12</td>
     </tr>
     <tr>
       <td>1</td>
       <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
       <td>1.83</td>
       <td>4.0</td>
-      <td>276.7</td>
+      <td>277</td>
       <td>7.9</td>
-      <td>138.1</td>
+      <td>138</td>
       <td>4.1</td>
-      <td>266.2</td>
+      <td>266</td>
       <td>4.2</td>
-      <td>262.0</td>
+      <td>262</td>
       <td>31.2</td>
-      <td>35.1</td>
+      <td>35</td>
       <td>31.8</td>
-      <td>34.4</td>
+      <td>34</td>
       <td>17.8</td>
-      <td>61.3</td>
+      <td>61</td>
       <td>61.4</td>
-      <td>17.8</td>
+      <td>18</td>
+    </tr>
+  </tbody>
+</table>
+
+**Use case profiles: prompt tokens / generation tokens
+
+**QPD: Queries per dollar, based on on-demand cost at [Lambda Labs](https://lambdalabs.com/service/gpu-cloud) (observed on 2/18/2025).
+
+
+### Multi-stream asynchronous performance (measured with vLLM version 0.7.2)
+<table>
+  <thead>
+    <tr>
+      <th></th>
+      <th></th>
+      <th></th>
+      <th style="text-align: center;" colspan="2" >Instruction Following<br>256 / 128</th>
+      <th style="text-align: center;" colspan="2" >Multi-turn Chat<br>512 / 256</th>
+      <th style="text-align: center;" colspan="2" >Docstring Generation<br>768 / 128</th>
+      <th style="text-align: center;" colspan="2" >RAG<br>1024 / 128</th>
+      <th style="text-align: center;" colspan="2" >Code Completion<br>256 / 1024</th>
+      <th style="text-align: center;" colspan="2" >Code Fixing<br>1024 / 1024</th>
+      <th style="text-align: center;" colspan="2" >Large Summarization<br>4096 / 512</th>
+      <th style="text-align: center;" colspan="2" >Large RAG<br>10240 / 1536</th>
+    </tr>
+    <tr>
+      <th>Hardware</th>
+      <th>Model</th>
+      <th>Average cost reduction</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+      <th>Maximum throughput (QPS)</th>
+      <th>QPD</th>
+    </tr>
+  </thead>
+  <tbody style="text-align: center" >
+    <tr>
+      <th rowspan="3" valign="top">A6000x4</th>
+      <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
+      <td>---</td>
+      <td>3.65</td>
+      <td>4102</td>
+      <td>1.56</td>
+      <td>1757</td>
+      <td>1.90</td>
+      <td>2143</td>
+      <td>1.48</td>
+      <td>1665</td>
+      <td>0.44</td>
+      <td>493</td>
+      <td>0.34</td>
+      <td>380</td>
+      <td>0.22</td>
+      <td>245</td>
+      <td>0.05</td>
+      <td>55</td>
+    </tr>
+    <tr>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8</th>
+      <td>1.76</td>
+      <td>5.89</td>
+      <td>6625</td>
+      <td>2.94</td>
+      <td>3307</td>
+      <td>3.36</td>
+      <td>3775</td>
+      <td>2.59</td>
+      <td>2916</td>
+      <td>0.74</td>
+      <td>828</td>
+      <td>0.53</td>
+      <td>601</td>
+      <td>0.35</td>
+      <td>398</td>
+      <td>0.11</td>
+      <td>120</td>
+    </tr>
+    <tr>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
+      <td>1.48</td>
+      <td>4.91</td>
+      <td>5528</td>
+      <td>2.01</td>
+      <td>2259</td>
+      <td>2.03</td>
+      <td>2280</td>
+      <td>1.12</td>
+      <td>1255</td>
+      <td>1.11</td>
+      <td>1251</td>
+      <td>0.76</td>
+      <td>852</td>
+      <td>0.24</td>
+      <td>267</td>
+      <td>0.07</td>
+      <td>81</td>
+    </tr>
+    <tr>
+      <th rowspan="3" valign="top">A100x4</th>
+      <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
+      <td>---</td>
+      <td>10.41</td>
+      <td>5235</td>
+      <td>5.10</td>
+      <td>2565</td>
+      <td>5.50</td>
+      <td>2766</td>
+      <td>4.36</td>
+      <td>2193</td>
+      <td>1.49</td>
+      <td>751</td>
+      <td>1.21</td>
+      <td>607</td>
+      <td>0.89</td>
+      <td>447</td>
+      <td>0.19</td>
+      <td>98</td>
+    </tr>
+    <tr>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w8a8</th>
+      <td>1.63</td>
+      <td>18.11</td>
+      <td>9103</td>
+      <td>8.90</td>
+      <td>4477</td>
+      <td>9.41</td>
+      <td>4730</td>
+      <td>7.42</td>
+      <td>3731</td>
+      <td>2.44</td>
+      <td>1229</td>
+      <td>1.89</td>
+      <td>948</td>
+      <td>1.26</td>
+      <td>631</td>
+      <td>0.30</td>
+      <td>149</td>
+    </tr>
+    <tr>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
+      <td>1.12</td>
+      <td>12.63</td>
+      <td>6353</td>
+      <td>5.32</td>
+      <td>2673</td>
+      <td>5.58</td>
+      <td>2804</td>
+      <td>4.27</td>
+      <td>2144</td>
+      <td>2.30</td>
+      <td>1158</td>
+      <td>1.45</td>
+      <td>729</td>
+      <td>0.76</td>
+      <td>381</td>
+      <td>0.22</td>
+      <td>110</td>
+    </tr>
+    <tr>
+      <th rowspan="3" valign="top">H100x4</th>
+      <th>deepseek-ai/DeepSeek-R1-Distill-Llama-70B</th>
+      <td>---</td>
+      <td>14.04</td>
+      <td>2113</td>
+      <td>10.85</td>
+      <td>1634</td>
+      <td>12.25</td>
+      <td>1844</td>
+      <td>9.93</td>
+      <td>1494</td>
+      <td>3.68</td>
+      <td>554</td>
+      <td>2.82</td>
+      <td>425</td>
+      <td>1.81</td>
+      <td>273</td>
+      <td>0.35</td>
+      <td>52</td>
+    </tr>
+    <tr>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-FP8-dynamic</th>
+      <td>1.78</td>
+      <td>41.44</td>
+      <td>6236</td>
+      <td>19.64</td>
+      <td>2956</td>
+      <td>21.03</td>
+      <td>3166</td>
+      <td>16.72</td>
+      <td>2516</td>
+      <td>6.01</td>
+      <td>904</td>
+      <td>4.46</td>
+      <td>672</td>
+      <td>2.55</td>
+      <td>383</td>
+      <td>0.49</td>
+      <td>74</td>
+    </tr>
+    <tr>
+      <th>neuralmagic/DeepSeek-R1-Distill-Llama-70B-quantized.w4a16</th>
+      <td>1.45</td>
+      <td>36.61</td>
+      <td>5509</td>
+      <td>15.12</td>
+      <td>2275</td>
+      <td>16.24</td>
+      <td>2443</td>
+      <td>13.22</td>
+      <td>1990</td>
+      <td>5.48</td>
+      <td>825</td>
+      <td>3.01</td>
+      <td>453</td>
+      <td>2.07</td>
+      <td>312</td>
+      <td>0.43</td>
+      <td>64</td>
     </tr>
   </tbody>
-</table>
+</table>
+
+**Use case profiles: prompt tokens / generation tokens
+
+**QPS: Queries per second.
+
+**QPD: Queries per dollar, based on on-demand cost at [Lambda Labs](https://lambdalabs.com/service/gpu-cloud) (observed on 2/18/2025).
+