Re-enable Arm CPU vLLM HUD Benchmarks

- re-enables the benchmarks disabled in #114
- related to vllm-project/vllm#26494 (not sure which needs to go in first)
- use default block_size in serving benchmarks (i.e. 128 instead of setting it to 16)

Signed-off-by: Fadi Arafeh <fadi.arafeh@arm.com>

Author: fadara01

.github/scripts/generate_vllm_benchmark_matrix.py

@@ -19,7 +19,7 @@
         "linux.rocm.gpu.gfx942.1",
         "linux.24xl.spr-metal",
         "linux.24xl.gnr",
-        # "linux.arm64.m7g.4xlarge",  # TODO (huydhn): This is not working yet
+        "linux.arm64.m8g.4xlarge",
         "linux.dgx.b200",
         "linux.hpu.gaudi3.8",
     ],
@@ -60,7 +60,7 @@
     "linux.rocm.gpu.gfx942.8": "rocm",
     "linux.24xl.spr-metal": "cpu",
     "linux.24xl.gnr": "cpu",
-    # "linux.arm64.m7g.4xlarge": "arm64-cpu",  # TODO (huydhn): This is not working yet
+    "linux.arm64.m8g.4xlarge": "arm64-cpu",
     "linux.hpu.gaudi3.8": "hpu",
 }
 

.github/scripts/test_generate_vllm_benchmark_matrix.py

@@ -22,7 +22,7 @@ def test_generate_benchmark_matrix():
 {
   "include": [
     {
-      "runner": "linux.arm64.m7g.4xlarge",
+      "runner": "linux.arm64.m8g.4xlarge",
       "models": "meta-llama/meta-llama-3.1-8b-instruct"
     },
     {
@@ -209,7 +209,7 @@ def test_generate_benchmark_matrix():
 {
   "include": [
     {
-      "runner": "linux.arm64.m7g.4xlarge",
+      "runner": "linux.arm64.m8g.4xlarge",
       "models": "meta-llama/meta-llama-3.1-8b-instruct"
     },
     {
@@ -247,7 +247,7 @@ def test_generate_benchmark_matrix():
 {
   "include": [
     {
-      "runner": "linux.arm64.m7g.4xlarge",
+      "runner": "linux.arm64.m8g.4xlarge",
       "models": "meta-llama/meta-llama-3.1-8b-instruct"
     },
     {
@@ -286,7 +286,7 @@ def test_generate_benchmark_matrix():
 {
   "include": [
     {
-      "runner": "linux.arm64.m7g.4xlarge",
+      "runner": "linux.arm64.m8g.4xlarge",
       "models": "meta-llama/meta-llama-3.1-8b-instruct"
     },
     {
@@ -321,7 +321,7 @@ def test_generate_benchmark_matrix():
 {
   "include": [
     {
-      "runner": "linux.arm64.m7g.4xlarge",
+      "runner": "linux.arm64.m8g.4xlarge",
       "models": "meta-llama/meta-llama-3.1-8b-instruct"
     },
     {
@@ -409,7 +409,7 @@ def test_generate_benchmark_matrix():
 
     # Select multiple runners
     models = []
-    runners = ["h100", "spr", "m7g"]
+    runners = ["h100", "spr", "m8g"]
     output = json.dumps(
         generate_benchmark_matrix(BENCHMARK_CONFIG_DIRS, models, runners), indent=2
     )
@@ -419,7 +419,7 @@ def test_generate_benchmark_matrix():
 {
   "include": [
     {
-      "runner": "linux.arm64.m7g.4xlarge",
+      "runner": "linux.arm64.m8g.4xlarge",
       "models": "meta-llama/meta-llama-3.1-8b-instruct"
     },
     {
@@ -624,7 +624,7 @@ def test_generate_benchmark_matrix():
         "meta-llama/meta-llama-3.1-8b-instruct",
         "mistralai/mixtral-8x7b-instruct-v0.1",
     ]
-    runners = ["rocm", "spr", "m7g"]
+    runners = ["rocm", "spr", "m8g"]
     output = json.dumps(
         generate_benchmark_matrix(BENCHMARK_CONFIG_DIRS, models, runners), indent=2
     )
@@ -634,7 +634,7 @@ def test_generate_benchmark_matrix():
 {
   "include": [
     {
-      "runner": "linux.arm64.m7g.4xlarge",
+      "runner": "linux.arm64.m8g.4xlarge",
       "models": "meta-llama/meta-llama-3.1-8b-instruct"
     },
     {

.github/workflows/vllm-benchmark.yml

@@ -25,10 +25,7 @@ on:
           A comma-separated list of runners from .github/scripts/generate_vllm_benchmark_matrix.py to run the benchmark (optional, default to run everything)
         required: true
         type: string
-        # TODO (huydhn): Remove aarch64 CPU benchmark running on m7g until the change
-        # from https://github.com/vllm-project/vllm/pull/26494#issuecomment-3537415441
-        # is resolved and merged
-        default: h100,rocm,spr,gnr,b200,gaudi3
+        default: h100,rocm,spr,gnr,b200,m8g,gaudi3
   pull_request:
     paths:
       - .github/workflows/vllm-benchmark.yml

vllm-benchmarks/benchmarks/arm64-cpu/serving-tests-arm64-cpu.json

@@ -14,7 +14,6 @@
             "device": "cpu",
             "dtype": "bfloat16",
             "distributed_executor_backend": "mp",
-            "block_size": 16,
             "trust_remote_code": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
@@ -43,7 +42,6 @@
             "device": "cpu",
             "dtype": "bfloat16",
             "distributed_executor_backend": "mp",
-            "block_size": 16,
             "trust_remote_code": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
@@ -72,7 +70,6 @@
             "device": "cpu",
             "dtype": "bfloat16",
             "distributed_executor_backend": "mp",
-            "block_size": 16,
             "trust_remote_code": "",
             "disable_log_stats": "",
             "disable_log_requests": "",
@@ -101,7 +98,6 @@
             "device": "cpu",
             "dtype": "bfloat16",
             "distributed_executor_backend": "mp",
-            "block_size": 16,
             "trust_remote_code": "",
             "enable_chunked_prefill": "",
             "disable_log_stats": "",