diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml index d61fd88e0..92a4e0038 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p1d-dep8-dep32-c388-b4-mtp.yaml @@ -30,6 +30,7 @@ resources: backend: type: trtllm prefill_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -43,6 +44,7 @@ backend: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES: 4x decode_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -147,7 +149,6 @@ frontend: PRECISION: fp4 CONC: '388' DURATION: '3600' - KV_OFFLOADING: none ETCD_LEASE_TTL: '120' DYN_ROUTER_QUEUE_THRESHOLD: None DYN_TOKENIZER_CACHE: '1' @@ -173,7 +174,6 @@ benchmark: PRECISION: fp4 CONC: '388' DURATION: '3600' - KV_OFFLOADING: none INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: '8000' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml index cf82deb41..f8bd69f24 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p4d-dep4-tep8-c4-b1-mtp.yaml @@ -30,6 +30,7 @@ resources: backend: type: trtllm prefill_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -43,6 +44,7 @@ backend: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES: 4x decode_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -147,7 +149,6 @@ frontend: PRECISION: fp4 CONC: '4' DURATION: '3600' - KV_OFFLOADING: none ETCD_LEASE_TTL: '120' DYN_ROUTER_QUEUE_THRESHOLD: None DYN_TOKENIZER_CACHE: '1' @@ -173,7 +174,6 @@ benchmark: PRECISION: fp4 CONC: '4' DURATION: '3600' - KV_OFFLOADING: none INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: '8000' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml index 93836d384..2739536a4 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-1p6d-dep4-tep4-c24-b4-mtp.yaml @@ -30,6 +30,7 @@ resources: backend: type: trtllm prefill_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -43,6 +44,7 @@ backend: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES: 4x decode_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -147,7 +149,6 @@ frontend: PRECISION: fp4 CONC: '24' DURATION: '3600' - KV_OFFLOADING: none ETCD_LEASE_TTL: '120' DYN_ROUTER_QUEUE_THRESHOLD: None DYN_TOKENIZER_CACHE: '1' @@ -173,7 +174,6 @@ benchmark: PRECISION: fp4 CONC: '24' DURATION: '3600' - KV_OFFLOADING: none INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: '8000' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml index e2a5a5b38..963ffa7c5 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-2p1d-dep8-dep32-c736-b8-mtp.yaml @@ -30,6 +30,7 @@ resources: backend: type: trtllm prefill_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -43,6 +44,7 @@ backend: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES: 4x decode_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -148,7 +150,6 @@ frontend: PRECISION: fp4 CONC: '736' DURATION: '3600' - KV_OFFLOADING: none ETCD_LEASE_TTL: '120' DYN_ROUTER_QUEUE_THRESHOLD: None DYN_TOKENIZER_CACHE: '1' @@ -174,7 +175,6 @@ benchmark: PRECISION: fp4 CONC: '736' DURATION: '3600' - KV_OFFLOADING: none INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: '8000' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml index 0d40222e4..b3d4a9604 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-3p1d-dep8-dep16-c1152-b32-mtp.yaml @@ -30,6 +30,7 @@ resources: backend: type: trtllm prefill_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -43,6 +44,7 @@ backend: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES: 4x decode_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -154,7 +156,6 @@ frontend: PRECISION: fp4 CONC: '1152' DURATION: '3600' - KV_OFFLOADING: none ETCD_LEASE_TTL: '120' DYN_ROUTER_QUEUE_THRESHOLD: None DYN_TOKENIZER_CACHE: '1' @@ -180,7 +181,6 @@ benchmark: PRECISION: fp4 CONC: '1152' DURATION: '3600' - KV_OFFLOADING: none INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: '8000' diff --git a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml index d35338f15..9353b74ac 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/trtllm/deepseek-v4/agentx-gb300-20260811/dynamo-disagg-gb300-5p1d-dep8-dep16-c2626-b96-mtp.yaml @@ -30,6 +30,7 @@ resources: backend: type: trtllm prefill_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -43,6 +44,7 @@ backend: PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True CUDA_SCALE_LAUNCH_QUEUES: 4x decode_environment: + OMP_NUM_THREADS: '1' TLLM_LOG_LEVEL: INFO TRTLLM_SERVER_DISABLE_GC: '1' TRTLLM_WORKER_DISABLE_GC: '1' @@ -170,7 +172,6 @@ frontend: PRECISION: fp4 CONC: '2626' DURATION: '3600' - KV_OFFLOADING: none ETCD_LEASE_TTL: '120' DYN_ROUTER_QUEUE_THRESHOLD: None DYN_TOKENIZER_CACHE: '1' @@ -196,7 +197,6 @@ benchmark: PRECISION: fp4 CONC: '2626' DURATION: '3600' - KV_OFFLOADING: none INFMAX_CONTAINER_WORKSPACE: /infmax-workspace RESULT_DIR: /logs/agentic PORT: '8000' diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9fcda3df3..8bbec03e0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6547,3 +6547,12 @@ - "Filter AgentX traces at the same 202,752-token context limit used by both TileRT roles so oversized Weka trajectories are excluded before replay." - "Pin SemiAnalysisAI/srt-slurm PR #10 commit d1e6c97b3baf3e87103b6d83189544c3c7d61c38, stacked on the AMD/native-router PR #7 and base runtime PR #1, including explicit native HTTP dependencies, GLM-5.1-compatible Transformers v5 router tokenization, incomplete-snapshot recovery, backend-declared conversion GPU resources, pre-container NVIDIA driver-hook activation, and lossless Slurm container-environment exports." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2750 + +- config-keys: + - dsv4-fp4-gb300-dynamo-trt-agentx + scenario-type: + - agentic-coding + description: + - "Set OMP_NUM_THREADS=1 for the prefill and decode workers across the GB300 Dynamo-TensorRT-LLM AgentX recipe matrix." + - "Allow the frontend and benchmark environments to inherit the runtime KV offloading setting." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2711