test: Add L0_backend_onnxruntime test for enabling bfloat16 dtype in ONNXRuntime backend

qa/L0_backend_onnxruntime/bfloat16_test.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import sys
+import unittest
+
+import numpy as np
+import pytest
+import tritonclient.grpc as grpcclient
+import tritonclient.http as httpclient
+
+# Client type can be passed as first arg (e.g. python bfloat16_test.py http) or via CLIENT_TYPE env.
+if len(sys.argv) >= 2 and sys.argv[1] in ("http", "grpc"):
+    os.environ["CLIENT_TYPE"] = sys.argv[1]
+    del sys.argv[1]
+
+
+class BFloat16Test(unittest.TestCase):
+    def setUp(self):
+        self.protocol = os.environ.get("CLIENT_TYPE", "http")
+        if self.protocol == "http":
+            self.client_ = httpclient.InferenceServerClient("localhost:8000")
+        else:
+            self.client_ = grpcclient.InferenceServerClient("localhost:8001")
+        self.model_name_ = "add_bf16"
+
+    def _infer_bf16(self, input0_data, input1_data):
+        """Helper to run BF16 inference and return the output numpy array."""
+        if self.protocol == "http":
+            input0 = httpclient.InferInput("INPUT0", [5, 5], "BF16")
+            input1 = httpclient.InferInput("INPUT1", [5, 5], "BF16")
+        else:
+            input0 = grpcclient.InferInput("INPUT0", [5, 5], "BF16")
+            input1 = grpcclient.InferInput("INPUT1", [5, 5], "BF16")
+        input0.set_data_from_numpy(input0_data)
+        input1.set_data_from_numpy(input1_data)
+
+        results = self.client_.infer(self.model_name_, [input0, input1])
+        return results.as_numpy("OUTPUT")
+
+    def test_bf16_add_variants(self):
+        """Run BF16 add for one case: zeros, negatives, large, small, cancellation, or identical."""
+        input0_val, input1_val, expected_val = (
+            [
+                (0.0, 0.0, 0.0),  # zeros
+                (-1.5, 3.5, 2.0),  # negatives / mixed
+                (100.0, 200.0, 300.0),  # large
+                (1e-2, 1e-2, 2e-2),  # small (near underflow)
+                (1.0, -1.0, 0.0),  # cancellation
+                (2.0, 2.0, 4.0),  # identical inputs
+            ],
+        )
+
+        shape = (5, 5)
+        output = self._infer_bf16(
+            np.full(shape, input0_val, dtype=np.float32),
+            np.full(shape, input1_val, dtype=np.float32),
+        )
+        self.assertEqual(output.dtype, np.float32)
+        np.testing.assert_allclose(output, expected_val)
+
+
+if __name__ == "__main__":
+    unittest.main()

qa/L0_backend_onnxruntime/models/add_bf16/1/model.onnx

@@ -0,0 +1,16 @@
+triton:w
+
+INPUT0
+INPUT1OUTPUT"Addbf16_addZ
+INPUT0
+
+
+Z
+INPUT1
+
+
+b
+OUTPUT
+
+
+B

qa/L0_backend_onnxruntime/models/add_bf16/config.pbtxt

@@ -0,0 +1,50 @@
+# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+platform: "onnxruntime_onnx"
+max_batch_size: 0
+input [
+  {
+    name: "INPUT0"
+    data_type: TYPE_BF16
+    dims: [5, 5]
+  },
+  {
+    name: "INPUT1"
+    data_type: TYPE_BF16
+    dims: [5, 5]
+  }
+]
+output [
+  {
+    name: "OUTPUT"
+    data_type: TYPE_BF16
+    dims: [5, 5]
+  }
+]
+instance_group: {
+  kind: KIND_GPU
+}

qa/L0_backend_onnxruntime/models/add_bf16/generate_model.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+
+import onnx
+
+
+def generate_bf16_add_model(models_dir):
+    """Generate a simple BFLOAT16 Add model (INPUT0 + INPUT1 = OUTPUT)."""
+    model_name = "add_bf16"
+    shape = [5, 5]
+    onnx_dtype = onnx.TensorProto.BFLOAT16
+
+    add = onnx.helper.make_node("Add", ["INPUT0", "INPUT1"], ["OUTPUT"])
+
+    input0 = onnx.helper.make_tensor_value_info("INPUT0", onnx_dtype, shape)
+    input1 = onnx.helper.make_tensor_value_info("INPUT1", onnx_dtype, shape)
+    output = onnx.helper.make_tensor_value_info("OUTPUT", onnx_dtype, shape)
+
+    graph_proto = onnx.helper.make_graph(
+        [add],
+        "bf16_add",
+        [input0, input1],
+        [output],
+    )
+    model_def = onnx.helper.make_model(graph_proto, producer_name="triton")
+    # Cap IR version for older ONNX Runtime (e.g. max supported 11)
+    model_def.ir_version = min(model_def.ir_version, 11)
+    # BFLOAT16 support requires opset 13+
+    model_def.opset_import[0].version = 13
+
+    model_dir = os.path.join(models_dir, model_name, "1")
+    os.makedirs(model_dir, exist_ok=True)
+    onnx.save(model_def, os.path.join(model_dir, "model.onnx"))
+
+    # Write config.pbtxt
+    config = """platform: "onnxruntime_onnx"
+max_batch_size: 0
+input [
+  {{
+    name: "INPUT0"
+    data_type: TYPE_BF16
+    dims: {shape}
+  }},
+  {{
+    name: "INPUT1"
+    data_type: TYPE_BF16
+    dims: {shape}
+  }}
+]
+output [
+  {{
+    name: "OUTPUT"
+    data_type: TYPE_BF16
+    dims: {shape}
+  }}
+]
+""".format(
+        shape=shape
+    )
+
+    config_path = os.path.join(models_dir, model_name, "config.pbtxt")
+    with open(config_path, "w") as f:
+        f.write(config)
+
+    print(f"Generated model '{model_name}' in {models_dir}")
+
+
+if __name__ == "__main__":
+    models_dir = os.path.join(os.getcwd(), "models")
+    os.makedirs(models_dir, exist_ok=True)
+    generate_bf16_add_model(models_dir)

qa/L0_backend_onnxruntime/test.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+export CUDA_VISIBLE_DEVICES=0
+
+SERVER=/opt/tritonserver/bin/tritonserver
+SERVER_LOG="./inference_server.log"
+CLIENT_LOG="./test.log"
+source ../common/util.sh
+
+rm -f *.log
+
+# BFLOAT16 test
+SERVER_ARGS="--model-repository=`pwd`/models"
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+RET=0
+
+set +e
+
+for client_type in http grpc; do
+    CLIENT_LOG="./bfloat16_test_${client_type}.log"
+    python bfloat16_test.py $client_type >>$CLIENT_LOG 2>&1
+    if [ $? -ne 0 ]; then
+        cat $CLIENT_LOG
+        echo -e "\n***\n*** Test Failed ($client_type)\n***"
+        RET=1
+    fi
+done
+
+set -e
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+if [ $RET -eq 0 ]; then
+    echo -e "\n***\n*** Test Passed\n***"
+else
+    echo -e "\n***\n*** Test FAILED\n***"
+fi
+
+exit $RET

qa/L0_infer/infer_test.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -157,9 +157,6 @@ def _infer_exact_helper(
                     input_dtype,
                     output0_dtype,
                     output1_dtype,
-                    (input_size,),
-                    (input_size,),
-                    (input_size,),
                 ):
                     ensemble_prefix.append(prefix)
 

qa/L0_infer_reshape/infer_reshape_test.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -155,9 +155,6 @@ def _full_reshape(self, dtype, input_shapes, output_shapes=None, no_batch=True):
                 dtype,
                 dtype,
                 dtype,
-                input_shapes[0],
-                input_shapes[0],
-                input_shapes[0],
             ):
                 # model that supports batching
                 for bs in (1, 8):

qa/L0_infer_variable/infer_variable_test.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# Copyright 2019-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -140,9 +140,6 @@ def _infer_exact_helper(
                 input_dtype,
                 output0_dtype,
                 output1_dtype,
-                input_shape,
-                input_shape,
-                input_shape,
             ):
                 ensemble_prefix.append(prefix)