From 98525927f0408a552b529585b7c7774ac9965a65 Mon Sep 17 00:00:00 2001 From: Joel Kiptoo Date: Wed, 30 Sep 2026 05:31:03 +0300 Subject: [PATCH 1/2] Convert matrix out/inout arguments with a matrix cast When the argument of an out or inout matrix parameter has a different element type (e.g. a half3x3 argument for an out float3x3 parameter), the copy back after the call went through ConvertScalarOrVector, which asserts on matrices, and the copy in for inout bitcast the matrix value, which is not a valid cast either. Use the same HLCast matrix conversion as an explicit matrix cast in both directions. Fixes #8809 --- docs/ReleaseNotes.md | 4 ++ tools/clang/lib/CodeGen/CGHLSLMS.cpp | 27 +++++++++++- .../matrix_out_param_conversion.hlsl | 42 +++++++++++++++++++ 3 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl diff --git a/docs/ReleaseNotes.md b/docs/ReleaseNotes.md index 4da2a4f644..e7323bcac7 100644 --- a/docs/ReleaseNotes.md +++ b/docs/ReleaseNotes.md @@ -67,6 +67,10 @@ line upon naming the release. Refer to previous for appropriate section names. - Fixed an optimizer crash when scalarizing an out-of-bounds vector access [#8940](https://github.com/microsoft/DirectXShaderCompiler/issues/8940). +- Fixed a crash when passing a matrix to an `out` or `inout` parameter whose + matrix type has a different element type, such as a `half3x3` argument for + an `out float3x3` parameter + [#8809](https://github.com/microsoft/DirectXShaderCompiler/issues/8809). ### Upcoming Preview Release diff --git a/tools/clang/lib/CodeGen/CGHLSLMS.cpp b/tools/clang/lib/CodeGen/CGHLSLMS.cpp index 1b806933e6..fefab8a577 100644 --- a/tools/clang/lib/CodeGen/CGHLSLMS.cpp +++ b/tools/clang/lib/CodeGen/CGHLSLMS.cpp @@ -4098,6 +4098,25 @@ EmitHLSLMatrixOperationCallImp(CGBuilderTy &Builder, HLOpcodeGroup group, return Builder.CreateCall(opFunc, opcodeParamList); } +// Converts a matrix value to another matrix type, the same way as an explicit +// matrix cast (e.g. float3x3 to half3x3). +static Value *EmitHLSLMatrixConversion(CGBuilderTy &Builder, Value *Val, + QualType FromTy, QualType ToTy, + llvm::Type *RetType, llvm::Module &M) { + const bool toUnsigned = hlsl::IsHLSLUnsigned(ToTy); + const bool fromUnsigned = hlsl::IsHLSLUnsigned(FromTy); + HLCastOpcode opcode = HLCastOpcode::DefaultCast; + if (toUnsigned && fromUnsigned) + opcode = HLCastOpcode::UnsignedUnsignedCast; + else if (toUnsigned) + opcode = HLCastOpcode::ToUnsignedCast; + else if (fromUnsigned) + opcode = HLCastOpcode::FromUnsignedCast; + return EmitHLSLMatrixOperationCallImp(Builder, HLOpcodeGroup::HLCast, + static_cast(opcode), RetType, + {Val}, M); +} + static Value *EmitHLSLArrayInit(CGBuilderTy &Builder, HLOpcodeGroup group, unsigned opcode, llvm::Type *RetType, ArrayRef paramList, llvm::Module &M) { @@ -6446,7 +6465,10 @@ void CGMSHLSLRuntime::EmitHLSLOutParamConversionInit( llvm::Type *ToTy = tmpArgAddr->getType()->getPointerElementType(); if (HLMatrixType::isa(ToTy)) { - Value *castVal = CGF.Builder.CreateBitCast(outVal, ToTy); + Value *castVal = outVal; + if (outVal->getType() != ToTy) + castVal = EmitHLSLMatrixConversion(CGF.Builder, outVal, ArgTy, + ParamTy, ToTy, TheModule); EmitHLSLMatrixStore(CGF, castVal, tmpArgAddr, ParamTy); } else { if (outVal->getType()->isVectorTy()) { @@ -6518,6 +6540,9 @@ void CGMSHLSLRuntime::EmitHLSLOutParamConversionCopyBack( castVal = CGF.Builder.CreateInsertElement(castVal, outVal, (uint64_t)0); } + } else if (HLMatrixType::isa(ToTy)) { + castVal = EmitHLSLMatrixConversion(CGF.Builder, outVal, ParamTy, + ArgTy, ToTy, TheModule); } else { castVal = ConvertScalarOrVector(CGF, outVal, tmpLV.getType(), argLV.getType()); diff --git a/tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl b/tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl new file mode 100644 index 0000000000..035b99b67e --- /dev/null +++ b/tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl @@ -0,0 +1,42 @@ +// RUN: %dxc -E main -T cs_6_2 -enable-16bit-types -Od %s | FileCheck %s + +// Out and inout matrix parameters whose argument has a different element type +// are converted like an explicit matrix cast when copied in and out, instead +// of hitting an assert in the scalar/vector conversion. + +StructuredBuffer In : register(t0); +StructuredBuffer InInt : register(t1); +RWStructuredBuffer Out : register(u0); +RWStructuredBuffer OutUint : register(u1); + +void fill(out float2x2 m) { m = float2x2(In[0].xy, In[0].zw); } +void twice(inout float2x2 m) { m = m * 2; } +void fillInt(out int2x2 m) { m = int2x2(InInt[0].xy, InInt[0].zw); } + +[numthreads(1, 1, 1)] +void main() { + // out float2x2 -> half2x2: converted when copied back. + // CHECK: fptrunc float %{{.*}} to half + // CHECK: fptrunc float %{{.*}} to half + // CHECK: fptrunc float %{{.*}} to half + // CHECK: fptrunc float %{{.*}} to half + half2x2 h; + fill(h); + Out[0] = float4(h[0], h[1]); + + // inout half2x2 -> float2x2: converted on the way in and on the way out. + // CHECK: rawBufferLoad.f32 + // CHECK: fpext half %{{.*}} to float + // CHECK: fmul fast float %{{.*}}, 2.000000e+00 + // CHECK: fptrunc float %{{.*}} to half + half2x2 m = half2x2(In[1].xy, In[1].zw); + twice(m); + Out[1] = float4(m[0], m[1]); + + // out int2x2 -> uint2x2: same bits, no conversion needed. + // CHECK: rawBufferLoad.i32 + // CHECK: rawBufferStore.i32 + uint2x2 u; + fillInt(u); + OutUint[0] = uint4(u[0], u[1]); +} From f857308b73aef35012edff3197d3ce7dc9ecb605 Mon Sep 17 00:00:00 2001 From: Joel Kiptoo Date: Thu, 1 Oct 2026 10:07:22 +0300 Subject: [PATCH 2/2] Test unsigned/float matrix conversions for inout arguments An int to uint matrix copy doesn't show which cast opcode was used, so also cover an inout uint2x2 argument for a float2x2 parameter, which needs uitofp on the way in and fptoui on the way out. --- .../arguments/matrix_out_param_conversion.hlsl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl b/tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl index 035b99b67e..bc2485d810 100644 --- a/tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl +++ b/tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl @@ -6,12 +6,14 @@ StructuredBuffer In : register(t0); StructuredBuffer InInt : register(t1); +StructuredBuffer InUint : register(t2); RWStructuredBuffer Out : register(u0); RWStructuredBuffer OutUint : register(u1); void fill(out float2x2 m) { m = float2x2(In[0].xy, In[0].zw); } void twice(inout float2x2 m) { m = m * 2; } void fillInt(out int2x2 m) { m = int2x2(InInt[0].xy, InInt[0].zw); } +void half_it(inout float2x2 m) { m = m * 0.5; } [numthreads(1, 1, 1)] void main() { @@ -39,4 +41,13 @@ void main() { uint2x2 u; fillInt(u); OutUint[0] = uint4(u[0], u[1]); + + // inout uint2x2 -> float2x2: the unsigned conversions are used both ways. + // CHECK: rawBufferLoad.i32 + // CHECK: uitofp i32 %{{.*}} to float + // CHECK: fmul fast float %{{.*}}, 5.000000e-01 + // CHECK: fptoui float %{{.*}} to i32 + uint2x2 q = uint2x2(InUint[0].xy, InUint[0].zw); + half_it(q); + OutUint[1] = uint4(q[0], q[1]); }