Convert matrix out/inout arguments with a matrix cast

docs/ReleaseNotes.md

@@ -67,6 +67,10 @@ line upon naming the release. Refer to previous for appropriate section names.
 
 - Fixed an optimizer crash when scalarizing an out-of-bounds vector access
   [#8940](https://github.com/microsoft/DirectXShaderCompiler/issues/8940).
+- Fixed a crash when passing a matrix to an `out` or `inout` parameter whose
+  matrix type has a different element type, such as a `half3x3` argument for
+  an `out float3x3` parameter
+  [#8809](https://github.com/microsoft/DirectXShaderCompiler/issues/8809).
 
 ### Upcoming Preview Release
 

tools/clang/lib/CodeGen/CGHLSLMS.cpp

@@ -4098,6 +4098,25 @@ EmitHLSLMatrixOperationCallImp(CGBuilderTy &Builder, HLOpcodeGroup group,
   return Builder.CreateCall(opFunc, opcodeParamList);
 }
 
+// Converts a matrix value to another matrix type, the same way as an explicit
+// matrix cast (e.g. float3x3 to half3x3).
+static Value *EmitHLSLMatrixConversion(CGBuilderTy &Builder, Value *Val,
+                                       QualType FromTy, QualType ToTy,
+                                       llvm::Type *RetType, llvm::Module &M) {
+  const bool toUnsigned = hlsl::IsHLSLUnsigned(ToTy);
+  const bool fromUnsigned = hlsl::IsHLSLUnsigned(FromTy);
+  HLCastOpcode opcode = HLCastOpcode::DefaultCast;
+  if (toUnsigned && fromUnsigned)
+    opcode = HLCastOpcode::UnsignedUnsignedCast;
+  else if (toUnsigned)
+    opcode = HLCastOpcode::ToUnsignedCast;
+  else if (fromUnsigned)
+    opcode = HLCastOpcode::FromUnsignedCast;
+  return EmitHLSLMatrixOperationCallImp(Builder, HLOpcodeGroup::HLCast,
+                                        static_cast<unsigned>(opcode), RetType,
+                                        {Val}, M);
+}
+
 static Value *EmitHLSLArrayInit(CGBuilderTy &Builder, HLOpcodeGroup group,
                                 unsigned opcode, llvm::Type *RetType,
                                 ArrayRef<Value *> paramList, llvm::Module &M) {
@@ -6446,7 +6465,10 @@ void CGMSHLSLRuntime::EmitHLSLOutParamConversionInit(
 
         llvm::Type *ToTy = tmpArgAddr->getType()->getPointerElementType();
         if (HLMatrixType::isa(ToTy)) {
-          Value *castVal = CGF.Builder.CreateBitCast(outVal, ToTy);
+          Value *castVal = outVal;
+          if (outVal->getType() != ToTy)
+            castVal = EmitHLSLMatrixConversion(CGF.Builder, outVal, ArgTy,
+                                               ParamTy, ToTy, TheModule);
           EmitHLSLMatrixStore(CGF, castVal, tmpArgAddr, ParamTy);
         } else {
           if (outVal->getType()->isVectorTy()) {
@@ -6518,6 +6540,9 @@ void CGMSHLSLRuntime::EmitHLSLOutParamConversionCopyBack(
             castVal =
                 CGF.Builder.CreateInsertElement(castVal, outVal, (uint64_t)0);
           }
+        } else if (HLMatrixType::isa(ToTy)) {
+          castVal = EmitHLSLMatrixConversion(CGF.Builder, outVal, ParamTy,
+                                             ArgTy, ToTy, TheModule);
         } else {
           castVal = ConvertScalarOrVector(CGF, outVal, tmpLV.getType(),
                                           argLV.getType());

tools/clang/test/HLSLFileCheck/hlsl/functions/arguments/matrix_out_param_conversion.hlsl

@@ -0,0 +1,42 @@
+// RUN: %dxc -E main -T cs_6_2 -enable-16bit-types -Od %s | FileCheck %s
+
+// Out and inout matrix parameters whose argument has a different element type
+// are converted like an explicit matrix cast when copied in and out, instead
+// of hitting an assert in the scalar/vector conversion.
+
+StructuredBuffer<float4> In : register(t0);
+StructuredBuffer<int4> InInt : register(t1);
+RWStructuredBuffer<float4> Out : register(u0);
+RWStructuredBuffer<uint4> OutUint : register(u1);
+
+void fill(out float2x2 m) { m = float2x2(In[0].xy, In[0].zw); }
+void twice(inout float2x2 m) { m = m * 2; }
+void fillInt(out int2x2 m) { m = int2x2(InInt[0].xy, InInt[0].zw); }
+
+[numthreads(1, 1, 1)]
+void main() {
+  // out float2x2 -> half2x2: converted when copied back.
+  // CHECK: fptrunc float %{{.*}} to half
+  // CHECK: fptrunc float %{{.*}} to half
+  // CHECK: fptrunc float %{{.*}} to half
+  // CHECK: fptrunc float %{{.*}} to half
+  half2x2 h;
+  fill(h);
+  Out[0] = float4(h[0], h[1]);
+
+  // inout half2x2 -> float2x2: converted on the way in and on the way out.
+  // CHECK: rawBufferLoad.f32
+  // CHECK: fpext half %{{.*}} to float
+  // CHECK: fmul fast float %{{.*}}, 2.000000e+00
+  // CHECK: fptrunc float %{{.*}} to half
+  half2x2 m = half2x2(In[1].xy, In[1].zw);
+  twice(m);
+  Out[1] = float4(m[0], m[1]);
+
+  // out int2x2 -> uint2x2: same bits, no conversion needed.
+  // CHECK: rawBufferLoad.i32
+  // CHECK: rawBufferStore.i32
+  uint2x2 u;
+  fillInt(u);
+  OutUint[0] = uint4(u[0], u[1]);
+}