diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 7671d2e026b03..bea3911686f5b 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -18,7 +18,7 @@ use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceRef; use rustc_codegen_ssa::traits::{ BackendTypes, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - LayoutTypeCodegenMethods, OverflowOp, StaticBuilderMethods, + LayoutTypeCodegenMethods, OverflowOp, ReturnSlot, StaticBuilderMethods, }; use rustc_data_structures::fx::FxHashSet; use rustc_middle::bug; @@ -603,6 +603,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, @@ -613,7 +614,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { let current_block = self.block; self.block = try_block; - let call = self.call(typ, fn_attrs, fn_abi, func, args, None, instance); // FIXME(antoyo): use funclet here? + // FIXME(antoyo): use funclet here? + let call = self.call(typ, fn_attrs, fn_abi, func, return_slot, args, None, instance); self.block = current_block; let return_value = @@ -641,13 +643,14 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>, instance: Option>, ) -> RValue<'gcc> { - let call_site = self.call(typ, fn_attrs, fn_abi, func, args, None, instance); + let call_site = self.call(typ, fn_attrs, fn_abi, func, return_slot, args, None, instance); let condition = self.context.new_rvalue_from_int(self.bool_type, 1); self.llbb().end_with_conditional(self.location, condition, then, catch); if let Some(_fn_abi) = fn_abi { @@ -1772,19 +1775,30 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, + return_slot: ReturnSlot>, args: &[RValue<'gcc>], funclet: Option<&Funclet>, _instance: Option>, ) -> RValue<'gcc> { + // FIXME: change this in the `rustc_codegen_gcc` repo after the sync, to use the `libgccjit` indirect return suppport. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { + let mut args = args.to_vec(); + // Prepend the indirect return pointer + args.insert(0, sret_ptr); + args + } + }; // FIXME(antoyo): remove when having a proper API. let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { // FIXME(antoyo): remove when the API supports a different type for functions. let func: Function<'gcc> = self.cx.rvalue_as_function(func); - self.function_call(func, args, funclet) + self.function_call(func, &args, funclet) } else { // If it's a not function that was defined, it's a function pointer. - self.function_ptr_call(typ, fn_abi, func, args, funclet) + self.function_ptr_call(typ, fn_abi, func, &args, funclet) }; if let Some(_fn_abi) = fn_abi { // FIXME(bjorn3): Apply function attributes @@ -1798,6 +1812,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { _fn_attrs: Option<&CodegenFnAttrs>, _fn_abi: &FnAbi<'tcx, Ty<'tcx>>, _llfn: Self::Value, + _return_slot: ReturnSlot, _args: &[Self::Value], _funclet: Option<&Self::Funclet>, _instance: Option>, diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 986c04e6b4813..5244b771724a5 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -16,7 +16,7 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, + IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, ReturnSlot, }; use rustc_codegen_ssa::{MemFlags, RetagInfo}; use rustc_data_structures::fx::FxHashSet; @@ -656,7 +656,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } // FIXME directly use the llvm intrinsic adjustment functions here - let llret = self.call(fn_ty, None, None, fn_ptr, &call_args, None, None); + let llret = + self.call(fn_ty, None, None, fn_ptr, ReturnSlot::Direct, &call_args, None, None); if is_cleanup { self.apply_attrs_to_cleanup_callsite(llret); } @@ -667,7 +668,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; - self.call(self.type_void(), None, None, func, &[], None, None); + self.call(self.type_void(), None, None, func, ReturnSlot::Direct, &[], None, None); } fn assume(&mut self, value: Self::Value) { @@ -1348,7 +1349,7 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>( let param_type = bx.u8_type.make_pointer(); let fn_type = bx.context.new_function_pointer_type(None, bx.type_void(), &[param_type], false); - bx.call(fn_type, None, None, try_func, &[data], None, None); + bx.call(fn_type, None, None, try_func, ReturnSlot::Direct, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. OperandValue::Immediate(bx.const_bool(false)).store(bx, dest); @@ -1421,21 +1422,41 @@ fn codegen_gnu_try<'gcc, 'tcx>( let zero = bx.cx.context.new_rvalue_zero(bx.int_type); let ptr = bx.cx.context.new_call(None, eh_pointer_builtin, &[zero]); let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); // NOTE: the blocks must be filled before adding the try/catch, otherwise gcc will not // generate a try/catch. // FIXME(antoyo): add a check in the libgccjit API to prevent this. bx.switch_to_block(current_block); - bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + then, + catch, + None, + None, + ); }); let func = unsafe { std::mem::transmute::, RValue<'gcc>>(func) }; // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, func, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + func, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); OperandValue::Immediate(ret).store(bx, dest); } diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 1efab9b7c496d..78f550ad1cedb 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -562,9 +562,20 @@ pub(crate) fn inline_asm_call<'ll>( assert!(catch_funclet.is_none()); bx.callbr(fty, None, None, v, inputs, dest.unwrap(), labels, None, None) } else if let Some((catch, funclet)) = catch_funclet { - bx.invoke(fty, None, None, v, inputs, dest.unwrap(), catch, funclet, None) + bx.invoke( + fty, + None, + None, + v, + ReturnSlot::Direct, + inputs, + dest.unwrap(), + catch, + funclet, + None, + ) } else { - bx.call(fty, None, None, v, inputs, None, None) + bx.call(fty, None, None, v, ReturnSlot::Direct, inputs, None, None) }; // Store mark in a metadata node so we can map LLVM errors diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 55bb6f1abeb96..ce5ecc2032ff9 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -451,15 +451,25 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, + return_slot: ReturnSlot<&'ll Value>, args: &[&'ll Value], then: &'ll BasicBlock, catch: &'ll BasicBlock, funclet: Option<&Funclet<'ll>>, instance: Option>, ) -> &'ll Value { + // If this function returns indirectly (`PassMode::Indirect`), + // the `return_slot` should be the first argument. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { + let mut args = args.to_vec(); + args.insert(0, sret_ptr); + args + } + }; debug!("invoke {:?} with args ({:?})", llfn, args); - - let args = self.check_call("invoke", llty, llfn, args); + let args = self.check_call("invoke", llty, llfn, &args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1452,13 +1462,23 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: &'ll Value, + return_slot: ReturnSlot<&'ll Value>, args: &[&'ll Value], funclet: Option<&Funclet<'ll>>, callee_instance: Option>, ) -> &'ll Value { + // If this function returns indirectly (`PassMode::Indirect`), + // the `return_slot` should be the first argument. + let args = match return_slot { + ReturnSlot::Direct => args.to_vec(), + ReturnSlot::Indirect(sret_ptr) => { + let mut args = args.to_vec(); + args.insert(0, sret_ptr); + args + } + }; debug!("call {:?} with args ({:?})", llfn, args); - - let args = self.check_call("call", llty, llfn, args); + let args = self.check_call("call", llty, llfn, &args); let funclet_bundle = funclet.map(|funclet| funclet.bundle()); let mut bundles: SmallVec<[_; 2]> = SmallVec::new(); if let Some(funclet_bundle) = funclet_bundle { @@ -1519,12 +1539,21 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { caller_attrs: Option<&CodegenFnAttrs>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, ) { - let call = - self.call(llty, caller_attrs, Some(fn_abi), llfn, args, funclet, callee_instance); + let call = self.call( + llty, + caller_attrs, + Some(fn_abi), + llfn, + return_slot, + args, + funclet, + callee_instance, + ); llvm::LLVMSetTailCallKind(call, llvm::TailCallKind::MustTail); match &fn_abi.ret.mode { @@ -1819,7 +1848,8 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { args: &[&'ll Value], ) -> &'ll Value { let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params); - self.call(ty, None, None, f, args, None, None) + // No LLVM intrinsic returns its data indirectly (via `sret`). + self.call(ty, None, None, f, ReturnSlot::Direct, args, None, None) } fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) { diff --git a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs index b0ac5c096efdf..8c0cad2ed7f72 100644 --- a/compiler/rustc_codegen_llvm/src/builder/autodiff.rs +++ b/compiler/rustc_codegen_llvm/src/builder/autodiff.rs @@ -6,7 +6,7 @@ use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::mir::IntrinsicResult; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; use rustc_codegen_ssa::mir::place::PlaceValue; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot}; use rustc_data_structures::thin_vec::ThinVec; use rustc_hir::attrs::RustcAutodiff; use rustc_middle::ty::{PseudoCanonicalInput, Ty, TyCtxt, TypingEnv}; @@ -377,7 +377,7 @@ pub(crate) fn generate_enzyme_call<'ll, 'tcx>( crate::typetree::add_tt(&bx, fn_to_diff, fnc_tree); } - let call = bx.call(enzyme_ty, None, None, ad_fn, &args, None, None); + let call = bx.call(enzyme_ty, None, None, ad_fn, ReturnSlot::Direct, &args, None, None); let fn_ret_ty = bx.cx.val_ty(call); if fn_ret_ty == bx.cx.type_void() || fn_ret_ty == bx.cx.type_struct(&[], false) { diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index 0b009321802cf..10081cec2ad56 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -6,7 +6,7 @@ use rustc_abi::Align; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::common::TypeKind; use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; -use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ReturnSlot}; use rustc_middle::bug; use rustc_middle::ty::offload_meta::{MappingFlags, OffloadMetadata, OffloadSize}; @@ -109,9 +109,18 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) { // } let bb = Builder::append_block(cx, desc_reg_fn, "entry"); let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, register_lib, &[omp_descriptor], None, None); - a.call(init_ty, None, None, init_rtls, &[], None, None); - a.call(atexit, None, None, atexit_fn, &[desc_unreg_fn], None, None); + a.call( + reg_lib_decl, + None, + None, + register_lib, + ReturnSlot::Direct, + &[omp_descriptor], + None, + None, + ); + a.call(init_ty, None, None, init_rtls, ReturnSlot::Direct, &[], None, None); + a.call(atexit, None, None, atexit_fn, ReturnSlot::Direct, &[desc_unreg_fn], None, None); a.ret_void(); // define internal void @.omp_offloading.descriptor_unreg() section ".text.startup" { @@ -121,7 +130,16 @@ pub(crate) fn register_offload<'ll>(cx: &CodegenCx<'ll, '_>) { // } let bb = Builder::append_block(cx, desc_unreg_fn, "entry"); let mut a = Builder::build(cx, bb); - a.call(reg_lib_decl, None, None, unregister_lib, &[omp_descriptor], None, None); + a.call( + reg_lib_decl, + None, + None, + unregister_lib, + ReturnSlot::Direct, + &[omp_descriptor], + None, + None, + ); a.ret_void(); // @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }] @@ -740,7 +758,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( let num_args = cx.get_const_i32(num_args); let args = vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; - builder.call(fn_ty, None, None, fn_to_call, &args, None, None); + builder.call(fn_ty, None, None, fn_to_call, ReturnSlot::Direct, &args, None, None); } // Step 2) @@ -784,7 +802,7 @@ pub(crate) fn gen_call_handling<'ll, 'tcx>( region_id, a5, ]; - builder.call(tgt_target_kernel_ty, None, None, tgt_decl, &args, None, None); + builder.call(tgt_target_kernel_ty, None, None, tgt_decl, ReturnSlot::Direct, &args, None, None); // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) // Step 4) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 4f80e5c6e81e1..a5419e3a024f1 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1340,7 +1340,7 @@ fn catch_unwind_intrinsic<'ll, 'tcx>( ) -> &'ll Value { if !bx.sess().panic_strategy().unwinds() { let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.call(try_func_ty, None, None, try_func, &[data], None, None); + bx.call(try_func_ty, None, None, try_func, ReturnSlot::Direct, &[data], None, None); // Return 0 unconditionally from the intrinsic call; // we can never unwind. bx.const_bool(false) @@ -1438,7 +1438,18 @@ fn codegen_msvc_try<'ll, 'tcx>( let ptr_align = bx.tcx().data_layout.pointer_align().abi; let slot = bx.alloca(ptr_size, ptr_align); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + normal, + catchswitch, + None, + None, + ); bx.switch_to_block(normal); bx.ret(bx.const_bool(false)); @@ -1486,7 +1497,16 @@ fn codegen_msvc_try<'ll, 'tcx>( let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); let ptr = bx.load(bx.type_ptr(), slot, ptr_align); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + ReturnSlot::Direct, + &[data, ptr], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); // The flag value of 64 indicates a "catch-all". @@ -1494,7 +1514,16 @@ fn codegen_msvc_try<'ll, 'tcx>( let flags = bx.const_i32(64); let null = bx.const_null(bx.type_ptr()); let funclet = bx.catch_pad(cs, &[null, flags, null]); - bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + ReturnSlot::Direct, + &[data, null], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -1503,7 +1532,16 @@ fn codegen_msvc_try<'ll, 'tcx>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); ret } @@ -1550,7 +1588,18 @@ fn codegen_wasm_try<'ll, 'tcx>( // } // let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + normal, + catchswitch, + None, + None, + ); bx.switch_to_block(normal); bx.ret(bx.const_bool(false)); @@ -1566,7 +1615,16 @@ fn codegen_wasm_try<'ll, 'tcx>( let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[], &[funclet.cleanuppad()]); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet), None); + bx.call( + catch_ty, + None, + None, + catch_func, + ReturnSlot::Direct, + &[data, ptr], + Some(&funclet), + None, + ); bx.catch_ret(&funclet, caught); bx.switch_to_block(caught); @@ -1575,7 +1633,16 @@ fn codegen_wasm_try<'ll, 'tcx>( // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); ret } @@ -1616,7 +1683,18 @@ fn codegen_gnu_try<'ll, 'tcx>( let data = llvm::get_param(bx.llfn(), 1); let catch_func = llvm::get_param(bx.llfn(), 2); let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); - bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None, None); + bx.invoke( + try_func_ty, + None, + None, + try_func, + ReturnSlot::Direct, + &[data], + then, + catch, + None, + None, + ); bx.switch_to_block(then); bx.ret(bx.const_bool(false)); @@ -1634,13 +1712,22 @@ fn codegen_gnu_try<'ll, 'tcx>( bx.add_clause(vals, tydesc); let ptr = bx.extract_value(vals, 0); let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); - bx.call(catch_ty, None, None, catch_func, &[data, ptr], None, None); + bx.call(catch_ty, None, None, catch_func, ReturnSlot::Direct, &[data, ptr], None, None); bx.ret(bx.const_bool(true)); }); // Note that no invoke is used here because by definition this function // can't panic (that's what it's catching). - let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None); + let ret = bx.call( + llty, + None, + None, + llfn, + ReturnSlot::Direct, + &[try_func, data, catch_func], + None, + None, + ); ret } diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs index b746bab643a34..0374af7dcf81c 100644 --- a/compiler/rustc_codegen_llvm/src/mono_item.rs +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -198,12 +198,21 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { args.push(llvm::get_param(alias_lldecl, index)); } + // For an indirect return, the alias's own first parameter is the + // caller-provided return slot: forward it to the aliasee as such. + let (return_slot, args) = if fn_abi.ret.is_indirect() { + let (sret_ptr, rest) = args.split_first().unwrap(); + (ReturnSlot::Indirect(*sret_ptr), rest) + } else { + (ReturnSlot::Direct, &args[..]) + }; let call = start_bx.call( fn_ty, Some(attrs), Some(fn_abi), aliasee, - &args, + return_slot, + args, None, Some(aliasee_instance), ); diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index 339c0a968e9d3..4e1faa58dfda9 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -592,7 +592,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( ) }; - let result = bx.call(start_ty, None, None, start_fn, &args, None, instance); + let result = + bx.call(start_ty, None, None, start_fn, ReturnSlot::Direct, &args, None, instance); if cx.sess().target.os == Os::Uefi { bx.ret(result); } else { diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 20d6c01f12f73..4228d39039e87 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -165,12 +165,16 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional /// return destination `destination` and the unwind action `unwind`. + /// The `return_slot` is [`ReturnSlot::Indirect`] for functions returning + /// via `PassMode::Indirect`, and points to a buffer where the return value + /// shall be stored. fn do_call>( &self, fx: &mut FunctionCx<'a, 'tcx, Bx>, bx: &mut Bx, fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, fn_ptr: Bx::Value, + return_slot: ReturnSlot, llargs: &[Bx::Value], destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>, mut unwind: mir::UnwindAction, @@ -244,8 +248,23 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { } }; + debug_assert_eq!( + return_slot.is_indirect(), + fn_abi.ret.is_indirect(), + "a return slot must be provided if and only if the return is `PassMode::Indirect`", + ); + if kind == CallKind::Tail { - bx.tail_call(fn_ty, caller_attrs, fn_abi, fn_ptr, llargs, self.funclet(fx), instance); + bx.tail_call( + fn_ty, + caller_attrs, + fn_abi, + fn_ptr, + return_slot, + llargs, + self.funclet(fx), + instance, + ); return MergingSucc::False; } @@ -260,6 +279,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, + return_slot, llargs, ret_llbb, unwind_block, @@ -291,6 +311,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> { caller_attrs, Some(fn_abi), fn_ptr, + return_slot, llargs, self.funclet(fx), instance, @@ -721,6 +742,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, drop_fn, + ReturnSlot::Direct, args, Some((ReturnDest::Nothing, target)), unwind, @@ -825,6 +847,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + ReturnSlot::Direct, &args, None, unwind, @@ -856,6 +879,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + ReturnSlot::Direct, &[], None, mir::UnwindAction::Unreachable, @@ -925,6 +949,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, llfn, + ReturnSlot::Direct, &[msg.0, msg.1], target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)), unwind, @@ -1205,21 +1230,24 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // We still need to call `make_return_dest` even if there's no `target`, since // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited, // and `make_return_dest` adds the return-place indirect pointer to `llargs`. - let destination = match kind { + let (destination, return_slot) = match kind { CallKind::Normal => { - let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs); - target.map(|target| (return_dest, target)) + let (return_dest, return_slot) = + self.make_return_dest(bx, destination, &fn_abi.ret); + (target.map(|target| (return_dest, target)), return_slot) } CallKind::Tail => { - if fn_abi.ret.is_indirect() { - match self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs) { - ReturnDest::Nothing => {} + let return_slot = if fn_abi.ret.is_indirect() { + match self.make_return_dest(bx, destination, &fn_abi.ret) { + (ReturnDest::Nothing, return_slot) => return_slot, _ => bug!( "tail calls to functions with indirect returns cannot store into a destination" ), } - } - None + } else { + ReturnSlot::Direct + }; + (None, return_slot) } }; @@ -1444,6 +1472,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx, fn_abi, fn_ptr, + return_slot, &llargs, destination, unwind, @@ -2369,7 +2398,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } else { let fn_ty = bx.fn_decl_backend_type(fn_abi); - let llret = bx.call(fn_ty, None, Some(fn_abi), fn_ptr, &[], funclet.as_ref(), None); + let llret = bx.call( + fn_ty, + None, + Some(fn_abi), + fn_ptr, + ReturnSlot::Direct, + &[], + funclet.as_ref(), + None, + ); bx.apply_attrs_to_cleanup_callsite(llret); } @@ -2405,11 +2443,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx: &mut Bx, dest: mir::Place<'tcx>, fn_ret: &ArgAbi<'tcx, Ty<'tcx>>, - llargs: &mut Vec, - ) -> ReturnDest<'tcx, Bx::Value> { + ) -> (ReturnDest<'tcx, Bx::Value>, ReturnSlot) { // If the return is ignored, we can just return a do-nothing `ReturnDest`. if fn_ret.is_ignore() { - return ReturnDest::Nothing; + return (ReturnDest::Nothing, ReturnSlot::Direct); } let dest = if let Some(index) = dest.as_local() { match self.locals[index] { @@ -2423,10 +2460,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // but the calling convention has an indirect return. let tmp = PlaceRef::alloca(bx, fn_ret.layout); tmp.storage_live(bx); - llargs.push(tmp.val.llval); - ReturnDest::IndirectOperand(tmp, index) + ( + ReturnDest::IndirectOperand(tmp, index), + ReturnSlot::Indirect(tmp.val.llval), + ) } else { - ReturnDest::DirectOperand(index) + (ReturnDest::DirectOperand(index), ReturnSlot::Direct) }; } LocalRef::Operand(_) => { @@ -2446,10 +2485,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // to create a temporary. span_bug!(self.mir.span, "can't directly store to unaligned value"); } - llargs.push(dest.val.llval); - ReturnDest::Nothing + (ReturnDest::Nothing, ReturnSlot::Indirect(dest.val.llval)) } else { - ReturnDest::Store(dest) + (ReturnDest::Store(dest), ReturnSlot::Direct) } } diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 4270073b29894..559eae3edbf03 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -760,6 +760,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { fn_attrs.as_deref(), Some(fn_abi), fn_ptr, + ReturnSlot::Direct, &[], None, Some(instance), diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs index 52ffc321cbb6f..e25421b7778a0 100644 --- a/compiler/rustc_codegen_ssa/src/size_of_val.rs +++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs @@ -77,6 +77,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( /* fn_attrs */ None, Some(fn_abi), llfn, + ReturnSlot::Direct, // we know the ABI here &[msg.0, msg.1], None, None, diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs index caf63abeef3ad..0eb0ed79fccf1 100644 --- a/compiler/rustc_codegen_ssa/src/traits/builder.rs +++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs @@ -34,6 +34,20 @@ pub enum OverflowOp { Mul, } +/// The location of the return value for the call. +#[derive(Copy, Clone, Debug)] +pub enum ReturnSlot { + Direct, + /// The return value will be passed via sret (e.g. `PassMode::Indirect`). + Indirect(V), +} + +impl ReturnSlot { + pub fn is_indirect(&self) -> bool { + matches!(self, ReturnSlot::Indirect(_)) + } +} + pub trait BuilderMethods<'a, 'tcx>: Sized + LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>> @@ -135,6 +149,7 @@ pub trait BuilderMethods<'a, 'tcx>: fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], then: Self::BasicBlock, catch: Self::BasicBlock, @@ -640,16 +655,20 @@ pub trait BuilderMethods<'a, 'tcx>: /// The typical case that they are None is during the codegen of intrinsics and lang-items, /// as those are "fake functions" with only a trivial ABI if any, et cetera. /// + /// `return_slot` must be `ReturnSlot::Indirect` if an argument uses `PassMode::Indirect`. + /// /// ## Return /// - /// Must return the value the function will return so it can be written to the destination, - /// assuming the function does not explicitly pass the destination as a pointer in `args`. + /// Must return the value the function will return so it can be written to the destination. + /// For calls with an indirect return, the returned value is meaningless and must not be + /// used: the return value lives in the return slot. fn call( &mut self, llty: Self::FunctionSignature, caller_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, fn_val: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, @@ -661,6 +680,7 @@ pub trait BuilderMethods<'a, 'tcx>: caller_attrs: Option<&CodegenFnAttrs>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, llfn: Self::Value, + return_slot: ReturnSlot, args: &[Self::Value], funclet: Option<&Self::Funclet>, callee_instance: Option>, diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs index f46d07ea5008e..1f013cb122070 100644 --- a/compiler/rustc_codegen_ssa/src/traits/mod.rs +++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs @@ -36,7 +36,7 @@ pub use self::asm::{ AsmBuilderMethods, AsmCodegenMethods, GlobalAsmOperandRef, InlineAsmOperandRef, }; pub use self::backend::{BackendTypes, CodegenBackend, ExtraBackendMethods}; -pub use self::builder::{BuilderMethods, OverflowOp}; +pub use self::builder::{BuilderMethods, OverflowOp, ReturnSlot}; pub use self::consts::ConstCodegenMethods; pub use self::coverageinfo::CoverageInfoBuilderMethods; pub use self::debuginfo::{DebugInfoBuilderMethods, DebugInfoCodegenMethods};