diff --git a/src/abi/mod.rs b/src/abi/mod.rs index cab5b35c1..60a128c8a 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -21,6 +21,7 @@ use rustc_session::Session; use rustc_span::source_map::Spanned; use rustc_target::callconv::{Conv, FnAbi, PassMode}; +pub(crate) use self::pass_mode::adjust_fn_abi_for_rust_abi_mistakes; use self::pass_mode::*; pub(crate) use self::returning::codegen_return; use crate::prelude::*; @@ -80,7 +81,10 @@ pub(crate) fn get_function_sig<'tcx>( clif_sig_from_fn_abi( tcx, default_call_conv, - &FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()), + &adjust_fn_abi_for_rust_abi_mistakes( + tcx, + FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()), + ), ) } @@ -125,7 +129,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { returns: Vec, args: &[Value], ) -> Cow<'_, [Value]> { - if self.tcx.sess.target.is_like_windows { + if self.tcx.sess.target.is_like_windows || self.tcx.sess.target.arch == "s390x" { let (mut params, mut args): (Vec<_>, Vec<_>) = params .into_iter() .zip(args) @@ -159,10 +163,36 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { pub(crate) fn lib_call_unadjusted( &mut self, name: &str, - params: Vec, + mut params: Vec, returns: Vec, args: &[Value], ) -> Cow<'_, [Value]> { + let adjust_ret_param = + if self.tcx.sess.target.is_like_windows || self.tcx.sess.target.arch == "s390x" { + returns.len() == 1 && returns[0].value_type == types::I128 + } else { + false + }; + + if adjust_ret_param { + params.insert(0, AbiParam::new(self.pointer_type)); + let ret_ptr = self.create_stack_slot(16, 16); + let mut args = args.to_vec(); + args.insert(0, ret_ptr.get_addr(self)); + self.lib_call_inner(name, params, vec![], &args); + Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]) + } else { + Cow::Borrowed(self.lib_call_inner(name, params, returns, &args)) + } + } + + fn lib_call_inner( + &mut self, + name: &str, + params: Vec, + returns: Vec, + args: &[Value], + ) -> &[Value] { let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv }; let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); @@ -175,7 +205,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } let results = self.bcx.inst_results(call_inst); assert!(results.len() <= 2, "{}", results.len()); - Cow::Borrowed(results) + results } } @@ -437,11 +467,14 @@ pub(crate) fn codegen_terminator_call<'tcx>( let extra_args = fx.tcx.mk_type_list_from_iter( extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.node.ty(fx.mir, fx.tcx))), ); - let fn_abi = if let Some(instance) = instance { - FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args) - } else { - FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_sig, extra_args) - }; + let fn_abi = adjust_fn_abi_for_rust_abi_mistakes( + fx.tcx, + if let Some(instance) = instance { + FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args) + } else { + FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_sig, extra_args) + }, + ); let is_cold = if fn_sig.abi() == ExternAbi::RustCold { true @@ -721,8 +754,11 @@ pub(crate) fn codegen_drop<'tcx>( def: ty::InstanceKind::Virtual(drop_instance.def_id(), 0), args: drop_instance.args, }; - let fn_abi = FullyMonomorphizedLayoutCx(fx.tcx) - .fn_abi_of_instance(virtual_drop, ty::List::empty()); + let fn_abi = adjust_fn_abi_for_rust_abi_mistakes( + fx.tcx, + FullyMonomorphizedLayoutCx(fx.tcx) + .fn_abi_of_instance(virtual_drop, ty::List::empty()), + ); let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); let sig = fx.bcx.import_signature(sig); @@ -764,8 +800,11 @@ pub(crate) fn codegen_drop<'tcx>( def: ty::InstanceKind::Virtual(drop_instance.def_id(), 0), args: drop_instance.args, }; - let fn_abi = FullyMonomorphizedLayoutCx(fx.tcx) - .fn_abi_of_instance(virtual_drop, ty::List::empty()); + let fn_abi = adjust_fn_abi_for_rust_abi_mistakes( + fx.tcx, + FullyMonomorphizedLayoutCx(fx.tcx) + .fn_abi_of_instance(virtual_drop, ty::List::empty()), + ); let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); let sig = fx.bcx.import_signature(sig); @@ -774,8 +813,11 @@ pub(crate) fn codegen_drop<'tcx>( _ => { assert!(!matches!(drop_instance.def, InstanceKind::Virtual(_, _))); - let fn_abi = FullyMonomorphizedLayoutCx(fx.tcx) - .fn_abi_of_instance(drop_instance, ty::List::empty()); + let fn_abi = adjust_fn_abi_for_rust_abi_mistakes( + fx.tcx, + FullyMonomorphizedLayoutCx(fx.tcx) + .fn_abi_of_instance(drop_instance, ty::List::empty()), + ); let arg_value = drop_place.place_ref( fx, diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index 7594a53fc..d0f2d04a6 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -3,7 +3,7 @@ use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose}; use rustc_abi::{Reg, RegKind}; use rustc_target::callconv::{ - ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, + ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, Conv, FnAbi, PassMode, }; use smallvec::{SmallVec, smallvec}; @@ -302,3 +302,33 @@ pub(super) fn cvalue_for_param<'tcx>( } } } + +pub(crate) fn adjust_fn_abi_for_rust_abi_mistakes<'tcx>( + tcx: TyCtxt<'tcx>, + fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>, +) -> &'tcx FnAbi<'tcx, Ty<'tcx>> { + if fn_abi.conv != Conv::Rust { + // Non-Rust ABI's should be correctly implemented. + return fn_abi; + } + + if !tcx.sess.target.is_like_windows && tcx.sess.target.arch != "s390x" { + // Out of the targets cg_clif supports, only Windows and s390x don't have two return + // registers. + return fn_abi; + } + + match fn_abi.ret.mode { + PassMode::Ignore | PassMode::Cast { .. } | PassMode::Indirect { .. } => return fn_abi, + PassMode::Direct(_) => { + if fn_abi.ret.layout.size <= tcx.data_layout.pointer_size { + return fn_abi; + } + } + PassMode::Pair(..) => {} + } + + let mut fn_abi = fn_abi.clone(); + fn_abi.ret.make_indirect(); + tcx.arena.alloc(fn_abi) +} diff --git a/src/base.rs b/src/base.rs index 85cf4c736..545a33797 100644 --- a/src/base.rs +++ b/src/base.rs @@ -105,7 +105,10 @@ pub(crate) fn codegen_fn<'tcx>( let block_map: IndexVec = (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect(); - let fn_abi = FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty()); + let fn_abi = adjust_fn_abi_for_rust_abi_mistakes( + tcx, + FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty()), + ); // Make FunctionCx let target_config = module.target_config(); @@ -186,6 +189,7 @@ pub(crate) fn compile_fn( context.clear(); context.func = codegened_func.func; + // FIXME run this code when define_function returns an error #[cfg(any())] // This is never true let _clif_guard = { use std::fmt::Write; diff --git a/src/lib.rs b/src/lib.rs index c9486a730..4b74aaedb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -294,16 +294,6 @@ fn build_isa(sess: &Session) -> Arc { } } - if let target_lexicon::OperatingSystem::Windows = target_triple.operating_system { - // FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510 - flags_builder.enable("enable_multi_ret_implicit_sret").unwrap(); - } - - if let target_lexicon::Architecture::S390x = target_triple.architecture { - // FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510 - flags_builder.enable("enable_multi_ret_implicit_sret").unwrap(); - } - if let target_lexicon::Architecture::Aarch64(_) | target_lexicon::Architecture::Riscv64(_) | target_lexicon::Architecture::X86_64 = target_triple.architecture