Remove dependency on enable_multi_ret_implicit_sret

src/abi/mod.rs

@@ -21,6 +21,7 @@ use rustc_session::Session;
 use rustc_span::source_map::Spanned;
 use rustc_target::callconv::{Conv, FnAbi, PassMode};
 
+pub(crate) use self::pass_mode::adjust_fn_abi_for_rust_abi_mistakes;
 use self::pass_mode::*;
 pub(crate) use self::returning::codegen_return;
 use crate::prelude::*;
@@ -80,7 +81,10 @@ pub(crate) fn get_function_sig<'tcx>(
     clif_sig_from_fn_abi(
         tcx,
         default_call_conv,
-        &FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()),
+        &adjust_fn_abi_for_rust_abi_mistakes(
+            tcx,
+            FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()),
+        ),
     )
 }
 
@@ -125,7 +129,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         returns: Vec<AbiParam>,
         args: &[Value],
     ) -> Cow<'_, [Value]> {
-        if self.tcx.sess.target.is_like_windows {
+        if self.tcx.sess.target.is_like_windows || self.tcx.sess.target.arch == "s390x" {
             let (mut params, mut args): (Vec<_>, Vec<_>) = params
                 .into_iter()
                 .zip(args)
@@ -159,10 +163,36 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     pub(crate) fn lib_call_unadjusted(
         &mut self,
         name: &str,
-        params: Vec<AbiParam>,
+        mut params: Vec<AbiParam>,
         returns: Vec<AbiParam>,
         args: &[Value],
     ) -> Cow<'_, [Value]> {
+        let adjust_ret_param =
+            if self.tcx.sess.target.is_like_windows || self.tcx.sess.target.arch == "s390x" {
+                returns.len() == 1 && returns[0].value_type == types::I128
+            } else {
+                false
+            };
+
+        if adjust_ret_param {
+            params.insert(0, AbiParam::new(self.pointer_type));
+            let ret_ptr = self.create_stack_slot(16, 16);
+            let mut args = args.to_vec();
+            args.insert(0, ret_ptr.get_addr(self));
+            self.lib_call_inner(name, params, vec![], &args);
+            Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())])
+        } else {
+            Cow::Borrowed(self.lib_call_inner(name, params, returns, &args))
+        }
+    }
+
+    fn lib_call_inner(
+        &mut self,
+        name: &str,
+        params: Vec<AbiParam>,
+        returns: Vec<AbiParam>,
+        args: &[Value],
+    ) -> &[Value] {
         let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
         let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
         let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
@@ -175,7 +205,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         }
         let results = self.bcx.inst_results(call_inst);
         assert!(results.len() <= 2, "{}", results.len());
-        Cow::Borrowed(results)
+        results
     }
 }
 
@@ -437,11 +467,14 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     let extra_args = fx.tcx.mk_type_list_from_iter(
         extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.node.ty(fx.mir, fx.tcx))),
     );
-    let fn_abi = if let Some(instance) = instance {
-        FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args)
-    } else {
-        FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_sig, extra_args)
-    };
+    let fn_abi = adjust_fn_abi_for_rust_abi_mistakes(
+        fx.tcx,
+        if let Some(instance) = instance {
+            FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args)
+        } else {
+            FullyMonomorphizedLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_sig, extra_args)
+        },
+    );
 
     let is_cold = if fn_sig.abi() == ExternAbi::RustCold {
         true
@@ -721,8 +754,11 @@ pub(crate) fn codegen_drop<'tcx>(
                     def: ty::InstanceKind::Virtual(drop_instance.def_id(), 0),
                     args: drop_instance.args,
                 };
-                let fn_abi = FullyMonomorphizedLayoutCx(fx.tcx)
-                    .fn_abi_of_instance(virtual_drop, ty::List::empty());
+                let fn_abi = adjust_fn_abi_for_rust_abi_mistakes(
+                    fx.tcx,
+                    FullyMonomorphizedLayoutCx(fx.tcx)
+                        .fn_abi_of_instance(virtual_drop, ty::List::empty()),
+                );
 
                 let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
                 let sig = fx.bcx.import_signature(sig);
@@ -764,8 +800,11 @@ pub(crate) fn codegen_drop<'tcx>(
                     def: ty::InstanceKind::Virtual(drop_instance.def_id(), 0),
                     args: drop_instance.args,
                 };
-                let fn_abi = FullyMonomorphizedLayoutCx(fx.tcx)
-                    .fn_abi_of_instance(virtual_drop, ty::List::empty());
+                let fn_abi = adjust_fn_abi_for_rust_abi_mistakes(
+                    fx.tcx,
+                    FullyMonomorphizedLayoutCx(fx.tcx)
+                        .fn_abi_of_instance(virtual_drop, ty::List::empty()),
+                );
 
                 let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
                 let sig = fx.bcx.import_signature(sig);
@@ -774,8 +813,11 @@ pub(crate) fn codegen_drop<'tcx>(
             _ => {
                 assert!(!matches!(drop_instance.def, InstanceKind::Virtual(_, _)));
 
-                let fn_abi = FullyMonomorphizedLayoutCx(fx.tcx)
-                    .fn_abi_of_instance(drop_instance, ty::List::empty());
+                let fn_abi = adjust_fn_abi_for_rust_abi_mistakes(
+                    fx.tcx,
+                    FullyMonomorphizedLayoutCx(fx.tcx)
+                        .fn_abi_of_instance(drop_instance, ty::List::empty()),
+                );
 
                 let arg_value = drop_place.place_ref(
                     fx,

src/abi/pass_mode.rs

@@ -3,7 +3,7 @@
 use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose};
 use rustc_abi::{Reg, RegKind};
 use rustc_target::callconv::{
-    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode,
+    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, Conv, FnAbi, PassMode,
 };
 use smallvec::{SmallVec, smallvec};
 
@@ -302,3 +302,33 @@ pub(super) fn cvalue_for_param<'tcx>(
         }
     }
 }
+
+pub(crate) fn adjust_fn_abi_for_rust_abi_mistakes<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
+) -> &'tcx FnAbi<'tcx, Ty<'tcx>> {
+    if fn_abi.conv != Conv::Rust {
+        // Non-Rust ABI's should be correctly implemented.
+        return fn_abi;
+    }
+
+    if !tcx.sess.target.is_like_windows && tcx.sess.target.arch != "s390x" {
+        // Out of the targets cg_clif supports, only Windows and s390x don't have two return
+        // registers.
+        return fn_abi;
+    }
+
+    match fn_abi.ret.mode {
+        PassMode::Ignore | PassMode::Cast { .. } | PassMode::Indirect { .. } => return fn_abi,
+        PassMode::Direct(_) => {
+            if fn_abi.ret.layout.size <= tcx.data_layout.pointer_size {
+                return fn_abi;
+            }
+        }
+        PassMode::Pair(..) => {}
+    }
+
+    let mut fn_abi = fn_abi.clone();
+    fn_abi.ret.make_indirect();
+    tcx.arena.alloc(fn_abi)
+}

src/base.rs

@@ -105,7 +105,10 @@ pub(crate) fn codegen_fn<'tcx>(
     let block_map: IndexVec<BasicBlock, Block> =
         (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect();
 
-    let fn_abi = FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty());
+    let fn_abi = adjust_fn_abi_for_rust_abi_mistakes(
+        tcx,
+        FullyMonomorphizedLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty()),
+    );
 
     // Make FunctionCx
     let target_config = module.target_config();
@@ -186,6 +189,7 @@ pub(crate) fn compile_fn(
     context.clear();
     context.func = codegened_func.func;
 
+    // FIXME run this code when define_function returns an error
     #[cfg(any())] // This is never true
     let _clif_guard = {
         use std::fmt::Write;

src/lib.rs

@@ -294,16 +294,6 @@ fn build_isa(sess: &Session) -> Arc<dyn TargetIsa + 'static> {
         }
     }
 
-    if let target_lexicon::OperatingSystem::Windows = target_triple.operating_system {
-        // FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510
-        flags_builder.enable("enable_multi_ret_implicit_sret").unwrap();
-    }
-
-    if let target_lexicon::Architecture::S390x = target_triple.architecture {
-        // FIXME remove dependency on this from the Rust ABI. cc bytecodealliance/wasmtime#9510
-        flags_builder.enable("enable_multi_ret_implicit_sret").unwrap();
-    }
-
     if let target_lexicon::Architecture::Aarch64(_)
     | target_lexicon::Architecture::Riscv64(_)
     | target_lexicon::Architecture::X86_64 = target_triple.architecture