Merge #709 #865 #1042

709: new feature flag: `deterministic` r=MarkMcCaskey a=YaronWittenstein

The motivation for the PR is for introducing a new feature flag called `deterministic`.

When `deterministic` will be enabled (turned-off by default) it'll guarantee deterministic
execution of wasm programs across different hardware/circumstances.

This is critical for Blockchain projects that require execution to be deterministic
in order to reach a consensus of the state transition of each smart-contract transaction.

865: adding tests for `state_creator` of `import_object` r=MarkMcCaskey a=YaronWittenstein

Part of the PR #807 changes was adding support for shared import objects between threads.

https://github.com/wasmerio/wasmer/pull/807/files#diff-d20cb4c5a883566b85be4cc046f45aa9R49

I've added tests/examples on how to create an `import object` with a state_creator
(function or closure)

1042: Make regression test work in release builds too. r=nlewycky a=nlewycky

Fix this regression test to detect the bug it was looking for in release builds too.

This bug triggered an assertion failure in debug, and by examining the pre-opt IR, we can check for the bug in release mode too.


Co-authored-by: Yaron Wittenstein <[email protected]>
Co-authored-by: Yaron Wittenstein <[email protected]>
Co-authored-by: Mark McCaskey <[email protected]>
Co-authored-by: Mark McCaskey <[email protected]>
Co-authored-by: Nick Lewycky <[email protected]>

Makefile

@@ -192,6 +192,12 @@ check: check-bench
 	# as default, and test a minimal set of features with only one backend
 	# at a time.
 	cargo check --manifest-path lib/runtime/Cargo.toml
+	# Check some of the cases where deterministic execution could matter
+	cargo check --manifest-path lib/runtime/Cargo.toml --features "deterministic-execution"
+	cargo check --manifest-path lib/runtime/Cargo.toml --no-default-features \
+		--features=default-backend-singlepass,deterministic-execution
+	cargo check --manifest-path lib/runtime/Cargo.toml --no-default-features \
+		--features=default-backend-llvm,deterministic-execution
 	cargo check --release --manifest-path lib/runtime/Cargo.toml
 
 	$(RUNTIME_CHECK) \

lib/llvm-backend-tests/tests/compile.rs

@@ -1,40 +1,68 @@
+use wasmer_llvm_backend::{InkwellMemoryBuffer, InkwellModule, LLVMBackendConfig, LLVMCallbacks};
 use wasmer_llvm_backend_tests::{get_compiler, wat2wasm};
-use wasmer_runtime::imports;
-use wasmer_runtime_core::compile_with;
+use wasmer_runtime::{imports, CompilerConfig};
+use wasmer_runtime_core::{backend::BackendCompilerConfig, compile_with, compile_with_config};
+
+use std::cell::RefCell;
+use std::rc::Rc;
 
 #[test]
 fn crash_return_with_float_on_stack() {
     const MODULE: &str = r#"
 (module
-  (type (;0;) (func))
-  (type (;1;) (func (param f64) (result f64)))
+  (type (func))
+  (type (func (param f64) (result f64)))
   (func $_start (type 0))
   (func $fmod (type 1) (param f64) (result f64)
     local.get 0
-    f64.const 0x0p+0 (;=0;)
+    f64.const 0x0p+0
     f64.mul
-    return)
-)
+    return))
 "#;
     let wasm_binary = wat2wasm(MODULE.as_bytes()).expect("WAST not valid or malformed");
     let module = compile_with(&wasm_binary, &get_compiler()).unwrap();
     module.instantiate(&imports! {}).unwrap();
 }
 
+#[derive(Debug, Default)]
+pub struct RecordPreOptIR {
+    preopt_ir: String,
+}
+
+impl LLVMCallbacks for RecordPreOptIR {
+    fn preopt_ir_callback(&mut self, module: &InkwellModule) {
+        self.preopt_ir = module.print_to_string().to_string();
+    }
+}
+
 #[test]
 fn crash_select_with_mismatched_pending() {
-    const MODULE: &str = r#"
+    const WAT: &str = r#"
  (module
-  (func (param f64)
-    f64.const 0x0p+0 (;=0;)
+  (func (param f64) (result f64)
+    f64.const 0x0p+0
     local.get 0
     f64.add
-    f64.const 0x0p+0 (;=0;)
+    f64.const 0x0p+0
     i32.const 0
-    select
-    drop))
+    select))
 "#;
-    let wasm_binary = wat2wasm(MODULE.as_bytes()).expect("WAST not valid or malformed");
-    let module = compile_with(&wasm_binary, &get_compiler()).unwrap();
+    let record_pre_opt_ir = Rc::new(RefCell::new(RecordPreOptIR::default()));
+    let compiler_config = CompilerConfig {
+        backend_specific_config: Some(BackendCompilerConfig(Box::new(LLVMBackendConfig {
+            callbacks: Some(record_pre_opt_ir.clone()),
+        }))),
+        ..Default::default()
+    };
+    let wasm_binary = wat2wasm(WAT.as_bytes()).expect("WAST not valid or malformed");
+    let module = compile_with_config(&wasm_binary, &get_compiler(), compiler_config).unwrap();
     module.instantiate(&imports! {}).unwrap();
+    const LLVM: &str = r#"
+  %s3 = fadd double 0.000000e+00, %s2
+  %nan = fcmp uno double %s3, 0.000000e+00
+  %2 = select i1 %nan, double 0x7FF8000000000000, double %s3
+  %s5 = select i1 false, double %2, double 0.000000e+00
+  br label %return
+"#;
+    assert!(&record_pre_opt_ir.borrow().preopt_ir.contains(LLVM));
 }

lib/llvm-backend/src/lib.rs

@@ -37,9 +37,9 @@ pub type InkwellModule<'ctx> = inkwell::module::Module<'ctx>;
 pub type InkwellMemoryBuffer = inkwell::memory_buffer::MemoryBuffer;
 
 pub trait LLVMCallbacks: std::any::Any + 'static {
-    fn preopt_ir_callback(&mut self, module: &InkwellModule);
-    fn postopt_ir_callback(&mut self, module: &InkwellModule);
-    fn obj_memory_buffer_callback(&mut self, memory_buffer: &InkwellMemoryBuffer);
+    fn preopt_ir_callback(&mut self, _module: &InkwellModule) {}
+    fn postopt_ir_callback(&mut self, _module: &InkwellModule) {}
+    fn obj_memory_buffer_callback(&mut self, _memory_buffer: &InkwellMemoryBuffer) {}
 }
 
 pub struct LLVMBackendConfig {

lib/runtime-core/Cargo.toml

@@ -57,3 +57,4 @@ trace = ["debug"]
 "backend-singlepass" = []
 "backend-llvm" = []
 managed = []
+deterministic-execution = ["wasmparser/deterministic"]

lib/runtime-core/src/codegen.rs

@@ -186,6 +186,9 @@ pub fn validating_parser_config(features: &Features) -> wasmparser::ValidatingPa
             enable_simd: features.simd,
             enable_bulk_memory: false,
             enable_multi_value: false,
+
+            #[cfg(feature = "deterministic-execution")]
+            deterministic_only: true,
         },
     }
 }

lib/runtime-core/src/import.rs

@@ -273,6 +273,54 @@ mod test {
     use crate::export::Export;
     use crate::global::Global;
     use crate::types::Value;
+    use std::ffi::c_void;
+    use std::sync::Arc;
+
+    struct Data {
+        inner: *const u32,
+    }
+
+    unsafe impl Send for Data {}
+    unsafe impl Sync for Data {}
+
+    fn dummy_state_creator(data: Arc<Data>) -> (*mut c_void, fn(*mut c_void)) {
+        let data: *mut Data = Arc::into_raw(data) as _;
+
+        (data as _, |_| {})
+    }
+
+    #[test]
+    fn state_creator_fn() {
+        let ptr = &0xAABBCCDDu32 as *const u32;
+        let data = Arc::new(Data { inner: ptr });
+
+        let imports = imports! {
+            move || dummy_state_creator(Arc::clone(&data)),
+        };
+
+        let (state, _dtor) = imports.call_state_creator().unwrap();
+        let data: &mut Data = unsafe { &mut *(state as *mut Data) };
+
+        assert_eq!(ptr, data.inner);
+    }
+
+    #[test]
+    fn state_creator_closure() {
+        let ptr = &0xAABBCCDDu32 as *const u32;
+        let data = Arc::new(Data { inner: ptr });
+
+        let imports = imports! {
+            move || {
+                let data: *mut Data = Arc::into_raw(Arc::clone(&data)) as _;
+                (data as _, |_| {})
+            },
+        };
+
+        let (state, _dtor) = imports.call_state_creator().unwrap();
+        let data: &mut Data = unsafe { &mut *(state as *mut Data) };
+
+        assert_eq!(ptr, data.inner);
+    }
 
     #[test]
     fn extending_works() {

lib/runtime-core/src/lib.rs

@@ -159,6 +159,9 @@ pub fn validate_and_report_errors_with_features(
             enable_multi_value: false,
             enable_reference_types: false,
             enable_threads: features.threads,
+
+            #[cfg(feature = "deterministic-execution")]
+            deterministic_only: true,
         },
     };
     let mut parser = wasmparser::ValidatingParser::new(wasm, Some(config));

lib/runtime/Cargo.toml

@@ -44,6 +44,7 @@ singlepass = ["wasmer-singlepass-backend"]
 default-backend-singlepass = ["singlepass"]
 default-backend-llvm = ["llvm"]
 default-backend-cranelift = ["cranelift"]
+deterministic-execution = ["wasmer-singlepass-backend/deterministic-execution", "wasmer-runtime-core/deterministic-execution"]
 
 [[bench]]
 name = "nginx"

lib/runtime/src/lib.rs

@@ -71,7 +71,7 @@
 //!     let value = add_one.call(42)?;
 //!
 //!     assert_eq!(value, 43);
-//!     
+//!
 //!     Ok(())
 //! }
 //! ```
@@ -246,14 +246,26 @@ pub fn compiler_for_backend(backend: Backend) -> Option<Box<dyn Compiler>> {
         #[cfg(feature = "cranelift")]
         Backend::Cranelift => Some(Box::new(wasmer_clif_backend::CraneliftCompiler::new())),
 
-        #[cfg(feature = "singlepass")]
+        #[cfg(any(feature = "singlepass"))]
         Backend::Singlepass => Some(Box::new(
             wasmer_singlepass_backend::SinglePassCompiler::new(),
         )),
 
         #[cfg(feature = "llvm")]
         Backend::LLVM => Some(Box::new(wasmer_llvm_backend::LLVMCompiler::new())),
 
+        Backend::Auto => {
+            #[cfg(feature = "default-backend-singlepass")]
+            return Some(Box::new(
+                wasmer_singlepass_backend::SinglePassCompiler::new(),
+            ));
+            #[cfg(feature = "default-backend-cranelift")]
+            return Some(Box::new(wasmer_clif_backend::CraneliftCompiler::new()));
+            #[cfg(feature = "default-backend-llvm")]
+            return Some(Box::new(wasmer_llvm_backend::LLVMCompiler::new()));
+        }
+
+        #[cfg(not(all(feature = "llvm", feature = "singlepass", feature = "cranelift")))]
         _ => None,
     }
 }

lib/singlepass-backend/Cargo.toml

@@ -22,3 +22,7 @@ smallvec = "0.6"
 serde = "1.0"
 serde_derive = "1.0"
 bincode = "1.2"
+
+[features]
+default = []
+deterministic-execution = ["wasmer-runtime-core/deterministic-execution"]