Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement extensible syscall interface for wasm #47102

Merged
merged 2 commits into from
Feb 2, 2018
+365 −198
Merged

Implement extensible syscall interface for wasm #47102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
36695a3
Implement extensible syscall interface for wasm
Diggsey Dec 31, 2017
0e6601f
Add wasm_syscall feature to build system
Diggsey Jan 11, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions config.toml.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,11 @@
# bootstrap)
#codegen-backends = ["llvm"]

# Flag indicating whether `libstd` calls an imported function to hande basic IO
# when targetting WebAssembly. Enable this to debug tests for the `wasm32-unknown-unknown`
# target, as without this option the test output will not be captured.
#wasm-syscall = false

# =============================================================================
# Options for specific targets
#
Expand Down
3 changes: 3 additions & 0 deletions src/bootstrap/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ pub struct Config {
pub debug_jemalloc: bool,
pub use_jemalloc: bool,
pub backtrace: bool, // support for RUST_BACKTRACE
pub wasm_syscall: bool,

// misc
pub low_priority: bool,
Expand Down Expand Up @@ -282,6 +283,7 @@ struct Rust {
test_miri: Option<bool>,
save_toolstates: Option<String>,
codegen_backends: Option<Vec<String>>,
wasm_syscall: Option<bool>,
}

/// TOML representation of how each build target is configured.
Expand Down Expand Up @@ -463,6 +465,7 @@ impl Config {
set(&mut config.rust_dist_src, rust.dist_src);
set(&mut config.quiet_tests, rust.quiet_tests);
set(&mut config.test_miri, rust.test_miri);
set(&mut config.wasm_syscall, rust.wasm_syscall);
config.rustc_parallel_queries = rust.experimental_parallel_queries.unwrap_or(false);
config.rustc_default_linker = rust.default_linker.clone();
config.musl_root = rust.musl_root.clone().map(PathBuf::from);
Expand Down
3 changes: 3 additions & 0 deletions src/bootstrap/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,9 @@ impl Build {
if self.config.profiler {
features.push_str(" profiler");
}
if self.config.wasm_syscall {
features.push_str(" wasm_syscall");
}
features
}

Expand Down
8 changes: 8 additions & 0 deletions src/bootstrap/test.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1286,6 +1286,14 @@ impl Step for Crate {
cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target)),
build.config.nodejs.as_ref().expect("nodejs not configured"));
} else if target.starts_with("wasm32") {
// Warn about running tests without the `wasm_syscall` feature enabled.
// The javascript shim implements the syscall interface so that test
// output can be correctly reported.
if !build.config.wasm_syscall {
println!("Libstd was built without `wasm_syscall` feature enabled: \
test output may not be visible.");
}

// On the wasm32-unknown-unknown target we're using LTO which is
// incompatible with `-C prefer-dynamic`, so disable that here
cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
Expand Down
149 changes: 84 additions & 65 deletions src/etc/wasm32-shim.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,76 @@ let m = new WebAssembly.Module(buffer);

let memory = null;

function viewstruct(data, fields) {
return new Uint32Array(memory.buffer).subarray(data/4, data/4 + fields);
}

function copystr(a, b) {
if (memory === null) {
return null
}
let view = new Uint8Array(memory.buffer).slice(a, a + b);
let view = new Uint8Array(memory.buffer).subarray(a, a + b);
return String.fromCharCode.apply(null, view);
}

function syscall_write([fd, ptr, len]) {
let s = copystr(ptr, len);
switch (fd) {
case 1: process.stdout.write(s); break;
case 2: process.stderr.write(s); break;
}
}

function syscall_exit([code]) {
process.exit(code);
}

function syscall_args(params) {
let [ptr, len] = params;

// Calculate total required buffer size
let totalLen = -1;
for (let i = 2; i < process.argv.length; ++i) {
totalLen += Buffer.byteLength(process.argv[i]) + 1;
}
if (totalLen < 0) { totalLen = 0; }
params[2] = totalLen;

// If buffer is large enough, copy data
if (len >= totalLen) {
let view = new Uint8Array(memory.buffer);
for (let i = 2; i < process.argv.length; ++i) {
let value = process.argv[i];
Buffer.from(value).copy(view, ptr);
ptr += Buffer.byteLength(process.argv[i]) + 1;
}
}
}

function syscall_getenv(params) {
let [keyPtr, keyLen, valuePtr, valueLen] = params;

let key = copystr(keyPtr, keyLen);
let value = process.env[key];

if (value == null) {
params[4] = 0xFFFFFFFF;
} else {
let view = new Uint8Array(memory.buffer);
let totalLen = Buffer.byteLength(value);
params[4] = totalLen;
if (valueLen >= totalLen) {
Buffer.from(value).copy(view, valuePtr);
}
}
}

function syscall_time(params) {
let t = Date.now();
let secs = Math.floor(t / 1000);
let millis = t % 1000;
params[1] = Math.floor(secs / 0x100000000);
params[2] = secs % 0x100000000;
params[3] = Math.floor(millis * 1000000);
}

let imports = {};
imports.env = {
// These are generated by LLVM itself for various intrinsic calls. Hopefully
Expand All @@ -48,68 +110,25 @@ imports.env = {
log10: Math.log10,
log10f: Math.log10,

// These are called in src/libstd/sys/wasm/stdio.rs and are used when
// debugging is enabled.
rust_wasm_write_stdout: function(a, b) {
let s = copystr(a, b);
if (s !== null) {
process.stdout.write(s);
}
},
rust_wasm_write_stderr: function(a, b) {
let s = copystr(a, b);
if (s !== null) {
process.stderr.write(s);
}
},

// These are called in src/libstd/sys/wasm/args.rs and are used when
// debugging is enabled.
rust_wasm_args_count: function() {
if (memory === null)
return 0;
return process.argv.length - 2;
},
rust_wasm_args_arg_size: function(i) {
return Buffer.byteLength(process.argv[i + 2]);
},
rust_wasm_args_arg_fill: function(idx, ptr) {
let arg = process.argv[idx + 2];
let view = new Uint8Array(memory.buffer);
Buffer.from(arg).copy(view, ptr);
},

// These are called in src/libstd/sys/wasm/os.rs and are used when
// debugging is enabled.
rust_wasm_getenv_len: function(a, b) {
let key = copystr(a, b);
if (key === null) {
return -1;
rust_wasm_syscall: function(index, data) {
switch (index) {
case 1: syscall_write(viewstruct(data, 3)); return true;
case 2: syscall_exit(viewstruct(data, 1)); return true;
case 3: syscall_args(viewstruct(data, 3)); return true;
case 4: syscall_getenv(viewstruct(data, 5)); return true;
case 6: syscall_time(viewstruct(data, 4)); return true;
default:
console.log("Unsupported syscall: " + index);
return false;
}
if (!(key in process.env)) {
return -1;
}
return Buffer.byteLength(process.env[key]);
},
rust_wasm_getenv_data: function(a, b, ptr) {
let key = copystr(a, b);
let value = process.env[key];
let view = new Uint8Array(memory.buffer);
Buffer.from(value).copy(view, ptr);
},
};

let module_imports = WebAssembly.Module.imports(m);

for (var i = 0; i < module_imports.length; i++) {
let imp = module_imports[i];
if (imp.module != 'env') {
continue
}
if (imp.name == 'memory' && imp.kind == 'memory') {
memory = new WebAssembly.Memory({initial: 20});
imports.env.memory = memory;
}
}
};

let instance = new WebAssembly.Instance(m, imports);
memory = instance.exports.memory;
try {
instance.exports.main();
} catch (e) {
console.error(e);
process.exit(101);
}
6 changes: 2 additions & 4 deletions src/librustc_trans/back/write.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -824,9 +824,7 @@ fn binaryen_assemble(cgcx: &CodegenContext,
if cgcx.debuginfo != config::NoDebugInfo {
options.debuginfo(true);
}
if cgcx.crate_types.contains(&config::CrateTypeExecutable) {
options.start("main");
}

options.stack(1024 * 1024);
options.import_memory(cgcx.wasm_import_memory);
let assembled = input.and_then(|input| {
Expand Down Expand Up @@ -1452,7 +1450,7 @@ fn start_executing_work(tcx: TyCtxt,
target_pointer_width: tcx.sess.target.target.target_pointer_width.clone(),
binaryen_linker: tcx.sess.linker_flavor() == LinkerFlavor::Binaryen,
debuginfo: tcx.sess.opts.debuginfo,
wasm_import_memory: wasm_import_memory,
wasm_import_memory,
assembler_cmd,
};

Expand Down
1 change: 1 addition & 0 deletions src/libstd/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,4 @@ jemalloc = ["alloc_jemalloc"]
force_alloc_system = []
panic-unwind = ["panic_unwind"]
profiler = ["profiler_builtins"]
wasm_syscall = []
Copy link
Member

@koute koute Jan 5, 2018 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So I guess this means the std will have to be recompiled to use this? ):

To be honest I would really prefer another way of opting into this than having to recompile the whole std. I really don't want to have to add support for std recompilation to cargo-web on top of the gazzilion of hoops I already have to go through to get something usable. ):

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like it will be fairly easy to use Xargo to build libstd with the additional feature, but I don't now how that will inter-operate with cargo-web.

If you find an alternative way to dynamically configure whether or not libstd imports a symbol, then please suggest it, but the way this is done currently for things like the allocator/panic selection is a hack, and @alexcrichton was reluctant to build on this hack for something specific to wasm.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you find an alternative way to dynamically configure whether or not libstd imports a symbol, then please suggest it, but the way this is done currently for things like the allocator/panic selection is a hack, and @alexcrichton was reluctant to build on this hack for something specific to wasm.

As far as I understand the appoach used for allocators (etc.) is something like this in a nutshell: in std you put an extern for a function, and then at link time you have two crates which implement that symbol and you link to one of them based on what the user wants. Correct? And this is the approach you wanted to originally take (but with only one crate instead of two, to force the generation of an import.)

To me personally this doesn't really feel like a hack, and seems like the simplest way to do it. Anything I can think of is going to be more complex than that, so I guess if Alex is against it then that's it. ):

So if we're going to oficially require a recompilation of the std here's a request from me - @alexcrichton could we at least define the empty syscall wrapper inside of std like this?

Before:

#[cfg(not(wasm_syscall))]
unsafe fn syscall<T>(_index: SysCallIndex, _data: &mut T) -> bool {
    false
}

After:

#[cfg(not(wasm_syscall))]
#[allow(private_no_mangle_fns)]
#[no_mangle]
#[inline(never)]
unsafe fn __rust_wasm_syscall(_index: SysCallIndex, _data: *mut c_void) -> bool {
    asm!(""); // To prevent inlining since #[inline(never)] in not enough on the wasm32-unknown-unknown target.
    false
}

Basically, make it not inlineable and give it a unique name. This puts zero maintenance burden on Rust developers and will allow me to trivially support this in cargo-web without having to recompile the std.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My personal feeling is that we should explicitly allow people to compile their own wasm stds easily - this PR is deliberately minimal and it's not hard to imagine people wanting to hook up additional bits and pieces (e.g. FS calls to a virtual filesystem in JS, tcp sockets to websockets). But to do this we need to make it dead easy to compile these different stds for different use-cases.

I'd personally prefer all the changes to the shim and to libstd to end up in a third party repository rather than being landed here so that it will (as I put it in my previous comment) set the scene for other people to get involved.

I am looking at Xargo at the moment and will continue tomorrow - potentially this shouldn't require much extra effort from cargo web, since the heavy lifting should be done by Xargo.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Basically, make it not inlineable and give it a unique name. This puts zero maintenance burden on Rust developers and will allow me to trivially support this in cargo-web without having to recompile the std.

Can you elaborate what this would do/how it would help you? If nothing else, the answer should probably be in a comment for the function :)

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My personal feeling is that we should explicitly allow people to compile their own wasm stds easily

I wouldn't really mind that much if it was oficially supported by cargo/rust (e.g. you could just put the std in your [dependencies] section), but unfortunately it's not yet.

Can you elaborate what this would do/how it would help you? If nothing else, the answer should probably be in a comment for the function :)

Sure. Currently to support the js! macro from stdweb I'm already doing a bunch of transformations on the WebAssembly bytecode which Rust generates. One of the things I can do with my current infrastructure is to freely remap the functions as I see fit and modify the import tables, so if I'll be able to identify which exact function in the bytecode is the syscall wrapper then I can just replace it with an import in something like ~15 lines of code. Rust itself can keep the status quo. Everyone's happy.

38 changes: 5 additions & 33 deletions src/libstd/sys/wasm/args.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@

use ffi::OsString;
use marker::PhantomData;
use mem;
use vec;
use sys::ArgsSysCall;

pub unsafe fn init(_argc: isize, _argv: *const *const u8) {
// On wasm these should always be null, so there's nothing for us to do here
Expand All @@ -21,38 +21,10 @@ pub unsafe fn cleanup() {
}

pub fn args() -> Args {
// When the runtime debugging is enabled we'll link to some extra runtime
// functions to actually implement this. These are for now just implemented
// in a node.js script but they're off by default as they're sort of weird
// in a web-wasm world.
if !super::DEBUG {
return Args {
iter: Vec::new().into_iter(),
_dont_send_or_sync_me: PhantomData,
}
}

// You'll find the definitions of these in `src/etc/wasm32-shim.js`. These
// are just meant for debugging and should not be relied on.
extern {
fn rust_wasm_args_count() -> usize;
fn rust_wasm_args_arg_size(a: usize) -> usize;
fn rust_wasm_args_arg_fill(a: usize, ptr: *mut u8);
}

unsafe {
let cnt = rust_wasm_args_count();
let mut v = Vec::with_capacity(cnt);
for i in 0..cnt {
let n = rust_wasm_args_arg_size(i);
let mut data = vec![0; n];
rust_wasm_args_arg_fill(i, data.as_mut_ptr());
v.push(mem::transmute::<Vec<u8>, OsString>(data));
}
Args {
iter: v.into_iter(),
_dont_send_or_sync_me: PhantomData,
}
let v = ArgsSysCall::perform();
Args {
iter: v.into_iter(),
_dont_send_or_sync_me: PhantomData,
}
}

Expand Down
Loading