vm: add experimental NodeRealm implementation

LICENSE

@@ -2186,3 +2186,28 @@ The externally maintained libraries used by Node.js are:
     NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
     SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   """
+
+- synchronous-worker, located at lib/internal/vm/node_realm.js, is licensed as follows:
+  """
+    The MIT License (MIT)
+
+    Copyright (c) 2020 Anna Henningsen
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE.
+  """

doc/api/cli.md

@@ -530,6 +530,22 @@ changes:
 Specify the `module` of a custom experimental [ECMAScript module loader][].
 `module` may be any string accepted as an [`import` specifier][].
 
+### `--experimental-node-realm`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Enable experimental support for `vm.NodeRealm`.
+
+### `--no-experimental-node-realm`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Disable experimental support for `vm.NodeRealm`.
+
 ### `--experimental-network-imports`
 
 <!-- YAML
@@ -2113,6 +2129,7 @@ Node.js options that are allowed are:
 * `--experimental-import-meta-resolve`
 * `--experimental-json-modules`
 * `--experimental-loader`
+* `--experimental-node-realm`
 * `--experimental-modules`
 * `--experimental-network-imports`
 * `--experimental-permission`

doc/api/errors.md

@@ -3536,6 +3536,17 @@ removed:
 
 The linker function returned a module for which linking has failed.
 
+<a id="ERR_VM_NODE_REALM_INVALID_PARENT"></a>
+
+### `ERR_VM_NODE_REALM_INVALID_PARENT`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+The `createImport()` function was passed a value that was neither
+a string nor a `URL`.
+
 <a id="ERR_WORKER_UNSUPPORTED_EXTENSION"></a>
 
 ### `ERR_WORKER_UNSUPPORTED_EXTENSION`

doc/api/vm.md

@@ -1573,6 +1573,84 @@ inside a `vm.Context`, functions passed to them will be added to global queues,
 which are shared by all contexts. Therefore, callbacks passed to those functions
 are not controllable through the timeout either.
 
+### Class: `NodeRealm`
+
+> Stability: 1 - Experimental. Use `--experimental-node-realm` CLI flag to
+> enable this feature.
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Extends: {EventEmitter}
+
+A `NodeRealm` is effectively a Node.js environment that runs within the
+same thread. It similar to a [ShadowRealm][], but with a few main differences:
+
+* `NodeRealm` supports loading both CommonJS and ES modules.
+* Full interoperability between the host realm and the `NodeRealm` instance
+  is allowed.
+* There is a deliberate `stop()` function.
+
+```mjs
+import { NodeRealm } from 'node:vm';
+const nodeRealm = new NodeRealm();
+const { myAsyncFunction } = await nodeRealm.createImport(import.meta.url)('my-module');
+console.log(await myAsyncFunction());
+```
+
+#### `new NodeRealm()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+#### `nodeRealm.stop()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Returns: <Promise>
+
+This will render the inner Node.js instance unusable.
+and is generally comparable to running `process.exit()`.
+
+This method returns a promise that will be resolved when all resources
+associated with this Node.js instance are released. This promise resolves on
+the event loop of the _outer_ Node.js instance.
+
+#### `nodeRealm.createImport(specifier)`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `specifier` {string} A module specifier like './file.js' or 'my-package'
+
+Creates a function that can be used for loading
+modules inside the inner Node.js instance.
+
+#### `nodeRealm.globalThis`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Type: {Object}
+
+Returns a reference to the global object of the inner Node.js instance.
+
+#### `nodeRealm.process`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Type: {Object}
+
+Returns a reference to the `process` object of the inner Node.js instance.
+
 [Cyclic Module Record]: https://tc39.es/ecma262/#sec-cyclic-module-records
 [ECMAScript Module Loader]: esm.md#modules-ecmascript-modules
 [Evaluate() concrete method]: https://tc39.es/ecma262/#sec-moduleevaluation
@@ -1599,3 +1677,4 @@ are not controllable through the timeout either.
 [global object]: https://es5.github.io/#x15.1
 [indirect `eval()` call]: https://es5.github.io/#x10.4.2
 [origin]: https://developer.mozilla.org/en-US/docs/Glossary/Origin
+[ShadowRealm]: https://github.com/tc39/proposal-shadowrealm

doc/node.1

@@ -166,6 +166,9 @@ to use as a custom module loader.
 .It Fl -experimental-network-imports
 Enable experimental support for loading modules using `import` over `https:`.
 .
+.It Fl -experimental-node-realm
+Enable experimental support for vm.NodeRealm.
+.
 .It Fl -experimental-permission
 Enable the experimental permission model.
 .

lib/internal/errors.js

@@ -1706,6 +1706,7 @@ E('ERR_VM_MODULE_LINK_FAILURE', function(message, cause) {
 E('ERR_VM_MODULE_NOT_MODULE',
   'Provided module is not an instance of Module', Error);
 E('ERR_VM_MODULE_STATUS', 'Module status %s', Error);
+E('ERR_VM_NODE_REALM_INVALID_PARENT', 'createImport() must be called with a string or URL; received "%s"', Error);
 E('ERR_WASI_ALREADY_STARTED', 'WASI instance has already started', Error);
 E('ERR_WEBASSEMBLY_RESPONSE', 'WebAssembly response %s', TypeError);
 E('ERR_WORKER_INIT_FAILED', 'Worker initialization failure: %s', Error);

lib/internal/process/pre_execution.js

@@ -67,6 +67,7 @@ function prepareExecution(options) {
   setupInspectorHooks();
   setupWarningHandler();
   setupFetch();
+  setupNodeRealm();
   setupWebCrypto();
   setupCustomEvent();
   setupCodeCoverage();
@@ -267,6 +268,16 @@ function setupFetch() {
   });
 }
 
+function setupNodeRealm() {
+  // Patch the vm module when --experimental-node-realm is on.
+  // Please update the comments in vm.js when this block changes.
+  if (getOptionValue('--experimental-node-realm')) {
+    const NodeRealm = require('internal/vm/node_realm');
+    const vm = require('vm');
+    vm.NodeRealm = NodeRealm;
+  }
+}
+
 // TODO(aduh95): move this to internal/bootstrap/web/* when the CLI flag is
 //               removed.
 function setupWebCrypto() {

lib/internal/vm/node_realm.js

@@ -0,0 +1,133 @@
+'use strict';
+
+// NodeRealm was originally a separate module developed by
+// Anna Henningsen and published separately on npm as the
+// synchronous-worker module under the MIT license. It has been
+// incorporated into Node.js with Anna's permission.
+// See the LICENSE file for LICENSE and copyright attribution.
+
+const {
+  Promise,
+} = primordials;
+
+const {
+  emitExperimentalWarning,
+} = require('internal/util');
+
+const {
+  ERR_VM_NODE_REALM_INVALID_PARENT,
+} = require('internal/errors').codes;
+
+const {
+  NodeRealm: NodeRealmImpl,
+} = internalBinding('contextify');
+
+const { URL } = require('internal/url');
+const EventEmitter = require('events');
+const { setTimeout } = require('timers');
+const { pathToFileURL } = require('url');
+
+let debug = require('internal/util/debuglog').debuglog('noderealm', (fn) => {
+  debug = fn;
+});
+
+class NodeRealm extends EventEmitter {
+  #handle = undefined;
+  #process = undefined;
+  #global = undefined;
+  #stoppedPromise = undefined;
+  #loader = undefined;
+
+  constructor() {
+    super();
+    emitExperimentalWarning('NodeRealm');
+    this.#handle = new NodeRealmImpl();
+    this.#handle.onexit = (code) => {
+      this.stop();
+      this.emit('exit', code);
+    };
+    try {
+      this.#handle.start();
+      this.#handle.load((process, nativeRequire, globalThis) => {
+        this.#process = process;
+        this.#global = globalThis;
+        process.on('uncaughtException', (err) => {
+          if (process.listenerCount('uncaughtException') === 1) {
+            // If we are stopping, silence all errors
+            if (!this.#stoppedPromise) {
+              this.emit('error', err);
+            }
+            process.exit(1);
+          }
+        });
+      });
+
+      const req = this.#handle.internalRequire();
+      this.#loader = req('internal/process/esm_loader').esmLoader;
+    } catch (err) {
+      this.#handle.stop();
+      throw err;
+    }
+  }
+
+  /**
+   * @returns {Promise<void>}
+   */
+  async stop() {
+    // TODO(@mcollina): add support for AbortController, we want to abort this,
+    // or add a timeout.
+    return this.#stoppedPromise ??= new Promise((resolve) => {
+      const tryClosing = () => {
+        const closed = this.#handle.tryCloseAllHandles();
+        debug('closed %d handles', closed);
+        if (closed > 0) {
+          // This is an active wait for the handles to close.
+          // We might want to change this in the future to use a callback,
+          // but at this point it seems like a premature optimization.
+          // We cannot unref() this because we need to shut this down properly.
+          // TODO(@mcollina): refactor to use a close callback
+          setTimeout(tryClosing, 100);
+        } else {
+
+          this.#handle.stop();
+          resolve();
+        }
+      };
+
+      // We use setTimeout instead of setImmediate because it runs in a different
+      // phase of the event loop. This is important because the immediate queue
+      // would crash if the environment it refers to has been already closed.
+      // We cannot unref() this because we need to shut this down properly.
+      setTimeout(tryClosing, 100);
+    });
+  }
+
+  get process() {
+    return this.#process;
+  }
+
+  get globalThis() {
+    return this.#global;
+  }
+
+  /**
+   * @param {string|URL} parentURL
+   */
+  createImport(parentURL) {
+    if (typeof parentURL === 'string') {
+      if (parentURL.indexOf('file://') === 0) {
+        parentURL = new URL(parentURL);
+      } else {
+        parentURL = pathToFileURL(parentURL);
+      }
+    } else if (!(parentURL instanceof URL)) {
+      throw new ERR_VM_NODE_REALM_INVALID_PARENT(parentURL);
+    }
+
+    return (specifiers, importAssertions) => {
+      return this.#loader.import(specifiers, parentURL, importAssertions || {});
+    };
+  }
+}
+
+module.exports = NodeRealm;

lib/vm.js

@@ -343,3 +343,5 @@ module.exports = {
 // The vm module is patched to include vm.Module, vm.SourceTextModule
 // and vm.SyntheticModule in the pre-execution phase when
 // --experimental-vm-modules is on.
+// The vm module is also patched to include vm.NodeRealm in the
+// pre-execution phase when --experimental-node-realm is on.

src/env.cc

@@ -1019,21 +1019,40 @@ void Environment::CleanupHandles() {
 
   RunAndClearNativeImmediates(true /* skip unrefed SetImmediate()s */);
 
-  for (ReqWrapBase* request : req_wrap_queue_)
+  CleanupHandlesNoUvRun();
+
+  while (handle_cleanup_waiting_ != 0 || request_waiting_ != 0 ||
+         !handle_wrap_queue_.IsEmpty()) {
+    uv_run(event_loop(), UV_RUN_ONCE);
+  }
+}
+
+int Environment::CleanupHandlesNoUvRun() {
+  {
+    Mutex::ScopedLock lock(native_immediates_threadsafe_mutex_);
+    task_queues_async_initialized_ = false;
+  }
+
+  int count = 0;
+
+  for (ReqWrapBase* request : req_wrap_queue_) {
+    count++;
     request->Cancel();
+  }
 
-  for (HandleWrap* handle : handle_wrap_queue_)
+  for (HandleWrap* handle : handle_wrap_queue_) {
+    count++;
     handle->Close();
+  }
 
-  for (HandleCleanup& hc : handle_cleanup_queue_)
+  for (HandleCleanup& hc : handle_cleanup_queue_) {
+    count++;
     hc.cb_(this, hc.handle_, hc.arg_);
+  }
+
   handle_cleanup_queue_.clear();
 
-  while (handle_cleanup_waiting_ != 0 ||
-         request_waiting_ != 0 ||
-         !handle_wrap_queue_.IsEmpty()) {
-    uv_run(event_loop(), UV_RUN_ONCE);
-  }
+  return count;
 }
 
 void Environment::StartProfilerIdleNotifier() {