Support multiple workers in a single Miniflare instance (#7251)

.changeset/gold-pandas-yell.md

@@ -0,0 +1,5 @@
+---
+"wrangler": minor
+---
+
+Improve Wrangler's multiworker support to allow running multiple workers at once with one command. To try it out, pass multiple `-c` flags to Wrangler: i.e. `wrangler dev -c wrangler.toml -c ../other-worker/wrangler.toml`. The first config will be treated as the _primary_ worker and will be exposed over HTTP as usual (localhost:8787) while the rest will be treated as _secondary_ and will only be accessible via a service binding from the primary worker. Notably, these workers all run in the same runtime instance, which should improve reliability of multiworker dev and fix some bugs (RPC to cross worker Durable Objects, for instance).

fixtures/interactive-dev-tests/src/worker-a.mjs

@@ -0,0 +1,7 @@
+export default {
+	async fetch(req, env) {
+		return new Response(
+			"hello from a & " + (await env.WORKER.fetch(req).then((r) => r.text()))
+		);
+	},
+};

fixtures/interactive-dev-tests/src/worker-b.mjs

@@ -0,0 +1,5 @@
+export default {
+	async fetch() {
+		return new Response("hello from b");
+	},
+};

fixtures/interactive-dev-tests/tests/index.test.ts

@@ -229,6 +229,34 @@ describe.each(devScripts)("wrangler $args", ({ args, expectedBody }) => {
 	});
 });
 
+it.each(exitKeys)("multiworker cleanly exits with $name", async ({ key }) => {
+	const beginProcesses = getStartedWorkerdProcesses();
+
+	const wrangler = await startWranglerDev([
+		"dev",
+		"-c",
+		"wrangler.a.toml",
+		"-c",
+		"wrangler.b.toml",
+	]);
+	const duringProcesses = getStartedWorkerdProcesses();
+
+	// Check dev server working correctly
+	const res = await fetch(wrangler.url);
+	expect((await res.text()).trim()).toBe("hello from a & hello from b");
+
+	// Check key cleanly exits dev server
+	wrangler.pty.write(key);
+	expect(await wrangler.exitPromise).toBe(0);
+	const endProcesses = getStartedWorkerdProcesses();
+
+	// Check no hanging workerd processes
+	if (process.platform !== "win32" || windowsProcessCwdSupported) {
+		expect(beginProcesses.length).toBe(endProcesses.length);
+		expect(duringProcesses.length).toBeGreaterThan(beginProcesses.length);
+	}
+});
+
 it.runIf(RUN_IF && nodePtySupported)(
 	"hotkeys should be unregistered when the initial build fails",
 	async () => {

fixtures/interactive-dev-tests/wrangler.a.toml

@@ -0,0 +1,7 @@
+name = "a"
+main = "src/worker-a.mjs"
+compatibility_date = "2023-12-01"
+
+[[services]]
+binding = "WORKER"
+service = "b"

fixtures/interactive-dev-tests/wrangler.b.toml

@@ -0,0 +1,3 @@
+name = "b"
+main = "src/worker-b.mjs"
+compatibility_date = "2023-12-01"

packages/wrangler/e2e/dev.test.ts

@@ -1,5 +1,4 @@
 import assert from "node:assert";
-import childProcess from "node:child_process";
 import { existsSync } from "node:fs";
 import * as nodeNet from "node:net";
 import { setTimeout } from "node:timers/promises";
@@ -11,6 +10,7 @@ import { fetchText } from "./helpers/fetch-text";
 import { fetchWithETag } from "./helpers/fetch-with-etag";
 import { generateResourceName } from "./helpers/generate-resource-name";
 import { retry } from "./helpers/retry";
+import { getStartedWorkerdProcesses } from "./helpers/workerd-processes";
 
 /**
  * We use the same workerName for all of the tests in this suite in hopes of reducing flakes.
@@ -210,35 +210,6 @@ describe.each([{ cmd: "wrangler dev" }, { cmd: "wrangler dev --remote" }])(
 	}
 );
 
-interface Process {
-	pid: string;
-	cmd: string;
-}
-
-function getProcesses(): Process[] {
-	return childProcess
-		.execSync("ps -e | awk '{print $1,$4}'", { encoding: "utf8" })
-		.trim()
-		.split("\n")
-		.map((line) => {
-			const [pid, cmd] = line.split(" ");
-			return { pid, cmd };
-		});
-}
-
-function getProcessCwd(pid: string | number) {
-	return childProcess
-		.execSync(`lsof -p ${pid} | awk '$4=="cwd" {print $9}'`, {
-			encoding: "utf8",
-		})
-		.trim();
-}
-function getStartedWorkerdProcesses(cwd: string): Process[] {
-	return getProcesses()
-		.filter(({ cmd }) => cmd.includes("workerd"))
-		.filter((c) => getProcessCwd(c.pid).includes(cwd));
-}
-
 // This fails on Windows because of https://github.com/cloudflare/workerd/issues/1664
 it.runIf(process.platform !== "win32")(
 	`leaves no orphaned workerd processes with port conflict`,

packages/wrangler/e2e/helpers/command.ts

@@ -169,6 +169,10 @@ export class LongLivedCommand {
 			});
 		});
 	}
+
+	async signal(signal: NodeJS.Signals) {
+		return this.commandProcess.kill(signal);
+	}
 }
 
 interface TimedOutError extends Error {

packages/wrangler/e2e/helpers/workerd-processes.ts

@@ -0,0 +1,30 @@
+import childProcess from "node:child_process";
+
+interface Process {
+	pid: string;
+	cmd: string;
+}
+
+function getProcesses(): Process[] {
+	return childProcess
+		.execSync("ps -e | awk '{print $1,$4}'", { encoding: "utf8" })
+		.trim()
+		.split("\n")
+		.map((line) => {
+			const [pid, cmd] = line.split(" ");
+			return { pid, cmd };
+		});
+}
+
+function getProcessCwd(pid: string | number) {
+	return childProcess
+		.execSync(`lsof -p ${pid} | awk '$4=="cwd" {print $9}'`, {
+			encoding: "utf8",
+		})
+		.trim();
+}
+export function getStartedWorkerdProcesses(cwd: string): Process[] {
+	return getProcesses()
+		.filter(({ cmd }) => cmd.includes("workerd"))
+		.filter((c) => getProcessCwd(c.pid).includes(cwd));
+}