feat: yarn 3 support for native modules via new electron/rebuild comp…

…ilation (#8112)

.changeset/twelve-points-whisper.md

@@ -0,0 +1,5 @@
+---
+"app-builder-lib": minor
+---
+
+feat: implementing electron/rebuild with config option `useLegacyRebuilder` default: `true` to support Yarn 3

.github/actions/pretest/action.yml

@@ -20,6 +20,11 @@ runs:
       with:
         version: ${{ inputs.version }}
 
+    - name: Setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+
     - name: Setup node
       uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
       with:

docs/configuration/configuration.md

@@ -93,6 +93,9 @@ Env file `electron-builder.env` in the current dir ([example](https://github.com
 <p><code id="Configuration-npmRebuild">npmRebuild</code> = <code>true</code> Boolean - Whether to <a href="https://docs.npmjs.com/cli/rebuild">rebuild</a> native dependencies before starting to package the app.</p>
 </li>
 <li>
+<p><code id="Configuration-nativeRebuilder">nativeRebuilder</code> = <code>legacy</code> “legacy” | “sequential” | “parallel” | “undefined” - Use <code>legacy</code> app-builder binary for installing native dependencies, or @electron/rebuild in <code>sequential</code> or <code>parallel</code> compilation modes.</p>
+</li>
+<li>
 <p><code id="Configuration-buildNumber">buildNumber</code> String | “undefined” - The build number. Maps to the <code>--iteration</code> flag for builds using FPM on Linux. If not defined, then it will fallback to <code>BUILD_NUMBER</code> or <code>TRAVIS_BUILD_NUMBER</code> or <code>APPVEYOR_BUILD_NUMBER</code> or <code>CIRCLE_BUILD_NUM</code> or <code>BUILD_BUILDNUMBER</code> or <code>CI_PIPELINE_IID</code> env.</p>
 </li>
 </ul>

packages/app-builder-lib/package.json

@@ -49,6 +49,7 @@
     "@develar/schema-utils": "~2.6.5",
     "@electron/notarize": "2.2.1",
     "@electron/osx-sign": "1.0.5",
+    "@electron/rebuild": "3.6.0",
     "@electron/universal": "1.5.1",
     "@malept/flatpak-bundler": "^0.4.0",
     "@types/fs-extra": "9.0.13",

packages/app-builder-lib/scheme.json

@@ -7463,6 +7463,23 @@
         }
       ]
     },
+    "nativeRebuilder": {
+      "anyOf": [
+        {
+          "enum": [
+            "legacy",
+            "parallel",
+            "sequential"
+          ],
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "default": "legacy",
+      "description": "Use `legacy` app-builder binary for installing native dependencies, or @electron/rebuild in `sequential` or `parallel` compilation modes."
+    },
     "nodeGypRebuild": {
       "default": false,
       "description": "Whether to execute `node-gyp rebuild` before starting to package the app.\n\nDon't [use](https://github.com/electron-userland/electron-builder/issues/683#issuecomment-241214075) [npm](http://electron.atom.io/docs/tutorial/using-native-node-modules/#using-npm) (neither `.npmrc`) for configuring electron headers. Use `electron-builder node-gyp-rebuild` instead.",

packages/app-builder-lib/src/configuration.ts

@@ -131,6 +131,11 @@ export interface Configuration extends PlatformSpecificBuildOptions {
    * @default true
    */
   readonly npmRebuild?: boolean
+  /**
+   * Use `legacy` app-builder binary for installing native dependencies, or @electron/rebuild in `sequential` or `parallel` compilation modes.
+   * @default legacy
+   */
+  readonly nativeRebuilder?: "legacy" | "sequential" | "parallel" | null
 
   /**
    * The build number. Maps to the `--iteration` flag for builds using FPM on Linux.

packages/app-builder-lib/src/electron/electronVersion.ts

@@ -1,3 +1,5 @@
+import { getProjectRootPath } from "@electron/rebuild/lib/search-module"
+
 import { InvalidConfigurationError, log } from "builder-util"
 import { parseXml } from "builder-util-runtime"
 import { httpExecutor } from "builder-util/out/nodeHttpExecutor"
@@ -56,8 +58,15 @@ export async function computeElectronVersion(projectDir: string): Promise<string
     return result
   }
 
-  const metadata = await orNullIfFileNotExist(readJson(path.join(projectDir, "package.json")))
-  const dependency = metadata ? findFromPackageMetadata(metadata) : null
+  const potentialRootDirs = [projectDir, await getProjectRootPath(projectDir)]
+  let dependency: NameAndVersion | null = null
+  for await (const dir of potentialRootDirs) {
+    const metadata = await orNullIfFileNotExist(readJson(path.join(dir, "package.json")))
+    dependency = metadata ? findFromPackageMetadata(metadata) : null
+    if (dependency) {
+      break
+    }
+  }
   if (dependency?.name === "electron-nightly") {
     log.info("You are using a nightly version of electron, be warned that those builds are highly unstable.")
     const feedXml = await httpExecutor.request({

packages/app-builder-lib/src/util/packageMetadata.ts

@@ -70,7 +70,7 @@ export function checkMetadata(metadata: Metadata, devMetadata: any | null, appPa
   const devDependencies = (metadata as any).devDependencies
   if (devDependencies != null && ("electron-rebuild" in devDependencies || "@electron/rebuild" in devDependencies)) {
     log.info(
-      '@electron/rebuild not required if you use electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
+      '@electron/rebuild already used by electron-builder, please consider to remove excess dependency from devDependencies\n\nTo ensure your native dependencies are always matched electron version, simply add script `"postinstall": "electron-builder install-app-deps" to your `package.json`'
     )
   }
 

packages/app-builder-lib/src/util/rebuild/rebuild.ts

@@ -0,0 +1,59 @@
+import * as cp from "child_process"
+import * as path from "path"
+import { RebuildOptions } from "@electron/rebuild"
+import { log } from "builder-util"
+
+export const rebuild = async (options: RebuildOptions): Promise<void> => {
+  const { arch } = options
+  log.info({ arch }, `installing native dependencies`)
+
+  const child = cp.fork(path.resolve(__dirname, "remote-rebuild.js"), [JSON.stringify(options)], {
+    stdio: ["pipe", "pipe", "pipe", "ipc"],
+  })
+
+  let pendingError: Error
+
+  child.stdout?.on("data", chunk => {
+    log.info(chunk.toString())
+  })
+  child.stderr?.on("data", chunk => {
+    log.error(chunk.toString())
+  })
+
+  child.on("message", (message: { msg: string; moduleName: string; err: { message: string; stack: string } }) => {
+    const { moduleName, msg } = message
+    switch (msg) {
+      case "module-found": {
+        log.info({ moduleName, arch }, "preparing")
+        break
+      }
+      case "module-done": {
+        log.info({ moduleName, arch }, "finished")
+        break
+      }
+      case "module-skip": {
+        log.debug?.({ moduleName, arch }, "skipped. set ENV=electron-rebuild to determine why")
+        break
+      }
+      case "rebuild-error": {
+        pendingError = new Error(message.err.message)
+        pendingError.stack = message.err.stack
+        break
+      }
+      case "rebuild-done": {
+        log.info("completed installing native dependencies")
+        break
+      }
+    }
+  })
+
+  await new Promise<void>((resolve, reject) => {
+    child.on("exit", code => {
+      if (code === 0 && !pendingError) {
+        resolve()
+      } else {
+        reject(pendingError || new Error(`Rebuilder failed with exit code: ${code}`))
+      }
+    })
+  })
+}

packages/app-builder-lib/src/util/rebuild/remote-rebuild.ts

@@ -0,0 +1,30 @@
+import { rebuild, RebuildOptions } from "@electron/rebuild"
+
+if (!process.send) {
+  console.error("The remote rebuilder expects to be spawned with an IPC channel")
+  process.exit(1)
+}
+
+const options: RebuildOptions = JSON.parse(process.argv[2])
+
+const rebuilder = rebuild(options)
+
+rebuilder.lifecycle.on("module-found", (moduleName: string) => process.send?.({ msg: "module-found", moduleName }))
+rebuilder.lifecycle.on("module-done", (moduleName: string) => process.send?.({ msg: "module-done", moduleName }))
+rebuilder.lifecycle.on("module-skip", (moduleName: string) => process.send?.({ msg: "module-skip", moduleName }))
+
+rebuilder
+  .then(() => {
+    process.send?.({ msg: "rebuild-done" })
+    return process.exit(0)
+  })
+  .catch(err => {
+    process.send?.({
+      msg: "rebuild-error",
+      err: {
+        message: err.message,
+        stack: err.stack,
+      },
+    })
+    process.exit(0)
+  })

packages/app-builder-lib/src/util/yarn.ts

@@ -4,11 +4,15 @@ import { Lazy } from "lazy-val"
 import { homedir } from "os"
 import * as path from "path"
 import { Configuration } from "../configuration"
-import { executeAppBuilderAndWriteJson } from "./appBuilder"
 import { NodeModuleDirInfo } from "./packageDependencies"
+import * as electronRebuild from "@electron/rebuild"
+import { getProjectRootPath } from "@electron/rebuild/lib/search-module"
+import { rebuild as remoteRebuild } from "./rebuild/rebuild"
+import { executeAppBuilderAndWriteJson } from "./appBuilder"
+import { RebuildMode } from "@electron/rebuild/lib/types"
 
 export async function installOrRebuild(config: Configuration, appDir: string, options: RebuildOptions, forceInstall = false) {
-  const effectiveOptions = {
+  const effectiveOptions: RebuildOptions = {
     buildFromSource: config.buildDependenciesFromSource === true,
     additionalArgs: asArray(config.npmArgs),
     ...options,
@@ -24,9 +28,9 @@ export async function installOrRebuild(config: Configuration, appDir: string, op
   }
 
   if (forceInstall || !isDependenciesInstalled) {
-    await installDependencies(appDir, effectiveOptions)
+    await installDependencies(config, appDir, effectiveOptions)
   } else {
-    await rebuild(appDir, effectiveOptions)
+    await rebuild(config, appDir, effectiveOptions)
   }
 }
 
@@ -83,7 +87,7 @@ function checkYarnBerry() {
   return yarnMajorVersion >= 2
 }
 
-function installDependencies(appDir: string, options: RebuildOptions): Promise<any> {
+async function installDependencies(config: Configuration, appDir: string, options: RebuildOptions): Promise<any> {
   const platform = options.platform || process.platform
   const arch = options.arch || process.arch
   const additionalArgs = options.additionalArgs
@@ -113,10 +117,14 @@ function installDependencies(appDir: string, options: RebuildOptions): Promise<a
   if (additionalArgs != null) {
     execArgs.push(...additionalArgs)
   }
-  return spawn(execPath, execArgs, {
+  await spawn(execPath, execArgs, {
     cwd: appDir,
     env: getGypEnv(options.frameworkInfo, platform, arch, options.buildFromSource === true),
   })
+
+  // Some native dependencies no longer use `install` hook for building their native module, (yarn 3+ removed implicit link of `install` and `rebuild` steps)
+  // https://github.com/electron-userland/electron-builder/issues/8024
+  return rebuild(config, appDir, options)
 }
 
 export async function nodeGypRebuild(platform: NodeJS.Platform, arch: string, frameworkInfo: DesktopFrameworkInfo) {
@@ -164,8 +172,8 @@ export interface RebuildOptions {
 }
 
 /** @internal */
-export async function rebuild(appDir: string, options: RebuildOptions) {
-  const configuration: any = {
+export async function rebuild(config: Configuration, appDir: string, options: RebuildOptions) {
+  const configuration = {
     dependencies: await options.productionDeps.value,
     nodeExecPath: process.execPath,
     platform: options.platform || process.platform,
@@ -174,7 +182,33 @@ export async function rebuild(appDir: string, options: RebuildOptions) {
     execPath: process.env.npm_execpath || process.env.NPM_CLI_JS,
     buildFromSource: options.buildFromSource === true,
   }
+  if ([undefined, null, "legacy"].includes(config.nativeRebuilder)) {
+    const env = getGypEnv(options.frameworkInfo, configuration.platform, configuration.arch, options.buildFromSource === true)
+    return executeAppBuilderAndWriteJson(["rebuild-node-modules"], configuration, { env, cwd: appDir })
+  }
 
-  const env = getGypEnv(options.frameworkInfo, configuration.platform, configuration.arch, options.buildFromSource === true)
-  await executeAppBuilderAndWriteJson(["rebuild-node-modules"], configuration, { env, cwd: appDir })
+  const {
+    frameworkInfo: { version: electronVersion },
+  } = options
+  const { arch, buildFromSource } = configuration
+  const logInfo = {
+    electronVersion,
+    arch,
+    buildFromSource,
+    appDir: log.filePath(appDir) || "./",
+  }
+  log.info(logInfo, "executing @electron/rebuild")
+
+  const rebuildOptions: electronRebuild.RebuildOptions = {
+    buildPath: appDir,
+    electronVersion,
+    arch,
+    debug: log.isDebugEnabled,
+    projectRootPath: await getProjectRootPath(appDir),
+    mode: config.nativeRebuilder as RebuildMode,
+  }
+  if (buildFromSource) {
+    rebuildOptions.prebuildTagPrefix = "totally-not-a-real-prefix-to-force-rebuild"
+  }
+  return remoteRebuild(rebuildOptions)
 }