update to Electron 15.1.2 and support Node.js 14

.github/workflows/build.yml

@@ -20,7 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-2019, ubuntu-18.04, macos-10.15]
-        node: ['12.x', '14.x']
+        node: ['12.x', '14.x', '16.x']
 
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
@@ -43,10 +43,9 @@ jobs:
       - name: Build
         shell: bash
         run: |
-          yarn --skip-integrity-check --network-timeout 100000 --ignore-engines
+          yarn --ignore-engines --skip-integrity-check --network-timeout 100000
+          ./scripts/check_git_status.sh
           yarn lint
-          npx electron-replace-ffmpeg
-          npx electron-codecs-test
           yarn build:examples
           ./scripts/check_git_status.sh
         env:

configs/base.tsconfig.json

@@ -19,6 +19,7 @@
     "jsx": "react",
     "lib": [
       "ES2017",
+      "ES2020.Promise",
       "dom"
     ],
     "sourceMap": true

dev-packages/application-manager/package.json

@@ -34,17 +34,21 @@
     "@babel/plugin-transform-runtime": "^7.10.0",
     "@babel/preset-env": "^7.10.0",
     "@theia/application-package": "1.21.0",
+    "@theia/ffmpeg": "1.21.0",
     "@types/fs-extra": "^4.0.2",
+    "@types/semver": "^7.3.8",
     "babel-loader": "^8.2.2",
     "buffer": "^6.0.3",
     "circular-dependency-plugin": "^5.2.2",
     "compression-webpack-plugin": "^9.0.0",
     "copy-webpack-plugin": "^8.1.1",
     "css-loader": "^6.2.0",
-    "electron-rebuild": "^1.8.6",
+    "electron-rebuild": "^1.11.0",
     "fs-extra": "^4.0.2",
     "ignore-loader": "^0.1.2",
     "less": "^3.0.3",
+    "node-abi": "*",
+    "semver": "^7.3.5",
     "setimmediate": "^1.0.5",
     "source-map-loader": "^2.0.1",
     "source-map-support": "^0.5.19",
@@ -55,8 +59,17 @@
     "worker-loader": "^3.0.8",
     "yargs": "^15.3.1"
   },
+  "peerDependencies": {
+    "@theia/electron": "1.21.0"
+  },
+  "peerDependenciesMeta": {
+    "@theia/electron": {
+      "optional": true
+    }
+  },
   "devDependencies": {
-    "@theia/ext-scripts": "1.21.0"
+    "@theia/ext-scripts": "1.21.0",
+    "@types/node-abi": "*"
   },
   "nyc": {
     "extends": "../../configs/nyc.json"

dev-packages/application-manager/src/application-package-manager.ts

@@ -17,12 +17,26 @@
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as cp from 'child_process';
+import * as semver from 'semver';
+import * as ffmpeg from '@theia/ffmpeg';
 import { ApplicationPackage, ApplicationPackageOptions } from '@theia/application-package';
 import { WebpackGenerator, FrontendGenerator, BackendGenerator } from './generator';
 import { ApplicationProcess } from './application-process';
 import { GeneratorOptions } from './generator/abstract-generator';
 import yargs = require('yargs');
 
+// Declare missing exports from `@types/semver@7`
+declare module 'semver' {
+    function minVersion(range: string): string;
+}
+
+class AbortError extends Error {
+    constructor(...args: Parameters<ErrorConstructor>) {
+        super(...args);
+        Object.setPrototypeOf(this, AbortError.prototype);
+    }
+}
+
 export class ApplicationPackageManager {
 
     static defineGeneratorOptions<T>(cli: yargs.Argv<T>): yargs.Argv<T & {
@@ -67,7 +81,22 @@ export class ApplicationPackageManager {
         ]);
     }
 
+    async prepare(): Promise<void> {
+        if (this.pck.isElectron()) {
+            await this.prepareElectron();
+        }
+    }
+
     async generate(options: GeneratorOptions = {}): Promise<void> {
+        try {
+            await this.prepare();
+        } catch (error) {
+            if (error instanceof AbortError) {
+                console.warn(error.message);
+                process.exit(1);
+            }
+            throw error;
+        }
         await Promise.all([
             new WebpackGenerator(this.pck, options).generate(),
             new BackendGenerator(this.pck, options).generate(),
@@ -121,6 +150,55 @@ export class ApplicationPackageManager {
         return this.__process.fork(this.pck.backend('main.js'), mainArgs, options);
     }
 
+    /**
+     * Inject Theia's Electron-specific dependencies into the application's package.json.
+     *
+     * Only overwrite the Electron range if the current minimum supported version is lower than the recommended one.
+     */
+    protected async prepareElectron(): Promise<void> {
+        let theiaElectron;
+        try {
+            theiaElectron = await import('@theia/electron');
+        } catch (error) {
+            if (error.code === 'ERR_MODULE_NOT_FOUND') {
+                throw new AbortError('Please install @theia/electron as part of your Theia Electron application');
+            }
+            throw error;
+        }
+        const expectedRange = theiaElectron.electronRange;
+        const appPackageJsonPath = this.pck.path('package.json');
+        const appPackageJson = await fs.readJSON(appPackageJsonPath) as { devDependencies?: Record<string, string> };
+        if (!appPackageJson.devDependencies) {
+            appPackageJson.devDependencies = {};
+        }
+        const currentRange: string | undefined = appPackageJson.devDependencies.electron;
+        if (!currentRange || semver.compare(semver.minVersion(currentRange), semver.minVersion(expectedRange)) < 0) {
+            // Update the range with the recommended one and write it on disk.
+            appPackageJson.devDependencies = this.insertAlphabetically(appPackageJson.devDependencies, 'electron', expectedRange);
+            await fs.writeJSON(appPackageJsonPath, appPackageJson, { spaces: 2 });
+            throw new AbortError('Updated dependencies, please run "install" again');
+        }
+        if (!theiaElectron.electronVersion || !semver.satisfies(theiaElectron.electronVersion, currentRange)) {
+            throw new AbortError('Dependencies are out of sync, please run "install" again');
+        }
+        await ffmpeg.replaceFfmpeg();
+        await ffmpeg.checkFfmpeg();
+    }
+
+    protected insertAlphabetically<T extends Record<string, string>>(object: T, key: string, value: string): T {
+        const updated: Record<string, unknown> = {};
+        for (const property of Object.keys(object)) {
+            if (property.localeCompare(key) > 0) {
+                updated[key] = value;
+            }
+            updated[property] = object[property];
+        }
+        if (!(key in updated)) {
+            updated[key] = value;
+        }
+        return updated as T;
+    }
+
     private adjustArgs(args: string[], forkOptions: cp.ForkOptions = {}): Readonly<{ mainArgs: string[]; options: cp.ForkOptions }> {
         const options = {
             ...this.forkOptions,
@@ -147,5 +225,4 @@ export class ApplicationPackageManager {
             }
         };
     }
-
 }

dev-packages/application-manager/src/generator/frontend-generator.ts

@@ -125,6 +125,7 @@ module.exports = nls.loadTranslations().then(() => {
         return `// @ts-check
 
 require('reflect-metadata');
+require('@theia/electron/shared/@electron/remote/main').initialize();
 
 // Useful for Electron/NW.js apps as GUI apps on macOS doesn't inherit the \`$PATH\` define
 // in your dotfiles (.bashrc/.bash_profile/.zshrc/etc).

dev-packages/application-manager/src/rebuild.ts

@@ -28,8 +28,10 @@ interface ExitToken {
     onSignal(callback: (signal: NodeJS.Signals) => void): void
 }
 
+type NodeABI = string | number;
+
 export const DEFAULT_MODULES = [
-    '@theia/node-pty',
+    'node-pty',
     'nsfw',
     'native-keymap',
     'find-git-repositories',
@@ -46,9 +48,10 @@ export interface RebuildOptions {
      */
     cacheRoot?: string
     /**
-     * Rebuild modules for Electron anyway.
+     * In the event that `node-abi` doesn't recognize the current Electron version,
+     * you can specify the Node ABI to rebuild for.
      */
-    force?: boolean
+    forceAbi?: NodeABI
 }
 
 /**
@@ -59,12 +62,13 @@ export function rebuild(target: RebuildTarget, options: RebuildOptions = {}): vo
     const {
         modules = DEFAULT_MODULES,
         cacheRoot = process.cwd(),
+        forceAbi,
     } = options;
     const cache = path.resolve(cacheRoot, '.browser_modules');
     const cacheExists = folderExists(cache);
     guardExit(async token => {
         if (target === 'electron' && !cacheExists) {
-            process.exitCode = await rebuildElectronModules(cache, modules, token);
+            process.exitCode = await rebuildElectronModules(cache, modules, forceAbi, token);
         } else if (target === 'browser' && cacheExists) {
             process.exitCode = await revertBrowserModules(cache, modules);
         } else {
@@ -100,7 +104,7 @@ interface ModuleBackup {
     originalLocation: string
 }
 
-async function rebuildElectronModules(browserModuleCache: string, modules: string[], token: ExitToken): Promise<number> {
+async function rebuildElectronModules(browserModuleCache: string, modules: string[], forceAbi: NodeABI | undefined, token: ExitToken): Promise<number> {
     const modulesJsonPath = path.join(browserModuleCache, 'modules.json');
     const modulesJson: ModulesJson = await fs.access(modulesJsonPath).then(
         () => fs.readJson(modulesJsonPath),
@@ -149,14 +153,21 @@ async function rebuildElectronModules(browserModuleCache: string, modules: strin
             ? m.substring(slash + 1)
             : m;
     });
+    let exitCode: number | undefined;
     try {
         if (process.env.THEIA_REBUILD_NO_WORKAROUND) {
-            return await runElectronRebuild(todo, token);
+            exitCode = await runElectronRebuild(todo, forceAbi, token);
         } else {
-            return await electronRebuildExtraModulesWorkaround(process.cwd(), todo, () => runElectronRebuild(todo, token), token);
+            exitCode = await electronRebuildExtraModulesWorkaround(process.cwd(), todo, () => runElectronRebuild(todo, forceAbi, token), token);
         }
     } catch (error) {
-        return revertBrowserModules(browserModuleCache, modules);
+        console.error(error);
+    } finally {
+        // If code is undefined or different from zero we need to revert back to the browser modules.
+        if (exitCode !== 0) {
+            await revertBrowserModules(browserModuleCache, modules);
+        }
+        return exitCode ?? 1;
     }
 }
 
@@ -197,10 +208,14 @@ async function revertBrowserModules(browserModuleCache: string, modules: string[
     return exitCode;
 }
 
-async function runElectronRebuild(modules: string[], token: ExitToken): Promise<number> {
+async function runElectronRebuild(modules: string[], forceAbi: NodeABI | undefined, token: ExitToken): Promise<number> {
     const todo = modules.join(',');
-    return new Promise((resolve, reject) => {
-        const electronRebuild = cp.spawn(`npx --no-install electron-rebuild -f -w="${todo}" -o="${todo}"`, {
+    return new Promise(async (resolve, reject) => {
+        let command = `npx --no-install electron-rebuild -f -w=${todo} -o=${todo}`;
+        if (forceAbi) {
+            command += ` --force-abi ${forceAbi}`;
+        }
+        const electronRebuild = cp.spawn(command, {
             stdio: 'inherit',
             shell: true,
         });

dev-packages/application-manager/tsconfig.json

@@ -11,6 +11,9 @@
   "references": [
     {
       "path": "../application-package"
+    },
+    {
+      "path": "../ffmpeg"
     }
   ]
 }

dev-packages/application-package/src/application-package.ts

@@ -45,14 +45,6 @@ export class ApplicationPackage {
         this.projectPath = options.projectPath;
         this.log = options.log || console.log.bind(console);
         this.error = options.error || console.error.bind(console);
-        if (this.isElectron()) {
-            const { version } = require('../package.json');
-            try {
-                require.resolve('@theia/electron/package.json', { paths: [this.projectPath] });
-            } catch {
-                console.warn(`please install @theia/electron@${version} as a runtime dependency`);
-            }
-        }
     }
 
     protected _registry: NpmRegistry | undefined;

dev-packages/cli/package.json

@@ -17,12 +17,14 @@
   "files": [
     "bin",
     "lib",
+    "native",
     "src"
   ],
   "bin": {
     "theia": "./bin/theia"
   },
   "scripts": {
+    "prepare": "tsc -b",
     "lint": "theiaext lint",
     "build": "theiaext build",
     "watch": "theiaext watch",
@@ -32,6 +34,7 @@
     "@theia/application-manager": "1.21.0",
     "@theia/application-package": "1.21.0",
     "@theia/localization-manager": "1.21.0",
+    "@theia/ffmpeg": "1.21.0",
     "@theia/ovsx-client": "1.21.0",
     "@types/chai": "^4.2.7",
     "@types/mkdirp": "^0.5.2",
@@ -50,9 +53,18 @@
     "puppeteer": "^2.0.0",
     "puppeteer-to-istanbul": "^1.2.2",
     "temp": "^0.9.1",
+    "unzipper": "^0.9.11",
     "yargs": "^15.3.1"
   },
   "devDependencies": {
-    "@types/proxy-from-env": "^1.0.1"
+    "@types/chai": "^4.2.7",
+    "@types/mkdirp": "^0.5.2",
+    "@types/mocha": "^5.2.7",
+    "@types/node-fetch": "^2.5.7",
+    "@types/proxy-from-env": "^1.0.1",
+    "@types/puppeteer": "^2.0.0",
+    "@types/requestretry": "^1.12.3",
+    "@types/tar": "^4.0.3",
+    "@types/unzipper": "^0.9.2"
   }
 }