refactor: bake ffmpeg into npm instead of CDN (#3799)

bin/README.md

@@ -0,0 +1,221 @@
+# Binary Files
+
+<!-- GEN:toc-top-level -->
+- [Building `PrintDeps.exe`](#building-printdepsexe)
+- [Building `ffmpeg-mac`](#building-ffmpeg-mac)
+- [Building `ffmpeg-win64.exe`](#building-ffmpeg-win64exe)
+- [Building `ffmpeg-win32.exe`](#building-ffmpeg-win32exe)
+<!-- GEN:stop -->
+
+## Building `PrintDeps.exe`
+
+See instructions at [`//browser_patches/tools/PrintDepsWindows/README.md`](../browser_patches/tools/PrintDepsWindows/README.md)
+
+## Building `ffmpeg-mac`
+
+> FFMPEG: [`n4.3.1`](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.3.1)
+> libvpx: [`v1.9.0`](https://github.com/webmproject/libvpx/releases/tag/v1.9.0)
+
+I mostly followed steps at https://trac.ffmpeg.org/wiki/CompilationGuide/macOS
+
+1. Clone repo & checkout release tag (we're building release v4.3.1)
+
+```sh
+~$ git clone https://git.ffmpeg.org/ffmpeg.git ffmpeg
+~$ cd ffmpeg
+~/ffmpeg$ git checkout n4.3.1
+```
+
+2. Install brew dependencies
+
+```sh
+~/ffmpeg$ brew install automake fdk-aac git lame libass libtool libvorbis libvpx \
+opus sdl shtool texi2html theora wget x264 x265 xvid nasm
+```
+
+3. Prepare output folders
+
+```sh
+~/ffmpeg$ mkdir -p output/bin
+```
+
+4. Configure with vpx 
+
+```sh
+~/ffmpeg$ ./configure --prefix=$PWD/output \
+                      --pkg-config-flags="--static" \
+                      --bindir=$PWD/output/bin \
+                      --disable-everything \
+                      --enable-ffmpeg \
+                      --enable-protocol=pipe \
+                      --enable-protocol=file \
+                      --enable-parser=mjpeg \
+                      --enable-decoder=mjpeg \
+                      --enable-demuxer=image2pipe \
+                      --enable-filter=pad \
+                      --enable-filter=crop \
+                      --enable-filter=scale \
+                      --enable-muxer=webm \
+                      --enable-encoder=libvpx_vp8 \
+                      --enable-libvpx \
+                      --enable-static
+```
+
+5. Make & install to the `output/bin`
+
+```sh
+~/ffmpeg$ make && make install
+```
+
+## Building `ffmpeg-win64.exe`
+
+> FFMPEG: [`n4.3.1`](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.3.1)
+> libvpx: [`d1a7897`](https://github.com/webmproject/libvpx/commit/d1a78971ebcfd728c9c73b0cfbee69f470d4dc72)
+
+1. Install `MSYS2` from `https://www.msys2.org` and install to `c:\msys64`
+
+2. Launch `c:\msys64\mingw64` to launch a shell with a proper environment.
+
+3. `MSYS2` uses `pacman` to install dependencies. Run the following commands to update & install packages:
+
+```sh
+$ pacman -Syu
+$ pacman -Su
+$ pacman -S make pkgconf diffutils yasm
+$ pacman -S mingw-w64-x86_64-nasm mingw-w64-x86_64-gcc mingw-w64-x86_64-SDL2
+```
+
+notes:
+- `yasm` is needed for `libvpx`
+- the rest is a general compilation toolchain
+
+4. Clone `libvpx` of proper version, compile manually and install to a well-known location (prefix):
+
+```sh
+$ cd /c/
+$ git clone https://chromium.googlesource.com/webm/libvpx
+$ cd libvpx
+$ git checkout d1a78971ebcfd728c9c73b0cfbee69f470d4dc72
+$ ./configure --prefix=/eee64 --target=x86_64-win64-gcc --enable-static --disable-shared --disable-docs --disable-tools --disable-unit-tests --disable-examples
+$ make && make install
+```
+
+Note: `libvpx` has a useful readme: https://chromium.googlesource.com/webm/libvpx/+/master/README
+
+5. Once `libvpx` is compiled, compile `ffmpeg` using the just-compiled `libvpx`:
+
+```sh
+$ cd /c/
+$ git clone git://source.ffmpeg.org/ffmpeg.git
+$ cd ffmpeg
+$ git checkout n4.3.1
+$ ./configure --extra-cflags="-I/eee64/include" \
+              --extra-ldflags="-L/eee64/lib -static" \
+              --prefix=/eee64 \
+              --pkg-config-flags="--static" \
+              --bindir=$PWD/output/bin \
+              --disable-everything \
+              --enable-ffmpeg \
+              --enable-protocol=pipe \
+              --enable-protocol=file \
+              --enable-parser=mjpeg \
+              --enable-decoder=mjpeg \
+              --enable-demuxer=image2pipe \
+              --enable-filter=pad \
+              --enable-filter=crop \
+              --enable-filter=scale \
+              --enable-muxer=webm \
+              --enable-libvpx \
+              --enable-static \
+              --enable-encoder=libvpx_vp8 \
+              --disable-pthreads \
+              --disable-zlib \
+              --disable-iconv \
+              --disable-bzlib \
+              --disable-w32threads
+$ make && make install
+```
+
+note: the following resources helped me to deal with some dynamic dependencies in the resulting `ffmpeg.exe`:
+- https://stackoverflow.com/questions/13768515/how-to-do-static-linking-of-libwinpthread-1-dll-in-mingw
+
+## Building `ffmpeg-win32.exe`
+
+> FFMPEG: [`n4.3.1`](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.3.1)
+> libvpx: [`d1a7897`](https://github.com/webmproject/libvpx/commit/d1a78971ebcfd728c9c73b0cfbee69f470d4dc72)
+
+> NOTE: these steps assume that `ffmpeg-win64.exe` was just built on the machine.
+
+1. Launch `c:\msys64\mingw32` to launch a shell with a proper environment. Not sure if this is required or everything could be done from `mingw64` - but it worked.
+
+2. Update libraries for mingw32
+
+```sh
+$ pacman -Syu
+$ pacman -Su
+```
+
+3. Uninstall the `x86_64` compilers that we installed to build win64 version:
+
+```sh
+$ pacman -R mingw-w64-x86_64-nasm mingw-w64-x86_64-gcc mingw-w64-x86_64-SDL2
+```
+
+4. Install the i686 compilers instead of their x86_64 counterparts:
+
+```sh
+$ pacman -S mingw-w64-i686-nasm mingw-w64-i686-gcc mingw-w64-i686-SDL2
+```
+
+5. Remove all old source folders - we'll re-clone everything later for simplicity
+
+```sh
+$ rm -rf /c/ffmpeg && rm -rf /c/libvpx
+```
+
+6. Clone & compile libvpx. Notice a change: `--target=x86-win32-gcc`
+
+```sh
+$ cd /c/
+$ git clone https://chromium.googlesource.com/webm/libvpx
+$ cd libvpx
+$ ./configure --prefix=/eee32 --target=x86-win32-gcc --enable-static --disable-shared --disable-docs --disable-tools --disable-unit-tests --disable-examples
+$ make && make install
+```
+
+7. Clone & compile ffmpeg
+
+```sh
+$ cd /c/
+$ git clone git://source.ffmpeg.org/ffmpeg.git
+$ cd ffmpeg
+$ git checkout n4.3.1
+$ ./configure --extra-cflags="-I/eee32/include" \
+              --extra-ldflags="-L/eee32/lib -static" \
+              --prefix=/eee32 \
+              --pkg-config-flags="--static" \
+              --bindir=$PWD/output/bin \
+              --disable-everything \
+              --enable-ffmpeg \
+              --enable-protocol=pipe \
+              --enable-protocol=file \
+              --enable-parser=mjpeg \
+              --enable-decoder=mjpeg \
+              --enable-demuxer=image2pipe \
+              --enable-filter=pad \
+              --enable-filter=crop \
+              --enable-filter=scale \
+              --enable-muxer=webm \
+              --enable-libvpx \
+              --enable-static \
+              --enable-encoder=libvpx_vp8 \
+              --disable-pthreads \
+              --disable-zlib \
+              --disable-iconv \
+              --disable-bzlib \
+              --disable-w32threads
+$ make && make install
+```
+
+note: using `-j` for make somehow breaks compilation - so I'd suggest compiling everything in one thread.
+

src/server/chromium/chromium.ts

@@ -24,7 +24,6 @@ import { rewriteErrorMessage } from '../../utils/stackTrace';
 import { BrowserTypeBase } from '../browserType';
 import { ConnectionTransport, ProtocolRequest } from '../transport';
 import type { BrowserDescriptor } from '../../utils/browserPaths';
-import { browserDirectory, browsersPath, ffmpegPath} from '../../utils/browserPaths';
 import { CRDevTools } from './crDevTools';
 import { BrowserOptions } from '../browser';
 import * as types from '../types';
@@ -33,7 +32,6 @@ import { isDebugMode, getFromENV } from '../../utils/utils';
 export class Chromium extends BrowserTypeBase {
   private _devtools: CRDevTools | undefined;
   private _debugPort: number | undefined;
-  private _ffmpegPath: string;
 
   constructor(packagePath: string, browser: BrowserDescriptor) {
     const debugPortStr = getFromENV('PLAYWRIGHT_CHROMIUM_DEBUG_PORT');
@@ -45,8 +43,6 @@ export class Chromium extends BrowserTypeBase {
 
     super(packagePath, browser, debugPort ? { webSocketRegex: /^DevTools listening on (ws:\/\/.*)$/, stream: 'stderr' } : null);
     this._debugPort = debugPort;
-    const browserDir = browserDirectory(browsersPath(packagePath), browser);
-    this._ffmpegPath = ffmpegPath(browserDir, browser);
     if (isDebugMode())
       this._devtools = this._createDevTools();
   }
@@ -61,7 +57,7 @@ export class Chromium extends BrowserTypeBase {
       devtools = this._createDevTools();
       await (options as any).__testHookForDevTools(devtools);
     }
-    return CRBrowser.connect(transport, options, this._ffmpegPath, devtools);
+    return CRBrowser.connect(transport, options, devtools);
   }
 
   _rewriteStartupError(error: Error): Error {

src/server/chromium/crBrowser.ts

@@ -44,11 +44,10 @@ export class CRBrowser extends Browser {
   private _tracingRecording = false;
   private _tracingPath: string | null = '';
   private _tracingClient: CRSession | undefined;
-  private _ffmpegPath: string;
 
-  static async connect(transport: ConnectionTransport, options: BrowserOptions, ffmpegPath: string, devtools?: CRDevTools): Promise<CRBrowser> {
+  static async connect(transport: ConnectionTransport, options: BrowserOptions, devtools?: CRDevTools): Promise<CRBrowser> {
     const connection = new CRConnection(transport);
-    const browser = new CRBrowser(connection, options, ffmpegPath);
+    const browser = new CRBrowser(connection, options);
     browser._devtools = devtools;
     const session = connection.rootSession;
     const version = await session.send('Browser.getVersion');
@@ -89,10 +88,9 @@ export class CRBrowser extends Browser {
     return browser;
   }
 
-  constructor(connection: CRConnection, options: BrowserOptions, ffmpegPath: string) {
+  constructor(connection: CRConnection, options: BrowserOptions) {
     super(options);
     this._connection = connection;
-    this._ffmpegPath = ffmpegPath;
     this._session = this._connection.rootSession;
     this._connection.on(ConnectionEvents.Disconnected, () => this._didClose());
     this._session.on('Target.attachedToTarget', this._onAttachedToTarget.bind(this));
@@ -148,7 +146,7 @@ export class CRBrowser extends Browser {
     assert(!this._serviceWorkers.has(targetInfo.targetId), 'Duplicate target ' + targetInfo.targetId);
 
     if (targetInfo.type === 'background_page') {
-      const backgroundPage = new CRPage(session, targetInfo.targetId, context, null, false, this._ffmpegPath);
+      const backgroundPage = new CRPage(session, targetInfo.targetId, context, null, false);
       this._backgroundPages.set(targetInfo.targetId, backgroundPage);
       backgroundPage.pageOrError().then(() => {
         context!.emit(CRBrowserContext.CREvents.BackgroundPage, backgroundPage._page);
@@ -158,7 +156,7 @@ export class CRBrowser extends Browser {
 
     if (targetInfo.type === 'page') {
       const opener = targetInfo.openerId ? this._crPages.get(targetInfo.openerId) || null : null;
-      const crPage = new CRPage(session, targetInfo.targetId, context, opener, true, this._ffmpegPath);
+      const crPage = new CRPage(session, targetInfo.targetId, context, opener, true);
       this._crPages.set(targetInfo.targetId, crPage);
       crPage.pageOrError().then(pageOrError => {
         const page = crPage._page;

src/server/chromium/crPage.ts

@@ -56,7 +56,6 @@ export class CRPage implements PageDelegate {
   readonly _browserContext: CRBrowserContext;
   private readonly _pagePromise: Promise<Page | Error>;
   _initializedPage: Page | null = null;
-  readonly _ffmpegPath: string;
 
   // Holds window features for the next popup being opened via window.open,
   // until the popup target arrives. This could be racy if two oopifs
@@ -65,10 +64,9 @@ export class CRPage implements PageDelegate {
   // of new popup targets.
   readonly _nextWindowOpenPopupFeatures: string[][] = [];
 
-  constructor(client: CRSession, targetId: string, browserContext: CRBrowserContext, opener: CRPage | null, hasUIWindow: boolean, ffmpegPath: string) {
+  constructor(client: CRSession, targetId: string, browserContext: CRBrowserContext, opener: CRPage | null, hasUIWindow: boolean) {
     this._targetId = targetId;
     this._opener = opener;
-    this._ffmpegPath = ffmpegPath;
     this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac);
     this.rawMouse = new RawMouseImpl(client);
     this._pdf = new CRPDF(client);
@@ -752,7 +750,7 @@ class FrameSession {
   async _startScreencast(screencastId: string, options: types.PageScreencastOptions): Promise<void> {
     if (this._screencastState !== 'stopped')
       throw new Error('Already started');
-    const videoRecorder = await VideoRecorder.launch(this._crPage._ffmpegPath, options);
+    const videoRecorder = await VideoRecorder.launch(options);
     this._screencastState = 'starting';
     try {
       await this._client.send('Page.startScreencast', {

src/server/chromium/videoRecorder.ts

@@ -18,6 +18,8 @@ import { launchProcess } from '../processLauncher';
 import { ChildProcess } from 'child_process';
 import { Progress, runAbortableTask } from '../progress';
 import * as types from '../types';
+import * as path from 'path';
+import * as os from 'os';
 import { assert } from '../../utils/utils';
 
 const fps = 25;
@@ -30,22 +32,20 @@ export class VideoRecorder {
   private _lastFrameBuffer: Buffer | null = null;
   private _lastWriteTimestamp: number = 0;
   private readonly _progress: Progress;
-  private readonly _ffmpegPath: string;
 
-  static async launch(ffmpegPath: string, options: types.PageScreencastOptions): Promise<VideoRecorder> {
+  static async launch(options: types.PageScreencastOptions): Promise<VideoRecorder> {
     if (!options.outputFile.endsWith('.webm'))
       throw new Error('File must have .webm extension');
 
     return await runAbortableTask(async progress => {
-      const recorder = new VideoRecorder(ffmpegPath, progress);
+      const recorder = new VideoRecorder(progress);
       await recorder._launch(options);
       return recorder;
     }, 0, 'browser');
   }
 
-  private constructor(ffmpegPath: string, progress: Progress) {
+  private constructor(progress: Progress) {
     this._progress = progress;
-    this._ffmpegPath = ffmpegPath;
     this._lastWritePromise = Promise.resolve();
   }
 
@@ -56,8 +56,15 @@ export class VideoRecorder {
     const args = `-loglevel error -f image2pipe -c:v mjpeg -i - -y -an -r ${fps} -c:v vp8 -vf pad=${w}:${h}:0:0:gray,crop=${w}:${h}:0:0`.split(' ');
     args.push(options.outputFile);
     const progress = this._progress;
+
+    let ffmpegPath = 'ffmpeg';
+    const binPath = path.join(__dirname, '../../../bin/');
+    if (os.platform() === 'win32')
+      ffmpegPath = path.join(binPath, os.arch() === 'x64' ? 'ffmpeg-win64.exe' : 'ffmpeg-win32.exe');
+    else if (os.platform() === 'darwin')
+      ffmpegPath = path.join(binPath, 'ffmpeg-mac');
     const { launchedProcess, gracefullyClose } = await launchProcess({
-      executablePath: this._ffmpegPath,
+      executablePath: ffmpegPath,
       args,
       pipeStdin: true,
       progress,

src/server/electron/electron.ts

@@ -184,7 +184,7 @@ export class Electron  {
         close: gracefullyClose,
         kill
       };
-      const browser = await CRBrowser.connect(chromeTransport, { name: 'electron', headful: true, persistent: { noDefaultViewport: true }, browserProcess }, '');
+      const browser = await CRBrowser.connect(chromeTransport, { name: 'electron', headful: true, persistent: { noDefaultViewport: true }, browserProcess });
       app = new ElectronApplication(browser, nodeConnection);
       await app._init();
       return app;