Watch directories instead of individual files (#1279)

This prevents EMFILE errors on large projects caused by running out of file descriptors.

src/Bundler.js

@@ -4,7 +4,7 @@ const Parser = require('./Parser');
 const WorkerFarm = require('./workerfarm/WorkerFarm');
 const Path = require('path');
 const Bundle = require('./Bundle');
-const {FSWatcher} = require('chokidar');
+const Watcher = require('./Watcher');
 const FSCache = require('./FSCache');
 const HMRServer = require('./HMRServer');
 const Server = require('./Server');
@@ -107,7 +107,9 @@ class Bundler extends EventEmitter {
       hmr:
         target === 'node'
           ? false
-          : typeof options.hmr === 'boolean' ? options.hmr : watch,
+          : typeof options.hmr === 'boolean'
+            ? options.hmr
+            : watch,
       https: options.https || false,
       logLevel: isNaN(options.logLevel) ? 3 : options.logLevel,
       entryFiles: this.entryFiles,
@@ -321,12 +323,7 @@ class Bundler extends EventEmitter {
     this.options.env = process.env;
 
     if (this.options.watch) {
-      // FS events on macOS are flakey in the tests, which write lots of files very quickly
-      // See https://github.com/paulmillr/chokidar/issues/612
-      this.watcher = new FSWatcher({
-        useFsEvents: process.env.NODE_ENV !== 'test'
-      });
-
+      this.watcher = new Watcher();
       this.watcher.on('change', this.onChange.bind(this));
     }
 
@@ -344,7 +341,7 @@ class Bundler extends EventEmitter {
     }
 
     if (this.watcher) {
-      this.watcher.close();
+      this.watcher.stop();
     }
 
     if (this.hmr) {
@@ -378,7 +375,7 @@ class Bundler extends EventEmitter {
     }
 
     if (!this.watchedAssets.has(path)) {
-      this.watcher.add(path);
+      this.watcher.watch(path);
       this.watchedAssets.set(path, new Set());
     }
 

src/Watcher.js

@@ -0,0 +1,124 @@
+const {FSWatcher} = require('chokidar');
+const Path = require('path');
+
+/**
+ * This watcher wraps chokidar so that we watch directories rather than individual files.
+ * This prevents us from hitting EMFILE errors when running out of file descriptors.
+ */
+class Watcher {
+  constructor() {
+    // FS events on macOS are flakey in the tests, which write lots of files very quickly
+    // See https://github.com/paulmillr/chokidar/issues/612
+    this.shouldWatchDirs = process.env.NODE_ENV !== 'test';
+    this.watcher = new FSWatcher({
+      useFsEvents: this.shouldWatchDirs,
+      ignoreInitial: true
+    });
+
+    this.watchedDirectories = new Map();
+  }
+
+  /**
+   * Find a parent directory of `path` which is already watched
+   */
+  getWatchedParent(path) {
+    path = Path.dirname(path);
+
+    let root = Path.parse(path).root;
+    while (path !== root) {
+      if (this.watchedDirectories.has(path)) {
+        return path;
+      }
+
+      path = Path.dirname(path);
+    }
+
+    return null;
+  }
+
+  /**
+   * Find a list of child directories of `path` which are already watched
+   */
+  getWatchedChildren(path) {
+    path = Path.dirname(path) + Path.sep;
+
+    let res = [];
+    for (let dir of this.watchedDirectories.keys()) {
+      if (dir.startsWith(path)) {
+        res.push(dir);
+      }
+    }
+
+    return res;
+  }
+
+  /**
+   * Add a path to the watcher
+   */
+  watch(path) {
+    if (this.shouldWatchDirs) {
+      // If there is no parent directory already watching this path, add a new watcher.
+      let parent = this.getWatchedParent(path);
+      if (!parent) {
+        // Find watchers on child directories, and remove them. They will be handled by the new parent watcher.
+        let children = this.getWatchedChildren(path);
+        let count = 1;
+
+        for (let dir of children) {
+          count += this.watchedDirectories.get(dir);
+          this.watcher.unwatch(dir);
+          this.watchedDirectories.delete(dir);
+        }
+
+        let dir = Path.dirname(path);
+        this.watcher.add(dir);
+        this.watchedDirectories.set(dir, count);
+      } else {
+        // Otherwise, increment the reference count of the parent watcher.
+        this.watchedDirectories.set(
+          parent,
+          this.watchedDirectories.get(parent) + 1
+        );
+      }
+    } else {
+      this.watcher.add(path);
+    }
+  }
+
+  /**
+   * Remove a path from the watcher
+   */
+  unwatch(path) {
+    if (this.shouldWatchDirs) {
+      let dir = this.getWatchedParent(path);
+      if (dir) {
+        // When the count of files watching a directory reaches zero, unwatch it.
+        let count = this.watchedDirectories.get(dir) - 1;
+        if (count === 0) {
+          this.watchedDirectories.delete(dir);
+          this.watcher.unwatch(dir);
+        } else {
+          this.watchedDirectories.set(dir, count);
+        }
+      }
+    } else {
+      this.watcher.unwatch(path);
+    }
+  }
+
+  /**
+   * Add an event handler
+   */
+  on(event, callback) {
+    this.watcher.on(event, callback);
+  }
+
+  /**
+   * Stop watching all paths
+   */
+  stop() {
+    this.watcher.close();
+  }
+}
+
+module.exports = Watcher;