feature(gatsby): Add experiment to run source plugins in parallel

packages/gatsby/src/utils/api-runner-node.js

@@ -558,90 +558,109 @@ module.exports = async (api, args = {}, { pluginSource, activity } = {}) =>
       }
     }
 
-    Promise.mapSeries(implementingPlugins, plugin => {
-      if (stopQueuedApiRuns) {
-        return null
-      }
+    let apiRunPromiseOptions = {}
+    let runPromise
+    if (
+      api === `sourceNodes` &&
+      process.env.GATSBY_EXPERIMENTAL_PARALLEL_SOURCING
+    ) {
+      runPromise = Promise.map
+      apiRunPromiseOptions.concurrency = 20
+    } else {
+      runPromise = Promise.mapSeries
+      apiRunPromiseOptions = undefined
+    }
 
-      let gatsbyNode = pluginNodeCache.get(plugin.name)
-      if (!gatsbyNode) {
-        gatsbyNode = require(`${plugin.resolve}/gatsby-node`)
-        pluginNodeCache.set(plugin.name, gatsbyNode)
-      }
+    runPromise(
+      implementingPlugins,
+      plugin => {
+        if (stopQueuedApiRuns) {
+          return null
+        }
 
-      const pluginName =
-        plugin.name === `default-site-plugin` ? `gatsby-node.js` : plugin.name
-
-      // TODO: rethink createNode API to handle this better
-      if (
-        api === `onCreateNode` &&
-        gatsbyNode?.unstable_shouldOnCreateNode && // Don't bail if this api is not exported
-        !gatsbyNode.unstable_shouldOnCreateNode(
-          { node: args.node },
-          plugin.pluginOptions
-        )
-      ) {
-        // Do not try to schedule an async event for this node for this plugin
-        return null
-      }
+        let gatsbyNode = pluginNodeCache.get(plugin.name)
+        if (!gatsbyNode) {
+          gatsbyNode = require(`${plugin.resolve}/gatsby-node`)
+          pluginNodeCache.set(plugin.name, gatsbyNode)
+        }
 
-      return new Promise(resolve => {
-        resolve(runAPI(plugin, api, { ...args, parentSpan: apiSpan }, activity))
-      }).catch(err => {
-        decorateEvent(`BUILD_PANIC`, {
-          pluginName: `${plugin.name}@${plugin.version}`,
-        })
+        const pluginName =
+          plugin.name === `default-site-plugin` ? `gatsby-node.js` : plugin.name
+
+        // TODO: rethink createNode API to handle this better
+        if (
+          api === `onCreateNode` &&
+          gatsbyNode?.unstable_shouldOnCreateNode && // Don't bail if this api is not exported
+          !gatsbyNode.unstable_shouldOnCreateNode(
+            { node: args.node },
+            plugin.pluginOptions
+          )
+        ) {
+          // Do not try to schedule an async event for this node for this plugin
+          return null
+        }
+
+        return new Promise(resolve => {
+          resolve(
+            runAPI(plugin, api, { ...args, parentSpan: apiSpan }, activity)
+          )
+        }).catch(err => {
+          decorateEvent(`BUILD_PANIC`, {
+            pluginName: `${plugin.name}@${plugin.version}`,
+          })
 
-        const localReporter = getLocalReporter({ activity, reporter })
+          const localReporter = getLocalReporter({ activity, reporter })
 
-        const file = stackTrace
-          .parse(err)
-          .find(file => /gatsby-node/.test(file.fileName))
+          const file = stackTrace
+            .parse(err)
+            .find(file => /gatsby-node/.test(file.fileName))
 
-        let codeFrame = ``
-        const structuredError = errorParser({ err })
+          let codeFrame = ``
+          const structuredError = errorParser({ err })
 
-        if (file) {
-          const { fileName, lineNumber: line, columnNumber: column } = file
+          if (file) {
+            const { fileName, lineNumber: line, columnNumber: column } = file
 
-          try {
-            const code = fs.readFileSync(fileName, { encoding: `utf-8` })
-            codeFrame = codeFrameColumns(
-              code,
-              {
-                start: {
-                  line,
-                  column,
+            try {
+              const code = fs.readFileSync(fileName, { encoding: `utf-8` })
+              codeFrame = codeFrameColumns(
+                code,
+                {
+                  start: {
+                    line,
+                    column,
+                  },
                 },
-              },
-              {
-                highlightCode: true,
-              }
-            )
-          } catch (_e) {
-            // sometimes stack trace point to not existing file
-            // particularly when file is transpiled and path actually changes
-            // (like pointing to not existing `src` dir or original typescript file)
-          }
+                {
+                  highlightCode: true,
+                }
+              )
+            } catch (_e) {
+              // sometimes stack trace point to not existing file
+              // particularly when file is transpiled and path actually changes
+              // (like pointing to not existing `src` dir or original typescript file)
+            }
 
-          structuredError.location = {
-            start: { line: line, column: column },
+            structuredError.location = {
+              start: { line: line, column: column },
+            }
+            structuredError.filePath = fileName
           }
-          structuredError.filePath = fileName
-        }
 
-        structuredError.context = {
-          ...structuredError.context,
-          pluginName,
-          api,
-          codeFrame,
-        }
+          structuredError.context = {
+            ...structuredError.context,
+            pluginName,
+            api,
+            codeFrame,
+          }
 
-        localReporter.panicOnBuild(structuredError)
+          localReporter.panicOnBuild(structuredError)
 
-        return null
-      })
-    }).then(results => {
+          return null
+        })
+      },
+      apiRunPromiseOptions
+    ).then(results => {
       if (onAPIRunComplete) {
         onAPIRunComplete()
       }

packages/gatsby/src/utils/flags.ts

@@ -60,6 +60,15 @@ const activeFlags: Array<IFlag> = [
     description: `Don't process images during development until they're requested from the browser. Speeds starting the develop server.`,
     umbrellaIssue: `https://github.com/gatsbyjs/gatsby/discussions/27603`,
   },
+  {
+    name: `PARALLEL_SOURCING`,
+    env: `GATSBY_EXPERIMENTAL_PARALLEL_SOURCING`,
+    command: `all`,
+    telemetryId: `ParallelSourcing`,
+    experimental: true,
+    description: `Run all source plugins at the same time instead of serially. For sites with multiple source plugins, this can speedup sourcing and transforming considerably.`,
+    umbrellaIssue: `https://github.com/gatsbyjs/gatsby/discussions/28336`,
+  },
 ]
 
 export default activeFlags