Include resources in dependency classpath

In #730, I claimed:

> Resources are files that are used at runtime, either when testing or
running an application. These files can be generated by the build tool
or be present in a resource directory of a project.

However, this was not the full story, it turns out that resources are
also required by macros, which happen at compile time, and therefore
resources are required by `compile`.

This patch modifies bloop to use the classpath with resources as the
default from now on for any action in bloop, as all of them require it.

Fixes #763

bridges/scala-native/src/main/scala/bloop/scalanative/NativeBridge.scala

@@ -10,14 +10,20 @@ import scala.scalanative.build.{Build, Config, Discover, GC, Mode, Logger => Nat
 
 object NativeBridge {
   private implicit val ctx: DebugFilter = DebugFilter.Link
-  def nativeLink(config0: NativeConfig, project: Project, entry: String, target: Path, logger: Logger): Path = {
+  def nativeLink(
+      config0: NativeConfig,
+      project: Project,
+      classpath: Array[Path],
+      entry: String,
+      target: Path,
+      logger: Logger
+  ): Path = {
     val workdir = project.out.resolve("native")
     if (workdir.isDirectory) Paths.delete(workdir)
     Files.createDirectories(workdir.underlying)
 
-    val classpath = project.compilationClasspath.map(_.underlying)
     val nativeLogger = NativeLogger(logger.debug _, logger.info _, logger.warn _, logger.error _)
-    val config = setUpNativeConfig(project, config0)
+    val config = setUpNativeConfig(project, classpath, config0)
     val nativeMode = config.mode match {
       case LinkerMode.Debug => Mode.debug
       case LinkerMode.Release => Mode.release
@@ -44,6 +50,7 @@ object NativeBridge {
 
   private[scalanative] def setUpNativeConfig(
       project: Project,
+      classpath: Array[Path],
       config: NativeConfig
   ): NativeConfig = {
     val mode = config.mode
@@ -66,7 +73,7 @@ object NativeBridge {
       if (config.nativelib.toString.nonEmpty) config.nativelib
       else {
         Discover
-          .nativelib(project.compilationClasspath.map(_.underlying))
+          .nativelib(classpath)
           .getOrElse(sys.error("Fatal: nativelib is missing and could not be found."))
       }
     }

bridges/scalajs-0.6/src/main/scala/bloop/scalajs/JsBridge.scala

@@ -53,12 +53,12 @@ object JsBridge {
   def link(
       config: JsConfig,
       project: Project,
+      classpath: Array[Path],
       runMain: java.lang.Boolean,
       mainClass: Option[String],
       target: Path,
       logger: BloopLogger
   ): Unit = {
-    val classpath = project.compilationClasspath.map(_.underlying)
     val classpathIrFiles = classpath
       .filter(Files.isDirectory(_))
       .flatMap(findIrFiles)
@@ -137,7 +137,8 @@ object JsBridge {
       } else {
         new CustomDomNodeEnv(
           JSDOMNodeJSEnv.Config().withEnv(fullEnv).withExecutable(nodePath),
-          customScripts)
+          customScripts
+        )
       }
     }
 

bridges/scalajs-1.0/src/main/scala/bloop/scalajs/JsBridge.scala

@@ -43,6 +43,7 @@ object JsBridge {
   def link(
       config: JsConfig,
       project: Project,
+      classpath: Array[Path],
       runMain: java.lang.Boolean,
       mainClass: Option[String],
       target: Path,
@@ -61,8 +62,7 @@ object JsBridge {
     }
 
     val cache = new IRFileCache().newCache
-    val irClasspath =
-      FileScalaJSIRContainer.fromClasspath(project.compilationClasspath.map(_.toFile))
+    val irClasspath = FileScalaJSIRContainer.fromClasspath(classpath.map(_.toFile))
     val irFiles = cache.cached(irClasspath)
 
     val moduleInitializers = mainClass match {

frontend/src/main/scala/bloop/bsp/BloopBspServices.scala

@@ -111,7 +111,8 @@ final class BloopBspServices(
    * @return An async computation that returns the response to the client.
    */
   def initialize(
-      params: bsp.InitializeBuildParams): BspEndpointResponse[bsp.InitializeBuildResult] = {
+      params: bsp.InitializeBuildParams
+  ): BspEndpointResponse[bsp.InitializeBuildResult] = {
     val uri = new java.net.URI(params.rootUri.value)
     val configDir = AbsolutePath(uri).resolve(relativeConfigPath)
     reloadState(configDir).map { state =>
@@ -561,18 +562,22 @@ final class BloopBspServices(
   }
 
   def scalacOptions(
-      request: bsp.ScalacOptionsParams): BspEndpointResponse[bsp.ScalacOptionsResult] = {
+      request: bsp.ScalacOptionsParams
+  ): BspEndpointResponse[bsp.ScalacOptionsResult] = {
     def scalacOptions(
         projects: Seq[ProjectMapping],
         state: State
     ): BspResult[bsp.ScalacOptionsResult] = {
       val response = bsp.ScalacOptionsResult(
         projects.iterator.map {
           case (target, project) =>
+            val dag = state.build.getDagFor(project)
+            val fullClasspath = project.dependencyClasspath(dag)
+            val classpath = fullClasspath.map(e => bsp.Uri(e.toBspUri)).toList
             bsp.ScalacOptionsItem(
               target = target,
               options = project.scalacOptions.toList,
-              classpath = project.compilationClasspath.map(e => bsp.Uri(e.toBspUri)).toList,
+              classpath = classpath,
               classDirectory = bsp.Uri(project.classesDir.toBspUri)
             )
         }.toList

frontend/src/main/scala/bloop/data/Project.scala

@@ -70,8 +70,8 @@ final case class Project(
     }
   }
 
-  def fullClasspathFor(dag: Dag[Project]): Array[AbsolutePath] = {
-    val cp = compilationClasspath.toBuffer
+  def dependencyClasspath(dag: Dag[Project]): Array[AbsolutePath] = {
+    val cp = (classesDir :: rawClasspath).toBuffer
     // Add the resources right before the classes directory if found in the classpath
     Dag.dfs(dag).foreach { p =>
       val index = cp.indexOf(p.classesDir)

frontend/src/main/scala/bloop/engine/tasks/CompilationTask.scala

@@ -46,6 +46,7 @@ object CompilationTask {
     import state.{logger, compilerCache}
     def compile(graphInputs: CompileGraph.Inputs): Task[Compiler.Result] = {
       val project = graphInputs.bundle.project
+      val classpath = graphInputs.bundle.classpath
       graphInputs.bundle.toSourcesAndInstance match {
         case Left(earlyResult) =>
           val complete = CompileExceptions.CompletePromise(graphInputs.store)
@@ -96,7 +97,7 @@ object CompilationTask {
                 instance,
                 compilerCache,
                 sources.toArray,
-                project.compilationClasspath,
+                classpath,
                 graphInputs.store,
                 project.classesDir,
                 project.out,
@@ -144,8 +145,8 @@ object CompilationTask {
       }
     }
 
-    def setup(project: Project): CompileBundle = CompileBundle(project)
-    CompileGraph.traverse(dag, setup(_), compile(_), pipeline, logger).flatMap { partialDag =>
+    def setup(project: Project, dag: Dag[Project]): CompileBundle = CompileBundle(project, dag)
+    CompileGraph.traverse(dag, setup(_, _), compile(_), pipeline, logger).flatMap { partialDag =>
       val partialResults = Dag.dfs(partialDag)
       val finalResults = partialResults.map(r => PartialCompileResult.toFinalResult(r))
       Task.gatherUnordered(finalResults).map(_.flatten).map { results =>

frontend/src/main/scala/bloop/engine/tasks/LinkTask.scala

@@ -16,22 +16,26 @@ object LinkTask {
       state: State,
       mainClass: String,
       target: AbsolutePath,
-      platform: Platform.Js,
+      platform: Platform.Js
   ): Task[State] = {
     val config0 = platform.config
     platform.toolchain match {
       case Some(toolchain) =>
         config0.output.flatMap(Tasks.reasonOfInvalidPath(_, ".js")) match {
           case Some(msg) => Task.now(state.withError(msg, ExitStatus.LinkingError))
           case None =>
+            val fullClasspath =
+              project.dependencyClasspath(state.build.getDagFor(project)).map(_.underlying)
             val config = config0.copy(mode = getOptimizerMode(cmd.optimize, config0.mode))
-            toolchain.link(config, project, true, Some(mainClass), target, state.logger).map {
-              case scala.util.Success(_) =>
-                state.withInfo(s"Generated JavaScript file '${target.syntax}'")
-              case scala.util.Failure(t) =>
-                val msg = Feedback.failedToLink(project, ScalaJsToolchain.name, t)
-                state.withError(msg, ExitStatus.LinkingError).withTrace(t)
-            }
+            toolchain
+              .link(config, project, fullClasspath, true, Some(mainClass), target, state.logger)
+              .map {
+                case scala.util.Success(_) =>
+                  state.withInfo(s"Generated JavaScript file '${target.syntax}'")
+                case scala.util.Failure(t) =>
+                  val msg = Feedback.failedToLink(project, ScalaJsToolchain.name, t)
+                  state.withError(msg, ExitStatus.LinkingError).withTrace(t)
+              }
         }
       case None =>
         val artifactName = ScalaJsToolchain.artifactNameFrom(config0.version)
@@ -54,8 +58,10 @@ object LinkTask {
         config0.output.flatMap(Tasks.reasonOfInvalidPath(_)) match {
           case Some(msg) => Task.now(state.withError(msg, ExitStatus.LinkingError))
           case None =>
+            val fullClasspath =
+              project.dependencyClasspath(state.build.getDagFor(project)).map(_.underlying)
             val config = config0.copy(mode = getOptimizerMode(cmd.optimize, config0.mode))
-            toolchain.link(config, project, mainClass, target, state.logger) map {
+            toolchain.link(config, project, fullClasspath, mainClass, target, state.logger) map {
               case scala.util.Success(_) =>
                 state.withInfo(s"Generated native binary '${target.syntax}'")
               case scala.util.Failure(t) =>

frontend/src/main/scala/bloop/engine/tasks/Tasks.scala

@@ -54,7 +54,7 @@ object Tasks {
     import state.logger
     project.scalaInstance match {
       case Some(instance) =>
-        val classpath = project.fullClasspathFor(state.build.getDagFor(project))
+        val classpath = project.dependencyClasspath(state.build.getDagFor(project))
         val entries = classpath.map(_.underlying.toFile).toSeq
         logger.debug(s"Setting up the console classpath with ${entries.mkString(", ")}")(
           DebugFilter.All)
@@ -180,7 +180,7 @@ object Tasks {
       args: Array[String],
       skipJargs: Boolean
   ): Task[State] = {
-    val classpath = project.fullClasspathFor(state.build.getDagFor(project))
+    val classpath = project.dependencyClasspath(state.build.getDagFor(project))
     val processConfig = Forker(javaEnv, classpath)
     val runTask =
       processConfig.runMain(cwd, fqn, args, skipJargs, state.logger, state.commonOptions)

frontend/src/main/scala/bloop/engine/tasks/TestTask.scala

@@ -59,8 +59,8 @@ object TestTask {
 
         val lastCompileResult = state.results.lastSuccessfulResultOrEmpty(project)
         val analysis = lastCompileResult.analysis().toOption.getOrElse {
-          logger.warn(
-            s"Test execution was triggered, but no compilation detected for ${project.name}")
+          logger
+            .warn(s"Test execution was triggered, but no compilation detected for ${project.name}")
           Analysis.empty
         }
 
@@ -141,32 +141,39 @@ object TestTask {
     implicit val logContext: DebugFilter = DebugFilter.Test
     project.platform match {
       case Platform.Jvm(env, _, _) =>
-        val classpath = project.fullClasspathFor(state.build.getDagFor(project))
+        val classpath = project.dependencyClasspath(state.build.getDagFor(project))
         val forker = Forker(env, classpath)
         val testLoader = forker.newClassLoader(Some(TestInternals.filteredLoader))
-        val frameworks = project.testFrameworks.flatMap(f =>
-          TestInternals.loadFramework(testLoader, f.names, logger))
+        val frameworks = project.testFrameworks.flatMap(
+          f => TestInternals.loadFramework(testLoader, f.names, logger)
+        )
         Task.now(Some(DiscoveredTestFrameworks.Jvm(frameworks, forker, testLoader)))
 
       case Platform.Js(config, toolchain, userMainClass) =>
         val target = ScalaJsToolchain.linkTargetFrom(project, config)
         toolchain match {
           case Some(toolchain) =>
-            toolchain.link(config, project, false, userMainClass, target, state.logger).map {
-              case Success(_) =>
-                logger.info(s"Generated JavaScript file '${target.syntax}'")
-                val fnames = project.testFrameworks.map(_.names)
-                logger.debug(s"Resolving test frameworks: $fnames")
-                val baseDir = project.baseDirectory
-                val env = state.commonOptions.env.toMap
-                Some(toolchain.discoverTestFrameworks(project, fnames, target, logger, config, env))
+            val fullClasspath =
+              project.dependencyClasspath(state.build.getDagFor(project)).map(_.underlying)
+            toolchain
+              .link(config, project, fullClasspath, false, userMainClass, target, state.logger)
+              .map {
+                case Success(_) =>
+                  logger.info(s"Generated JavaScript file '${target.syntax}'")
+                  val fnames = project.testFrameworks.map(_.names)
+                  logger.debug(s"Resolving test frameworks: $fnames")
+                  val baseDir = project.baseDirectory
+                  val env = state.commonOptions.env.toMap
+                  Some(
+                    toolchain.discoverTestFrameworks(project, fnames, target, logger, config, env)
+                  )
 
-              case Failure(ex) =>
-                ex.printStackTrace()
-                logger.trace(ex)
-                logger.error(s"JavaScript linking failed with '${ex.getMessage}'")
-                None
-            }
+                case Failure(ex) =>
+                  ex.printStackTrace()
+                  logger.trace(ex)
+                  logger.error(s"JavaScript linking failed with '${ex.getMessage}'")
+                  None
+              }
 
           case None =>
             val artifactName = ScalaJsToolchain.artifactNameFrom(config.version)

frontend/src/main/scala/bloop/engine/tasks/compilation/CompileBundle.scala

@@ -2,6 +2,7 @@ package bloop.engine.tasks.compilation
 
 import bloop.data.Project
 import bloop.engine.Feedback
+import bloop.engine.Dag
 import bloop.io.{AbsolutePath, Paths}
 import bloop.{Compiler, ScalaInstance}
 
@@ -27,8 +28,9 @@ import bloop.{Compiler, ScalaInstance}
  */
 final case class CompileBundle(
     project: Project,
+    classpath: Array[AbsolutePath],
     javaSources: List[AbsolutePath],
-    scalaSources: List[AbsolutePath],
+    scalaSources: List[AbsolutePath]
 ) {
   val isJavaOnly: Boolean = scalaSources.isEmpty && !javaSources.isEmpty
 
@@ -70,10 +72,11 @@ case class CompileSourcesAndInstance(
 )
 
 object CompileBundle {
-  def apply(project: Project): CompileBundle = {
+  def apply(project: Project, dag: Dag[Project]): CompileBundle = {
     val sources = project.sources.distinct
+    val classpath = project.dependencyClasspath(dag)
     val javaSources = sources.flatMap(src => Paths.pathFilesUnder(src, "glob:**.java")).distinct
     val scalaSources = sources.flatMap(src => Paths.pathFilesUnder(src, "glob:**.scala")).distinct
-    new CompileBundle(project, javaSources, scalaSources)
+    new CompileBundle(project, classpath, javaSources, scalaSources)
   }
 }

frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala

@@ -40,7 +40,7 @@ object CompileGraph {
    */
   def traverse(
       dag: Dag[Project],
-      setup: Project => CompileBundle,
+      setup: (Project, Dag[Project]) => CompileBundle,
       compile: Inputs => Task[Compiler.Result],
       pipeline: Boolean,
       logger: Logger
@@ -101,7 +101,7 @@ object CompileGraph {
    */
   private def normalTraversal(
       dag: Dag[Project],
-      setup: Project => CompileBundle,
+      setup: (Project, Dag[Project]) => CompileBundle,
       compile: Inputs => Task[Compiler.Result],
       logger: Logger
   ): CompileTask = {
@@ -125,7 +125,7 @@ object CompileGraph {
         case None =>
           val task: Task[Dag[PartialCompileResult]] = dag match {
             case Leaf(project) =>
-              val bundle = setup(project)
+              val bundle = setup(project, dag)
               val cf = new CompletableFuture[IRs]()
               compile(Inputs(bundle, es, cf, JavaCompleted, JavaContinue, CompilerOracle.empty, false)).map {
                 case Compiler.Result.Ok(res) =>
@@ -140,7 +140,7 @@ object CompileGraph {
               }
 
             case Parent(project, dependencies) =>
-              val bundle = setup(project)
+              val bundle = setup(project, dag)
               val downstream = dependencies.map(loop)
               Task.gatherUnordered(downstream).flatMap { dagResults =>
                 val failed = dagResults.flatMap(dag => blockedBy(dag).toList)
@@ -192,7 +192,7 @@ object CompileGraph {
    */
   private def pipelineTraversal(
       dag: Dag[Project],
-      setup: Project => CompileBundle,
+      setup: (Project, Dag[Project]) => CompileBundle,
       compile: Inputs => Task[Compiler.Result],
       logger: Logger
   ): CompileTask = {
@@ -207,7 +207,7 @@ object CompileGraph {
           val task = dag match {
             case Leaf(project) =>
               Task.now(new CompletableFuture[IRs]()).flatMap { cf =>
-                val bundle = setup(project)
+                val bundle = setup(project, dag)
                 val jcf = new CompletableFuture[Unit]()
                 val t = compile(Inputs(bundle, es, cf, jcf, JavaContinue, CompilerOracle.empty, true))
                 val running =
@@ -237,7 +237,7 @@ object CompileGraph {
               }
 
             case Parent(project, dependencies) =>
-              val bundle = setup(project)
+              val bundle = setup(project, dag)
               val downstream = dependencies.map(loop)
               Task.gatherUnordered(downstream).flatMap { dagResults =>
                 val failed = dagResults.flatMap(dag => blockedBy(dag).toList)
@@ -252,7 +252,7 @@ object CompileGraph {
 
                   // Let's order the IRs exactly in the same order as provided in the classpath!
                   // Required for symbol clashes in dependencies (`AppLoader` in guardian/frontend)
-                  val indexDirs = project.compilationClasspath.iterator.filter(_.isDirectory).zipWithIndex.toMap
+                  val indexDirs = bundle.classpath.iterator.filter(_.isDirectory).zipWithIndex.toMap
                   val dependentStore = {
                     val transitiveStores =
                       results.flatMap(r => indexDirs.get(r.bundle.project.classesDir).iterator.map(i => i -> r.store))