Merge pull request #635 from scalacenter/topic/fix-master-pipelined

Enable community build under the pipelined mode

.drone.yml

@@ -20,6 +20,11 @@ matrix:
           "gradleBloop212/test" \
           "docs/makeSite"
 
+  RUN_COMMUNITY_BUILD_EXTRA:
+    - bin/sbt-ci.sh \
+          "frontend/integrationSetUpBloop" \
+          "frontend/testOnly bloop.tasks.IntegrationTestSuite"
+
   RUN_BENCHMARKS_AND_SCRIPTED:
     - bin/sbt-ci.sh \
           "sbtBloop10/scripted" \
@@ -41,6 +46,7 @@ matrix:
     - linux
   OPS:
     - basic
+    - community-build
     - extra
 
 clone:
@@ -85,7 +91,7 @@ pipeline:
     commands:
       - export DRONE_DIR="/drone"
       - git log | head -n 20
-      - . bin/detect-community-build.sh # We need to source it for the env variable to be exported
+      - . bin/detect-community-build.sh # Source it because it exports variables
       - ${TEST_BLOOP}
 
   run_benchmarks_scripted:
@@ -101,6 +107,21 @@ pipeline:
       - git log | head -n 20
       - ${RUN_BENCHMARKS_AND_SCRIPTED}
 
+#  run_community_pipelined:
+#    image: scalacenter/scala-docs:1.3
+#    group: build
+#    when:
+#      ref: [ refs/heads/master, refs/tags/*, refs/pull/*/head ]
+#      matrix:
+#        OS: linux
+#        OPS: community-build
+#    commands:
+#      - export DRONE_DIR="/drone"
+#      - export PIPELINE_COMMUNITY_BUILD="true"
+#      - git log | head -n 20
+#      - . bin/detect-community-build.sh # Source it because it exports variables
+#      - ${RUN_COMMUNITY_BUILD_EXTRA}
+
   build_windows:
     group: build
     image: scalacenter/scala-docs:1.0

build-integrations/sbt-1.0-2/build.sbt

@@ -1,5 +1,6 @@
 val Scalajs1 = Integrations.Scalajs1
-val integrations = List(Scalajs1)
+val Atlas = Integrations.Atlas
+val integrations = List(Scalajs1, Atlas)
 
 import bloop.build.integrations.PluginKeys
 val dummy = project
@@ -11,13 +12,15 @@ val dummy = project
     integrationIndex := {
       Map(
         "scalajs-1" -> bloopConfigDir.in(Scalajs1).in(Compile).value,
+        "atlas" -> bloopConfigDir.in(Atlas).in(Compile).value,
       )
     },
     cleanAllBuilds := {
       // Do it sequentially, there seems to be a race condition in windows
       Def.sequential(
         cleanAllBuilds,
-        clean.in(Scalajs1)
+        clean.in(Scalajs1),
+        clean.in(Atlas),
       )
     }
   )

build-integrations/sbt-1.0-2/project/Integrations.scala

@@ -3,4 +3,6 @@ import sbt.{RootProject, uri}
 object Integrations {
   val Scalajs1 = RootProject(
     uri("git://github.com/jvican/cross-platform-scalajs.git#0e776ae4b8ef5b9fae7f2374db01b227dc1ad9fc"))
+  val Atlas = RootProject(
+    uri("git://github.com/scalacenter/atlas.git#ab4251e17a2a02f2d5fe9c7ee83ebcde132c354c"))
 }

build-integrations/sbt-1.0/project/Integrations.scala

@@ -5,7 +5,7 @@ object Integrations {
   val SbtSbt = RootProject(
     uri("git://github.com/scalacenter/sbt.git#9d60c11e6bbd09f4b7e3976a7bc4a5888f96ca37"))
   val GuardianFrontend = RootProject(
-    uri("git://github.com/scalacenter/frontend.git#674c77745cb0d3959a04f90b2d9d57fdb8723c64"))
+    uri("git://github.com/scalacenter/frontend.git#fd8da1929d8a3bd39ca6027ffba6c0850e036ce3"))
   val MiniBetterFiles = RootProject(uri(
     "git://github.com/scalacenter/mini-better-files.git#0ed848993a2fd5a36e4366b5efb9c68dce958fc2"))
   val WithResources = RootProject(

frontend/src/main/scala/bloop/engine/tasks/Pipelined.scala

@@ -103,11 +103,11 @@ object Pipelined {
     def toInputs(
         inputs: PipelineInputs,
         instance: ScalaInstance,
+        sources: Array[AbsolutePath],
         config: ReporterConfig,
         result: PreviousResult
     ): CompileInputs = {
       val project = inputs.project
-      val sources = project.sources.toArray
       val classpath = project.classpath
       val picklepath = inputs.picklepath
       val classesDir = project.classesDir
@@ -154,8 +154,8 @@ object Pipelined {
       startTimings += (project -> System.currentTimeMillis())
       val previous = state.results.lastSuccessfulResult(project)
 
-      def runCompile(instance: ScalaInstance) = {
-        Compiler.compile(toInputs(inputs, instance, reporterConfig, previous)).map { result =>
+      def runCompile(instance: ScalaInstance, sources: Array[AbsolutePath]) = {
+        Compiler.compile(toInputs(inputs, instance, sources, reporterConfig, previous)).map { result =>
           // Do some book-keeping before returning the result to the caller
           endTimings += (project -> System.currentTimeMillis())
 
@@ -189,7 +189,7 @@ object Pipelined {
 
       def err(msg: String): Problem = Problem(-1, Severity.Error, msg, JavaNoPosition, "")
       project.scalaInstance match {
-        case Some(instance) => runCompile(instance)
+        case Some(instance) => runCompile(instance, uniqueSources)
         case None =>
           val addScalaConfiguration = err(
             "Add Scala configuration to the project. If that doesn't fix it, report it upstream")

frontend/src/test/scala/bloop/tasks/IntegrationTestSuite.scala

@@ -32,7 +32,12 @@ object IntegrationTestSuite {
 class IntegrationTestSuite(testDirectory: Path) {
   val integrationTestName = TestUtil.getBaseFromConfigDir(testDirectory).getFileName.toString
 
-  def isCommunityBuildEnabled: Boolean = {
+  val isCommunityBuildEnabled: Boolean =
+    isEnvironmentEnabled(List("RUN_COMMUNITY_BUILD", "run.community.build"), "false")
+  val isPipeliningEnabled: Boolean =
+    isEnvironmentEnabled(List("PIPELINE_COMMUNITY_BUILD", "pipeline.community.build"), "false")
+
+  private def isEnvironmentEnabled(keys: List[String], default: String): Boolean = {
     import scala.util.Try
     def bool(v: String): Boolean = {
       Try(java.lang.Boolean.parseBoolean(v)) match {
@@ -43,16 +48,21 @@ class IntegrationTestSuite(testDirectory: Path) {
       }
     }
 
-    bool(sys.env.getOrElse("RUN_COMMUNITY_BUILD", "false")) ||
-    bool(sys.props.getOrElse("run.community.build", "false"))
+    keys.exists(k => bool(sys.env.getOrElse(k, default)))
   }
 
   @Test
   def compileProject: Unit = {
-    if (!isCommunityBuildEnabled) () else compileProject0
+    if (!isCommunityBuildEnabled)
+      println(s"Skipping ${testDirectory} (community build is disabled)")
+    else {
+      if (isPipeliningEnabled) println(s"*** PIPELINING COMPILATION FOR ${testDirectory} ***")
+      else println(s"*** NORMAL COMPILATION FOR ${testDirectory} ***")
+      // After reporting the state of the execution, compile the projects accordingly.
+      compileProject0
+    }
   }
 
-
   def compileProject0: Unit = {
     val state0 = TestUtil.loadTestProject(testDirectory, integrationTestName, identity)
     val (initialState, projectToCompile) = getModuleToCompile(testDirectory) match {
@@ -104,8 +114,16 @@ class IntegrationTestSuite(testDirectory: Path) {
                  TestUtil.noPreviousResult(p, state))
     }
 
-    val action =
-      Run(Commands.Compile(projectToCompile.name, incremental = true, pipelined = false), Exit(ExitStatus.Ok))
+    val enablePipelining = isPipeliningEnabled
+    val action = Run(
+      Commands.Compile(
+        projectToCompile.name,
+        incremental = true,
+        pipelined = isPipeliningEnabled
+      ),
+      Exit(ExitStatus.Ok)
+    )
+
     val state1 = TestUtil.blockingExecute(action, state)
     reachable.foreach { p =>
       assertTrue(s"Project `$integrationTestName/${p.name}` has not been compiled.",