Delete betasty in newClassesDir, completely restart BE compilations a…

…fter an error

backend/src/main/scala/bloop/Compiler.scala

@@ -45,6 +45,7 @@ import bloop.Compiler.Result.Failed
 import bloop.util.BestEffortUtils
 import bloop.util.BestEffortUtils.BestEffortProducts
 import bloop.rtexport.RtJarCache
+import java.nio.file.Paths
 
 case class CompileInputs(
     scalaInstance: ScalaInstance,
@@ -249,7 +250,8 @@ object Compiler {
   def compile(
       compileInputs: CompileInputs,
       isBestEffortMode: Boolean,
-      isBestEffortDep: Boolean
+      isBestEffortDep: Boolean,
+      firstCompilation: Boolean
   ): Task[Result] = {
     val logger = compileInputs.logger
     val tracer = compileInputs.tracer
@@ -290,6 +292,11 @@ object Compiler {
       )
     }
 
+    val previousWasBestEffort = compileInputs.previousCompilerResult match {
+      case Failed(_, _, _, _, Some(BestEffortProducts(_, _, _))) => true
+      case _ => !firstCompilation
+    }
+
     val isFatalWarningsEnabled: Boolean =
       compileInputs.scalacOptions.exists(_ == "-Xfatal-warnings")
     def getInputs(compilers: Compilers): Inputs = {
@@ -377,7 +384,9 @@ object Compiler {
             t,
             elapsed,
             _,
-            bestEffortProducts @ Some(BestEffortProducts(previousCompilationResults, previousHash))
+            bestEffortProducts @ Some(
+              BestEffortProducts(previousCompilationResults, previousHash, _)
+            )
           ) if isBestEffortMode =>
         val newHash = BestEffortUtils.hashResult(
           previousCompilationResults.newClassesDir,
@@ -468,7 +477,8 @@ object Compiler {
             allInvalidatedClassFilesForProject,
             allInvalidatedExtraCompileProducts,
             previousSuccessfulProblems,
-            None
+            errorCause = None,
+            previousWasBestEffort
           )
         case Success(result) =>
           // Report end of compilation only after we have reported all warnings from previous runs
@@ -494,16 +504,14 @@ object Compiler {
             Task(persist(out, analysis, result.setup, tracer, logger)).memoize
           }
 
-          // betasty files are always produced with -Ybest-effort, even when
+          // .betasty files are always produced with -Ybest-effort, even when
           // the compilation is successful.
-          // We might want to change this in the commpiler itself...
-          // Alternatively, whether downstream projects use betasty can be
-          // controlled with -Ywith-best-effort-tasty
-          val deleteBestEffortDir =
+          // We might want to change this in the compiler itself...
+          def deleteBestEffortDir() =
             if (isBestEffortMode)
               Task(
                 BloopPaths
-                  .delete(compileOut.internalReadOnlyClassesDir.resolve("META-INF/best-effort"))
+                  .delete(compileOut.internalNewClassesDir.resolve("META-INF/best-effort"))
               )
             else Task {}
 
@@ -568,7 +576,8 @@ object Compiler {
                   )
                 }
                   .flatMap(clientClassesObserver.nextAnalysis)
-                Task
+
+                deleteBestEffortDir() *> Task
                   .gatherUnordered(
                     List(
                       deleteBestEffortDir,
@@ -624,11 +633,13 @@ object Compiler {
               ): Task[Unit] = {
                 val clientClassesDir = clientClassesObserver.classesDir
                 val successBackgroundTasks =
-                  backgroundTasksWhenNewSuccessfulAnalysis
-                    .map(f => f(clientClassesDir, clientReporter, clientTracer))
+                  deleteBestEffortDir() *> Task.gatherUnordered(
+                    backgroundTasksWhenNewSuccessfulAnalysis
+                      .map(f => f(clientClassesDir, clientReporter, clientTracer))
+                  )
                 val persistTask =
                   persistAnalysis(analysisForFutureCompilationRuns, compileOut.analysisOut)
-                val initialTasks = persistTask :: successBackgroundTasks.toList
+                val initialTasks = List(persistTask, successBackgroundTasks)
                 val allClientSyncTasks = Task.gatherUnordered(initialTasks).flatMap { _ =>
                   // Only start these tasks after the previous IO tasks in the external dir are done
                   val firstTask = updateExternalClassesDirWithReadOnly(
@@ -666,7 +677,7 @@ object Compiler {
                     )
                   }.flatMap(clientClassesObserver.nextAnalysis)
                   Task
-                    .gatherUnordered(List(deleteBestEffortDir, firstTask, secondTask))
+                    .gatherUnordered(List(firstTask, secondTask))
                     .flatMap(_ => publishClientAnalysis)
                 }
 
@@ -707,7 +718,8 @@ object Compiler {
             allInvalidatedClassFilesForProject,
             allInvalidatedExtraCompileProducts,
             previousSuccessfulProblems,
-            Some(cause)
+            errorCause = Some(cause),
+            previousWasBestEffort
           )
 
         case Failure(_: xsbti.CompileCancelled) => handleCancellation
@@ -912,7 +924,8 @@ object Compiler {
       allInvalidatedClassFilesForProject: mutable.HashSet[File],
       allInvalidatedExtraCompileProducts: mutable.HashSet[File],
       previousSuccessfulProblems: List[ProblemPerPhase],
-      errorCause: Option[xsbti.CompileFailed]
+      errorCause: Option[xsbti.CompileFailed],
+      previousWasBestEffort: Boolean
   ): Result = {
     val uniqueInputs = compileInputs.uniqueInputs
     val readOnlyClassesDir = compileOut.internalReadOnlyClassesDir.underlying
@@ -952,49 +965,52 @@ object Compiler {
           backgroundTasksWhenNewSuccessfulAnalysis
             .map(f => f(clientClassesDir, clientReporter, clientTracer))
         val allClientSyncTasks = Task.gatherUnordered(successBackgroundTasks.toList).flatMap { _ =>
-          // Only start these tasks after the previous IO tasks in the external dir are done
-          val firstTask = updateExternalClassesDirWithReadOnly(
-            clientClassesDir,
-            clientTracer,
-            clientLogger,
-            compileInputs,
-            readOnlyClassesDir,
-            readOnlyCopyDenylist = mutable.HashSet.empty,
-            allInvalidatedClassFilesForProject,
-            allInvalidatedExtraCompileProducts
-          )
-
-          val secondTask = Task {
+          // Only start this task after the previous IO tasks in the external dir are done
+          Task {
             // Delete everything outside of betasty and semanticdb
             val deletedCompileProducts =
               BloopClassFileManager.supportedCompileProducts.filter(_ != ".betasty") :+ ".class"
             Files
               .walk(clientClassesDir.underlying)
-              .filter(path => Files.isRegularFile(path))
+              .filter(path => if (Files.exists(path)) Files.isRegularFile(path) else false)
               .filter(path => deletedCompileProducts.exists(path.toString.endsWith(_)))
-              .forEach(Files.delete(_))
-          }
-          Task
-            .gatherUnordered(List(firstTask, secondTask))
-            .map(_ => ())
+              .forEach(path => if (Files.exists(path)) Files.delete(path))
+          }.map(_ => ())
         }
 
         allClientSyncTasks.doOnFinish(_ => Task(clientReporter.reportEndCompilation()))
       }
     }
 
-    val newHash = BestEffortUtils.hashResult(
-      products.newClassesDir,
-      compileInputs.sources,
-      compileInputs.classpath
-    )
+    val recompile =
+      if (
+        !previousWasBestEffort && !(compileOut.internalReadOnlyClassesDir.exists && BloopPaths
+          .list(compileOut.internalReadOnlyClassesDir)
+          .length == 0)
+      ) {
+        if (compileOut.analysisOut.exists) BloopPaths.delete(compileOut.analysisOut)
+        BloopPaths.delete(compileOut.internalReadOnlyClassesDir)
+        Files.createDirectories(Paths.get(compileOut.internalReadOnlyClassesDir.toString))
+        BloopPaths.delete(compileOut.internalNewClassesDir)
+        Files.createDirectories(Paths.get(compileOut.internalNewClassesDir.toString))
+        true
+      } else false
+
+    val newHash =
+      if (previousWasBestEffort)
+        BestEffortUtils.hashResult(
+          products.newClassesDir,
+          compileInputs.sources,
+          compileInputs.classpath
+        )
+      else ""
     val failedProblems = findFailedProblems(reporter, errorCause)
     Result.Failed(
       failedProblems,
       None,
       elapsed(),
       backgroundTasksExecution,
-      Some(BestEffortProducts(products, newHash))
+      Some(BestEffortProducts(products, newHash, recompile))
     )
   }
 

backend/src/main/scala/bloop/util/BestEffortUtils.scala

@@ -10,7 +10,11 @@ import bloop.io.AbsolutePath
 
 object BestEffortUtils {
 
-  case class BestEffortProducts(compileProducts: bloop.CompileProducts, hash: String)
+  case class BestEffortProducts(
+      compileProducts: bloop.CompileProducts,
+      hash: String,
+      recompile: Boolean
+  )
 
   /* Hashes results of a projects compilation, to mimic how it would have been handled in zinc.
    * Returns SHA-1 of a project.

frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala

@@ -39,6 +39,11 @@ import bloop.util.BestEffortUtils.BestEffortProducts
 import monix.execution.CancelableFuture
 import monix.reactive.MulticastStrategy
 import monix.reactive.Observable
+import sbt.internal.inc.BloopComponentCompiler
+import xsbti.compile.PreviousResult
+import java.util.Optional
+import xsbti.compile.MiniSetup
+import xsbti.compile.CompileAnalysis
 
 object CompileTask {
   private implicit val logContext: DebugFilter = DebugFilter.Compilation
@@ -171,9 +176,72 @@ object CompileTask {
 
           // Block on the task associated with this result that sets up the read-only classes dir
           waitOnReadClassesDir.flatMap { _ =>
+            def getAllSourceInputs(project: Project): List[AbsolutePath] = {
+              import java.nio.file.Files
+              import scala.collection.JavaConverters._
+
+              val uniqueSourceDirs = project.sources
+
+              val sourceExts = Seq(".scala", ".java")
+              val unmanagedSources: mutable.Set[AbsolutePath] = mutable.Set()
+
+              uniqueSourceDirs.map(_.underlying).foreach { file =>
+                if (!Files.exists(file)) ()
+                else if (
+                  Files.isRegularFile(file) && sourceExts.exists(ext => file.toString.endsWith(ext))
+                ) {
+                  unmanagedSources.add(AbsolutePath(file))
+                } else if (Files.isDirectory(file)) {
+                  Files.walk(file).iterator().asScala.foreach { file =>
+                    if (
+                      Files.isRegularFile(file) && sourceExts
+                        .exists(ext => file.toString.endsWith(ext))
+                    ) {
+                      unmanagedSources.add(AbsolutePath(file))
+                    }
+                  }
+                }
+              }
+
+              project.sourcesGlobs.foreach { glob =>
+                Files.walk(glob.directory.underlying).iterator().asScala.foreach { file =>
+                  if (
+                    Files.isRegularFile(file) && sourceExts
+                      .exists(ext => file.toString.endsWith(ext)) && glob.matches(file)
+                  ) {
+                    unmanagedSources.add(AbsolutePath(file))
+                  }
+                }
+              }
+
+              unmanagedSources.toList
+            }
+
             // Only when the task is finished, we kickstart the compilation
-            def compile(inputs: CompileInputs) =
-              Compiler.compile(inputs, isBestEffort, isBestEffortDep)
+            def compile(inputs: CompileInputs) = {
+              val firstResult = Compiler.compile(inputs, isBestEffort, isBestEffortDep, true)
+              firstResult.flatMap {
+                case result @ Compiler.Result.Failed(
+                      _,
+                      _,
+                      _,
+                      _,
+                      Some(BestEffortProducts(_, _, recompile))
+                    ) if recompile =>
+                  // we restart the compilation, starting from scratch (without any previous artifacts)
+                  inputs.reporter.reset()
+                  val foundSrcs = getAllSourceInputs(project)
+                  val emptyResult =
+                    PreviousResult.of(Optional.empty[CompileAnalysis], Optional.empty[MiniSetup])
+                  val newInputs = inputs.copy(
+                    sources = foundSrcs.toArray,
+                    previousCompilerResult = result,
+                    previousResult = emptyResult
+                  )
+                  Compiler.compile(newInputs, isBestEffort, isBestEffortDep, false)
+                case result => Task(result)
+              }
+            }
             inputs.flatMap(inputs => compile(inputs)).map { result =>
               def runPostCompilationTasks(
                   backgroundTasks: CompileBackgroundTasks
@@ -467,10 +535,10 @@ object CompileTask {
           logger.debug(s"Scheduling to delete ${previousClassesDir} superseded by $newClassesDir")
           Some(previousClassesDir)
         }
-      case Failed(_, _, _, _, Some(BestEffortProducts(products, _))) =>
+      case Failed(_, _, _, _, Some(BestEffortProducts(products, _, _))) =>
         val newClassesDir = products.newClassesDir
         previousResult match {
-          case Some(Failed(_, _, _, _, Some(BestEffortProducts(previousProducts, _)))) =>
+          case Some(Failed(_, _, _, _, Some(BestEffortProducts(previousProducts, _, _)))) =>
             val previousClassesDir = previousProducts.newClassesDir
             if (previousClassesDir != newClassesDir) {
               logger.debug(

frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala

@@ -449,7 +449,7 @@ object CompileGraph {
                           .+=(newProducts.readOnlyClassesDir.toFile -> newResult)
                       case (p, ResultBundle(f: Compiler.Result.Failed, _, _, _)) =>
                         f.bestEffortProducts.foreach {
-                          case BestEffortProducts(products, _) =>
+                          case BestEffortProducts(products, _, _) =>
                             dependentProducts += (p -> Right(products))
                         }
                       case _ => ()

frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala

@@ -665,8 +665,9 @@ class BspMetalsClientSpec(
       )
       loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state =>
         val compiledState = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState
-        assertBetastyFile("A.betasty", compiledState, "A")
-        assertBetastyFile("B.betasty", compiledState, "A")
+        // we remove betasty from successful compilations
+        assertNoBetastyFile("A.betasty", compiledState, "A")
+        assertNoBetastyFile("B.betasty", compiledState, "A")
         assertCompilationFile("A.class", compiledState, "A")
         updateProject(updatedFile1WithError)
         val compiledState2 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState
@@ -677,8 +678,8 @@ class BspMetalsClientSpec(
         updateProject(updatedFile2WithoutError)
         val compiledState3 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState
         assertNoBetastyFile("A.betasty", compiledState3, "A")
-        assertBetastyFile("B.betasty", compiledState3, "A")
-        assertBetastyFile("C.betasty", compiledState3, "A")
+        assertNoBetastyFile("B.betasty", compiledState3, "A")
+        assertNoBetastyFile("C.betasty", compiledState3, "A")
         assertCompilationFile("B.class", compiledState, "A")
         updateProject(updatedFile3WithError)
         val compiledState4 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState