diff --git a/backend/src/main/scala/bloop/Compiler.scala b/backend/src/main/scala/bloop/Compiler.scala index ae64018ba..39f0dbd86 100644 --- a/backend/src/main/scala/bloop/Compiler.scala +++ b/backend/src/main/scala/bloop/Compiler.scala @@ -45,6 +45,7 @@ import bloop.Compiler.Result.Failed import bloop.util.BestEffortUtils import bloop.util.BestEffortUtils.BestEffortProducts import bloop.rtexport.RtJarCache +import java.nio.file.Paths case class CompileInputs( scalaInstance: ScalaInstance, @@ -249,7 +250,8 @@ object Compiler { def compile( compileInputs: CompileInputs, isBestEffortMode: Boolean, - isBestEffortDep: Boolean + isBestEffortDep: Boolean, + firstCompilation: Boolean ): Task[Result] = { val logger = compileInputs.logger val tracer = compileInputs.tracer @@ -290,6 +292,11 @@ object Compiler { ) } + val previousWasBestEffort = compileInputs.previousCompilerResult match { + case Failed(_, _, _, _, Some(BestEffortProducts(_, _, _))) => true + case _ => !firstCompilation + } + val isFatalWarningsEnabled: Boolean = compileInputs.scalacOptions.exists(_ == "-Xfatal-warnings") def getInputs(compilers: Compilers): Inputs = { @@ -377,7 +384,9 @@ object Compiler { t, elapsed, _, - bestEffortProducts @ Some(BestEffortProducts(previousCompilationResults, previousHash)) + bestEffortProducts @ Some( + BestEffortProducts(previousCompilationResults, previousHash, _) + ) ) if isBestEffortMode => val newHash = BestEffortUtils.hashResult( previousCompilationResults.newClassesDir, @@ -468,7 +477,8 @@ object Compiler { allInvalidatedClassFilesForProject, allInvalidatedExtraCompileProducts, previousSuccessfulProblems, - None + errorCause = None, + previousWasBestEffort ) case Success(result) => // Report end of compilation only after we have reported all warnings from previous runs @@ -494,16 +504,14 @@ object Compiler { Task(persist(out, analysis, result.setup, tracer, logger)).memoize } - // betasty files are always produced with -Ybest-effort, even when + // .betasty files are always produced with -Ybest-effort, even when // the compilation is successful. - // We might want to change this in the commpiler itself... - // Alternatively, whether downstream projects use betasty can be - // controlled with -Ywith-best-effort-tasty - val deleteBestEffortDir = + // We might want to change this in the compiler itself... + def deleteBestEffortDir() = if (isBestEffortMode) Task( BloopPaths - .delete(compileOut.internalReadOnlyClassesDir.resolve("META-INF/best-effort")) + .delete(compileOut.internalNewClassesDir.resolve("META-INF/best-effort")) ) else Task {} @@ -568,7 +576,8 @@ object Compiler { ) } .flatMap(clientClassesObserver.nextAnalysis) - Task + + deleteBestEffortDir() *> Task .gatherUnordered( List( deleteBestEffortDir, @@ -624,11 +633,13 @@ object Compiler { ): Task[Unit] = { val clientClassesDir = clientClassesObserver.classesDir val successBackgroundTasks = - backgroundTasksWhenNewSuccessfulAnalysis - .map(f => f(clientClassesDir, clientReporter, clientTracer)) + deleteBestEffortDir() *> Task.gatherUnordered( + backgroundTasksWhenNewSuccessfulAnalysis + .map(f => f(clientClassesDir, clientReporter, clientTracer)) + ) val persistTask = persistAnalysis(analysisForFutureCompilationRuns, compileOut.analysisOut) - val initialTasks = persistTask :: successBackgroundTasks.toList + val initialTasks = List(persistTask, successBackgroundTasks) val allClientSyncTasks = Task.gatherUnordered(initialTasks).flatMap { _ => // Only start these tasks after the previous IO tasks in the external dir are done val firstTask = updateExternalClassesDirWithReadOnly( @@ -666,7 +677,7 @@ object Compiler { ) }.flatMap(clientClassesObserver.nextAnalysis) Task - .gatherUnordered(List(deleteBestEffortDir, firstTask, secondTask)) + .gatherUnordered(List(firstTask, secondTask)) .flatMap(_ => publishClientAnalysis) } @@ -707,7 +718,8 @@ object Compiler { allInvalidatedClassFilesForProject, allInvalidatedExtraCompileProducts, previousSuccessfulProblems, - Some(cause) + errorCause = Some(cause), + previousWasBestEffort ) case Failure(_: xsbti.CompileCancelled) => handleCancellation @@ -912,7 +924,8 @@ object Compiler { allInvalidatedClassFilesForProject: mutable.HashSet[File], allInvalidatedExtraCompileProducts: mutable.HashSet[File], previousSuccessfulProblems: List[ProblemPerPhase], - errorCause: Option[xsbti.CompileFailed] + errorCause: Option[xsbti.CompileFailed], + previousWasBestEffort: Boolean ): Result = { val uniqueInputs = compileInputs.uniqueInputs val readOnlyClassesDir = compileOut.internalReadOnlyClassesDir.underlying @@ -952,49 +965,52 @@ object Compiler { backgroundTasksWhenNewSuccessfulAnalysis .map(f => f(clientClassesDir, clientReporter, clientTracer)) val allClientSyncTasks = Task.gatherUnordered(successBackgroundTasks.toList).flatMap { _ => - // Only start these tasks after the previous IO tasks in the external dir are done - val firstTask = updateExternalClassesDirWithReadOnly( - clientClassesDir, - clientTracer, - clientLogger, - compileInputs, - readOnlyClassesDir, - readOnlyCopyDenylist = mutable.HashSet.empty, - allInvalidatedClassFilesForProject, - allInvalidatedExtraCompileProducts - ) - - val secondTask = Task { + // Only start this task after the previous IO tasks in the external dir are done + Task { // Delete everything outside of betasty and semanticdb val deletedCompileProducts = BloopClassFileManager.supportedCompileProducts.filter(_ != ".betasty") :+ ".class" Files .walk(clientClassesDir.underlying) - .filter(path => Files.isRegularFile(path)) + .filter(path => if (Files.exists(path)) Files.isRegularFile(path) else false) .filter(path => deletedCompileProducts.exists(path.toString.endsWith(_))) - .forEach(Files.delete(_)) - } - Task - .gatherUnordered(List(firstTask, secondTask)) - .map(_ => ()) + .forEach(path => if (Files.exists(path)) Files.delete(path)) + }.map(_ => ()) } allClientSyncTasks.doOnFinish(_ => Task(clientReporter.reportEndCompilation())) } } - val newHash = BestEffortUtils.hashResult( - products.newClassesDir, - compileInputs.sources, - compileInputs.classpath - ) + val recompile = + if ( + !previousWasBestEffort && !(compileOut.internalReadOnlyClassesDir.exists && BloopPaths + .list(compileOut.internalReadOnlyClassesDir) + .length == 0) + ) { + if (compileOut.analysisOut.exists) BloopPaths.delete(compileOut.analysisOut) + BloopPaths.delete(compileOut.internalReadOnlyClassesDir) + Files.createDirectories(Paths.get(compileOut.internalReadOnlyClassesDir.toString)) + BloopPaths.delete(compileOut.internalNewClassesDir) + Files.createDirectories(Paths.get(compileOut.internalNewClassesDir.toString)) + true + } else false + + val newHash = + if (previousWasBestEffort) + BestEffortUtils.hashResult( + products.newClassesDir, + compileInputs.sources, + compileInputs.classpath + ) + else "" val failedProblems = findFailedProblems(reporter, errorCause) Result.Failed( failedProblems, None, elapsed(), backgroundTasksExecution, - Some(BestEffortProducts(products, newHash)) + Some(BestEffortProducts(products, newHash, recompile)) ) } diff --git a/backend/src/main/scala/bloop/util/BestEffortUtils.scala b/backend/src/main/scala/bloop/util/BestEffortUtils.scala index 318507fe0..f29cdb6df 100644 --- a/backend/src/main/scala/bloop/util/BestEffortUtils.scala +++ b/backend/src/main/scala/bloop/util/BestEffortUtils.scala @@ -10,7 +10,11 @@ import bloop.io.AbsolutePath object BestEffortUtils { - case class BestEffortProducts(compileProducts: bloop.CompileProducts, hash: String) + case class BestEffortProducts( + compileProducts: bloop.CompileProducts, + hash: String, + recompile: Boolean + ) /* Hashes results of a projects compilation, to mimic how it would have been handled in zinc. * Returns SHA-1 of a project. diff --git a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala index 0366d8829..1f8042d85 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/CompileTask.scala @@ -39,6 +39,11 @@ import bloop.util.BestEffortUtils.BestEffortProducts import monix.execution.CancelableFuture import monix.reactive.MulticastStrategy import monix.reactive.Observable +import sbt.internal.inc.BloopComponentCompiler +import xsbti.compile.PreviousResult +import java.util.Optional +import xsbti.compile.MiniSetup +import xsbti.compile.CompileAnalysis object CompileTask { private implicit val logContext: DebugFilter = DebugFilter.Compilation @@ -171,9 +176,72 @@ object CompileTask { // Block on the task associated with this result that sets up the read-only classes dir waitOnReadClassesDir.flatMap { _ => + def getAllSourceInputs(project: Project): List[AbsolutePath] = { + import java.nio.file.Files + import scala.collection.JavaConverters._ + + val uniqueSourceDirs = project.sources + + val sourceExts = Seq(".scala", ".java") + val unmanagedSources: mutable.Set[AbsolutePath] = mutable.Set() + + uniqueSourceDirs.map(_.underlying).foreach { file => + if (!Files.exists(file)) () + else if ( + Files.isRegularFile(file) && sourceExts.exists(ext => file.toString.endsWith(ext)) + ) { + unmanagedSources.add(AbsolutePath(file)) + } else if (Files.isDirectory(file)) { + Files.walk(file).iterator().asScala.foreach { file => + if ( + Files.isRegularFile(file) && sourceExts + .exists(ext => file.toString.endsWith(ext)) + ) { + unmanagedSources.add(AbsolutePath(file)) + } + } + } + } + + project.sourcesGlobs.foreach { glob => + Files.walk(glob.directory.underlying).iterator().asScala.foreach { file => + if ( + Files.isRegularFile(file) && sourceExts + .exists(ext => file.toString.endsWith(ext)) && glob.matches(file) + ) { + unmanagedSources.add(AbsolutePath(file)) + } + } + } + + unmanagedSources.toList + } + // Only when the task is finished, we kickstart the compilation - def compile(inputs: CompileInputs) = - Compiler.compile(inputs, isBestEffort, isBestEffortDep) + def compile(inputs: CompileInputs) = { + val firstResult = Compiler.compile(inputs, isBestEffort, isBestEffortDep, true) + firstResult.flatMap { + case result @ Compiler.Result.Failed( + _, + _, + _, + _, + Some(BestEffortProducts(_, _, recompile)) + ) if recompile => + // we restart the compilation, starting from scratch (without any previous artifacts) + inputs.reporter.reset() + val foundSrcs = getAllSourceInputs(project) + val emptyResult = + PreviousResult.of(Optional.empty[CompileAnalysis], Optional.empty[MiniSetup]) + val newInputs = inputs.copy( + sources = foundSrcs.toArray, + previousCompilerResult = result, + previousResult = emptyResult + ) + Compiler.compile(newInputs, isBestEffort, isBestEffortDep, false) + case result => Task(result) + } + } inputs.flatMap(inputs => compile(inputs)).map { result => def runPostCompilationTasks( backgroundTasks: CompileBackgroundTasks @@ -467,10 +535,10 @@ object CompileTask { logger.debug(s"Scheduling to delete ${previousClassesDir} superseded by $newClassesDir") Some(previousClassesDir) } - case Failed(_, _, _, _, Some(BestEffortProducts(products, _))) => + case Failed(_, _, _, _, Some(BestEffortProducts(products, _, _))) => val newClassesDir = products.newClassesDir previousResult match { - case Some(Failed(_, _, _, _, Some(BestEffortProducts(previousProducts, _)))) => + case Some(Failed(_, _, _, _, Some(BestEffortProducts(previousProducts, _, _)))) => val previousClassesDir = previousProducts.newClassesDir if (previousClassesDir != newClassesDir) { logger.debug( diff --git a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala index 0fc070736..e43e738be 100644 --- a/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala +++ b/frontend/src/main/scala/bloop/engine/tasks/compilation/CompileGraph.scala @@ -449,7 +449,7 @@ object CompileGraph { .+=(newProducts.readOnlyClassesDir.toFile -> newResult) case (p, ResultBundle(f: Compiler.Result.Failed, _, _, _)) => f.bestEffortProducts.foreach { - case BestEffortProducts(products, _) => + case BestEffortProducts(products, _, _) => dependentProducts += (p -> Right(products)) } case _ => () diff --git a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala index 8f28bce51..955f687cb 100644 --- a/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala +++ b/frontend/src/test/scala/bloop/bsp/BspMetalsClientSpec.scala @@ -665,8 +665,9 @@ class BspMetalsClientSpec( ) loadBspState(workspace, projects, logger, "Metals", bloopExtraParams = extraParams) { state => val compiledState = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState - assertBetastyFile("A.betasty", compiledState, "A") - assertBetastyFile("B.betasty", compiledState, "A") + // we remove betasty from successful compilations + assertNoBetastyFile("A.betasty", compiledState, "A") + assertNoBetastyFile("B.betasty", compiledState, "A") assertCompilationFile("A.class", compiledState, "A") updateProject(updatedFile1WithError) val compiledState2 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState @@ -677,8 +678,8 @@ class BspMetalsClientSpec( updateProject(updatedFile2WithoutError) val compiledState3 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState assertNoBetastyFile("A.betasty", compiledState3, "A") - assertBetastyFile("B.betasty", compiledState3, "A") - assertBetastyFile("C.betasty", compiledState3, "A") + assertNoBetastyFile("B.betasty", compiledState3, "A") + assertNoBetastyFile("C.betasty", compiledState3, "A") assertCompilationFile("B.class", compiledState, "A") updateProject(updatedFile3WithError) val compiledState4 = state.compile(`A`, arguments = Some(List("--best-effort"))).toTestState