Skip to content

Commit

Permalink
Revert "Merge pull request scalacenter#2410 from jchyb/fix-leftover-b…
Browse files Browse the repository at this point in the history 
…etasty"

This reverts commit 57eb430, reversing
changes made to fa29730.
  • Loading branch information
@jchyb
jchyb committed Oct 30, 2024
1 parent 751a990 commit 82ff408
Show file tree
Hide file tree
Showing 8 changed files with 117 additions and 247 deletions.
42 changes: 29 additions & 13 deletions backend/src/main/scala/bloop/BloopClassFileManager.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,20 +219,36 @@ final class BloopClassFileManager(
clientTracer.traceTaskVerbose("copy new products to external classes dir") { _ =>
val config =
ParallelOps.CopyConfiguration(5, CopyMode.ReplaceExisting, Set.empty, Set.empty)

ParallelOps
.copyDirectories(config)(
newClassesDir,
clientExternalClassesDir.underlying,
inputs.ioScheduler,
enableCancellation = false,
inputs.logger
)
.map { walked =>
readOnlyCopyDenylist.++=(walked.target)
val clientExternalBestEffortDir =
clientExternalClassesDir.underlying.resolve("META-INF/best-effort")

// Deletes all previous best-effort artifacts to get rid of all of the outdated ones.
// Since best effort compilation is not affected by incremental compilation,
// all relevant files are always produced by the compiler. Because of this,
// we can always delete all previous files and copy newly created ones
// without losing anything in the process.
val deleteClientExternalBestEffortDir =
Task {
if (Files.exists(clientExternalBestEffortDir)) {
BloopPaths.delete(AbsolutePath(clientExternalBestEffortDir))
}
()
}
.flatMap(_ => deleteAfterCompilation)
}.memoize

deleteClientExternalBestEffortDir *>
ParallelOps
.copyDirectories(config)(
newClassesDir,
clientExternalClassesDir.underlying,
inputs.ioScheduler,
enableCancellation = false,
inputs.logger
)
.map { walked =>
readOnlyCopyDenylist.++=(walked.target)
()
}
.flatMap(_ => deleteAfterCompilation)
}
}
)
Expand Down
164 changes: 55 additions & 109 deletions backend/src/main/scala/bloop/Compiler.scala
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package bloop

import java.io.File
import java.io.PrintWriter
import java.io.StringWriter
import java.nio.file.Files
import java.nio.file.Path
import java.util.Optional
Expand Down Expand Up @@ -251,8 +249,7 @@ object Compiler {
def compile(
compileInputs: CompileInputs,
isBestEffortMode: Boolean,
isBestEffortDep: Boolean,
firstCompilation: Boolean
isBestEffortDep: Boolean
): Task[Result] = {
val logger = compileInputs.logger
val tracer = compileInputs.tracer
Expand Down Expand Up @@ -378,9 +375,6 @@ object Compiler {
reporter.reportStartCompilation(previousProblems, wasPreviousSuccessful)
val fileManager = newFileManager

val shouldAttemptRestartingCompilationForBestEffort =
firstCompilation && !isBestEffortDep && previousAnalysis.isDefined

// Manually skip redundant best-effort compilations. This is necessary because compiler
// phases supplying the data needed to skip compilations in zinc remain unimplemented for now.
val noopBestEffortResult = compileInputs.previousCompilerResult match {
Expand All @@ -389,9 +383,7 @@ object Compiler {
t,
elapsed,
_,
bestEffortProducts @ Some(
BestEffortProducts(previousCompilationResults, previousHash, _)
)
bestEffortProducts @ Some(BestEffortProducts(previousCompilationResults, previousHash))
) if isBestEffortMode =>
val newHash = BestEffortUtils.hashResult(
previousCompilationResults.newClassesDir,
Expand Down Expand Up @@ -466,8 +458,7 @@ object Compiler {
fileManager,
cancelPromise,
tracer,
classpathOptions,
!(isBestEffortMode && isBestEffortDep)
classpathOptions
)
.materialize
.doOnCancel(Task(cancel()))
Expand All @@ -480,9 +471,10 @@ object Compiler {
() => elapsed,
reporter,
backgroundTasksWhenNewSuccessfulAnalysis,
allInvalidatedClassFilesForProject,
allInvalidatedExtraCompileProducts,
previousSuccessfulProblems,
errorCause = None,
shouldAttemptRestartingCompilationForBestEffort
None
)
case Success(result) =>
// Report end of compilation only after we have reported all warnings from previous runs
Expand All @@ -508,17 +500,6 @@ object Compiler {
Task(persist(out, analysis, result.setup, tracer, logger)).memoize
}

// .betasty files are always produced with -Ybest-effort, even when
// the compilation is successful.
// We might want to change this in the compiler itself...
def deleteBestEffortDir() =
if (isBestEffortMode)
Task(
BloopPaths
.delete(compileOut.internalNewClassesDir.resolve("META-INF/best-effort"))
)
else Task {}

val isNoOp = previousAnalysis.contains(analysis)
if (isNoOp) {
// If no-op, return previous result with updated classpath hashes
Expand Down Expand Up @@ -548,25 +529,16 @@ object Compiler {
val clientClassesDir = clientClassesObserver.classesDir
clientLogger.debug(s"Triggering background tasks for $clientClassesDir")
val updateClientState =
Task
.gatherUnordered(
List(
deleteClientExternalBestEffortDirTask(clientClassesDir),
deleteBestEffortDir()
)
)
.flatMap { _ =>
updateExternalClassesDirWithReadOnly(
clientClassesDir,
clientTracer,
clientLogger,
compileInputs,
readOnlyClassesDir,
readOnlyCopyDenylist,
allInvalidatedClassFilesForProject,
allInvalidatedExtraCompileProducts
)
}
updateExternalClassesDirWithReadOnly(
clientClassesDir,
clientTracer,
clientLogger,
compileInputs,
readOnlyClassesDir,
readOnlyCopyDenylist,
allInvalidatedClassFilesForProject,
allInvalidatedExtraCompileProducts
)

val writeAnalysisIfMissing = {
if (compileOut.analysisOut.exists) Task.unit
Expand Down Expand Up @@ -600,8 +572,7 @@ object Compiler {
)
.flatMap(_ => publishClientAnalysis)
.onErrorHandleWith(err => {
clientLogger.debug("Caught error in background tasks");
clientLogger.trace(err);
clientLogger.debug("Caught error in background tasks"); clientLogger.trace(err);
Task.raiseError(err)
})
.doOnFinish(_ => Task(clientReporter.reportEndCompilation()))
Expand Down Expand Up @@ -645,22 +616,11 @@ object Compiler {
): Task[Unit] = {
val clientClassesDir = clientClassesObserver.classesDir
val successBackgroundTasks =
Task
.gatherUnordered(
List(
deleteBestEffortDir(),
deleteClientExternalBestEffortDirTask(clientClassesDir)
)
)
.flatMap { _ =>
Task.gatherUnordered(
backgroundTasksWhenNewSuccessfulAnalysis
.map(f => f(clientClassesDir, clientReporter, clientTracer))
)
}
backgroundTasksWhenNewSuccessfulAnalysis
.map(f => f(clientClassesDir, clientReporter, clientTracer))
val persistTask =
persistAnalysis(analysisForFutureCompilationRuns, compileOut.analysisOut)
val initialTasks = List(persistTask, successBackgroundTasks)
val initialTasks = persistTask :: successBackgroundTasks.toList
val allClientSyncTasks = Task.gatherUnordered(initialTasks).flatMap { _ =>
// Only start these tasks after the previous IO tasks in the external dir are done
val firstTask = updateExternalClassesDirWithReadOnly(
Expand Down Expand Up @@ -736,9 +696,10 @@ object Compiler {
() => elapsed,
reporter,
backgroundTasksWhenNewSuccessfulAnalysis,
allInvalidatedClassFilesForProject,
allInvalidatedExtraCompileProducts,
previousSuccessfulProblems,
errorCause = Some(cause),
shouldAttemptRestartingCompilationForBestEffort
Some(cause)
)

case Failure(_: xsbti.CompileCancelled) => handleCancellation
Expand All @@ -755,13 +716,9 @@ object Compiler {
Result.Failed(failedProblems, None, elapsed, backgroundTasks, None)
case t: Throwable =>
t.printStackTrace()
val sw = new StringWriter()
t.printStackTrace(new PrintWriter(sw))
logger.error(sw.toString())
val backgroundTasks =
toBackgroundTasks(backgroundTasksForFailedCompilation.toList)
val failedProblems = findFailedProblems(reporter, None)
Result.Failed(failedProblems, Some(t), elapsed, backgroundTasks, None)
Result.Failed(Nil, Some(t), elapsed, backgroundTasks, None)
}
}
}
Expand Down Expand Up @@ -964,11 +921,13 @@ object Compiler {
elapsed: () => Long,
reporter: ZincReporter,
backgroundTasksWhenNewSuccessfulAnalysis: mutable.ListBuffer[CompileBackgroundTasks.Sig],
allInvalidatedClassFilesForProject: mutable.HashSet[File],
allInvalidatedExtraCompileProducts: mutable.HashSet[File],
previousSuccessfulProblems: List[ProblemPerPhase],
errorCause: Option[xsbti.CompileFailed],
shouldAttemptRestartingCompilation: Boolean
errorCause: Option[xsbti.CompileFailed]
): Result = {
val uniqueInputs = compileInputs.uniqueInputs
val readOnlyClassesDir = compileOut.internalReadOnlyClassesDir.underlying
val newClassesDir = compileOut.internalNewClassesDir.underlying

reporter.processEndCompilation(
Expand All @@ -985,7 +944,7 @@ object Compiler {
)

val products = CompileProducts(
newClassesDir, // let's not use readonly dir
readOnlyClassesDir,
newClassesDir,
noOpPreviousResult,
noOpPreviousResult,
Expand All @@ -1002,15 +961,22 @@ object Compiler {
): Task[Unit] = {
val clientClassesDir = clientClassesObserver.classesDir
val successBackgroundTasks =
deleteClientExternalBestEffortDirTask(clientClassesDir).flatMap { _ =>
Task.gatherUnordered(
backgroundTasksWhenNewSuccessfulAnalysis
.map(f => f(clientClassesDir, clientReporter, clientTracer))
)
}
val allClientSyncTasks = successBackgroundTasks.flatMap { _ =>
// Only start this task after the previous IO tasks in the external dir are done
Task {
backgroundTasksWhenNewSuccessfulAnalysis
.map(f => f(clientClassesDir, clientReporter, clientTracer))
val allClientSyncTasks = Task.gatherUnordered(successBackgroundTasks.toList).flatMap { _ =>
// Only start these tasks after the previous IO tasks in the external dir are done
val firstTask = updateExternalClassesDirWithReadOnly(
clientClassesDir,
clientTracer,
clientLogger,
compileInputs,
readOnlyClassesDir,
readOnlyCopyDenylist = mutable.HashSet.empty,
allInvalidatedClassFilesForProject,
allInvalidatedExtraCompileProducts
)

val secondTask = Task {
// Delete everything outside of betasty and semanticdb
val deletedCompileProducts =
BloopClassFileManager.supportedCompileProducts.filter(_ != ".betasty") :+ ".class"
Expand All @@ -1019,32 +985,28 @@ object Compiler {
.filter(path => Files.isRegularFile(path))
.filter(path => deletedCompileProducts.exists(path.toString.endsWith(_)))
.forEach(Files.delete(_))
}.map(_ => ())
}
Task
.gatherUnordered(List(firstTask, secondTask))
.map(_ => ())
}

allClientSyncTasks.doOnFinish(_ => Task(clientReporter.reportEndCompilation()))
}
}

if (shouldAttemptRestartingCompilation) {
BloopPaths.delete(compileOut.internalNewClassesDir)
}

val newHash =
if (!shouldAttemptRestartingCompilation)
BestEffortUtils.hashResult(
products.newClassesDir,
compileInputs.sources,
compileInputs.classpath
)
else ""
val newHash = BestEffortUtils.hashResult(
products.newClassesDir,
compileInputs.sources,
compileInputs.classpath
)
val failedProblems = findFailedProblems(reporter, errorCause)
Result.Failed(
failedProblems,
None,
elapsed(),
backgroundTasksExecution,
Some(BestEffortProducts(products, newHash, shouldAttemptRestartingCompilation))
Some(BestEffortProducts(products, newHash))
)
}

Expand Down Expand Up @@ -1212,20 +1174,4 @@ object Compiler {
}
}
}

// Deletes all previous best-effort artifacts to get rid of all of the outdated ones.
// Since best effort compilation is not affected by incremental compilation,
// all relevant files are always produced by the compiler. Because of this,
// we can always delete all previous files and copy newly created ones
// without losing anything in the process.
def deleteClientExternalBestEffortDirTask(clientClassesDir: AbsolutePath) = {
val clientExternalBestEffortDir =
clientClassesDir.underlying.resolve("META-INF/best-effort")
Task {
if (Files.exists(clientExternalBestEffortDir)) {
BloopPaths.delete(AbsolutePath(clientExternalBestEffortDir))
}
()
}.memoize
}
}
13 changes: 4 additions & 9 deletions backend/src/main/scala/bloop/util/BestEffortUtils.scala
View file
Original file line number Diff line number Diff line change
@@ -1,21 +1,16 @@
package bloop.util

import java.math.BigInteger
import java.nio.file.Files
import java.nio.file.Path
import java.security.MessageDigest

import java.math.BigInteger
import java.nio.file.Files
import scala.collection.JavaConverters._

import java.nio.file.Path
import bloop.io.AbsolutePath

object BestEffortUtils {

case class BestEffortProducts(
compileProducts: bloop.CompileProducts,
hash: String,
recompile: Boolean
)
case class BestEffortProducts(compileProducts: bloop.CompileProducts, hash: String)

/* Hashes results of a projects compilation, to mimic how it would have been handled in zinc.
* Returns SHA-1 of a project.
Expand Down
7 changes: 3 additions & 4 deletions backend/src/main/scala/sbt/internal/inc/bloop/BloopZincCompiler.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,7 @@ object BloopZincCompiler {
manager: ClassFileManager,
cancelPromise: Promise[Unit],
tracer: BraveTracer,
classpathOptions: ClasspathOptions,
withPreviousResult: Boolean
classpathOptions: ClasspathOptions
): Task[CompileResult] = {
val config = in.options()
val setup = in.setup()
Expand All @@ -82,8 +81,8 @@ object BloopZincCompiler {
scalacOptions,
javacOptions,
classpathOptions,
if (withPreviousResult) in.previousResult.analysis.toOption else None,
if (withPreviousResult) in.previousResult.setup.toOption else None,
in.previousResult.analysis.toOption,
in.previousResult.setup.toOption,
perClasspathEntryLookup,
reporter,
order,
Expand Down
Loading

0 comments on commit 82ff408

Please sign in to comment.