Skip to content

Commit

Permalink
Merge pull request #585 from scalacenter/topic/wip-incremental-common
Browse files Browse the repository at this point in the history
Document internals of invalidation and hook profiler
  • Loading branch information
eed3si9n authored Aug 31, 2018
2 parents 7fbb06c + 11092ad commit b125d82
Show file tree
Hide file tree
Showing 8 changed files with 1,063 additions and 490 deletions.
41 changes: 41 additions & 0 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -243,6 +243,47 @@ lazy val zincCore = (project in internalPath / "zinc-core")
name := "zinc Core",
compileOrder := sbt.CompileOrder.Mixed,
mimaSettings,
PB.targets in Compile := List(scalapb.gen() -> (sourceManaged in Compile).value),
mimaBinaryIssueFilters ++= {
import com.typesafe.tools.mima.core._
import com.typesafe.tools.mima.core.ProblemFilters._
List(
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalNameHashing.allDeps"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalNameHashing.sameAPI"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalNameHashing.invalidateClass"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalNameHashing.invalidateByExternal"),
exclude[DirectAbstractMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidatedPackageObjects"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalNameHashing.this"),
exclude[MissingClassProblem]("sbt.internal.inc.ClassToSourceMapper"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.Incremental.compile"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.Incremental.prune"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.changes"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.sameClass"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.allDeps"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.sameAPI"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateIntermediate"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateByAllExternal"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateDuplicates"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.transitiveDeps"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateClass"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.externalBinaryModified"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateIncremental"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.changedInitial"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.transitiveDeps$default$2"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.orTrue"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateByExternal"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.wrappedLog"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.shortcutSameClass"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.orEmpty"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.changedIncremental"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.currentExternalAPI"),
exclude[DirectMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.this"),
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findClassDependencies"),
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateClassesInternally"),
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.invalidateClassesExternally"),
exclude[ReversedMissingMethodProblem]("sbt.internal.inc.IncrementalCommon.findAPIChange")
)
}
)
.configure(addSbtIO, addSbtUtilLogging, addSbtUtilRelation)

Expand Down
72 changes: 72 additions & 0 deletions internal/zinc-core/src/main/protobuf/zprof.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
syntax = "proto3";

package sbt.internal.inc;

///////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////// ZINC PROF ///////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////

// This protobuf scheme is inspired by https://github.com/google/pprof/blob/master/proto/profile.proto
// As pprof, it uses a string table and all the supposed strings in the format are represented as an
// index (int32) of that string table. This is done to minimize overhead in memory and disk.

message Profile {
repeated ZincRun runs = 1;
repeated string string_table = 2;
}

message ZincRun {
InitialChanges initial = 1;
repeated CycleInvalidation cycles = 3;
}

message CycleInvalidation {
repeated int32 invalidated = 1;
repeated int32 invalidatedByPackageObjects = 2;
repeated int32 initialSources = 3;
repeated int32 invalidatedSources = 4;
repeated int32 recompiledClasses = 5;

int64 startTimeNanos = 6; // Start time of compilation (UTC) as nanoseconds past the epoch
int64 compilationDurationNanos = 7; // Duration of the compilation profile in nanoseconds
repeated ApiChange changesAfterRecompilation = 8;

repeated InvalidationEvent events = 9;
repeated int32 nextInvalidations = 10;
bool shouldCompileIncrementally = 11;
}

message InvalidationEvent {
string kind = 1;
repeated int32 inputs = 2;
repeated int32 outputs = 3;
string reason = 4;
}

message Changes {
repeated int32 added = 1;
repeated int32 removed = 2;
repeated int32 modified = 3;
}

message ApiChange {
int32 modifiedClass = 1;
string reason = 2;
repeated UsedName usedNames = 3; // Can be empty if the change is not related to names
}

message InitialChanges {
Changes changes = 1;
repeated int32 removedProducts = 2;
repeated int32 binaryDependencies = 3;
repeated ApiChange externalChanges = 4;
}

message UsedName {
int32 name = 1;
repeated Scope scopes = 2;
}

message Scope {
int32 kind = 1;
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@ import java.io.File
import sbt.util.{ Level, Logger }
import xsbti.compile.analysis.{ ReadStamps, Stamp => XStamp }
import xsbti.compile.{
ClassFileManager => XClassFileManager,
CompileAnalysis,
DependencyChanges,
IncOptions
IncOptions,
ClassFileManager => XClassFileManager
}

/**
Expand Down Expand Up @@ -46,6 +46,7 @@ object Incremental {
* @param callbackBuilder The builder that builds callback where we report dependency issues.
* @param log The log where we write debugging information
* @param options Incremental compilation options
* @param profiler An implementation of an invalidation profiler, empty by default.
* @param equivS The means of testing whether two "Stamps" are the same.
* @return
* A flag of whether or not compilation completed successfully, and the resulting dependency analysis object.
Expand All @@ -58,12 +59,13 @@ object Incremental {
compile: (Set[File], DependencyChanges, xsbti.AnalysisCallback, XClassFileManager) => Unit,
callbackBuilder: AnalysisCallback.Builder,
log: sbt.util.Logger,
options: IncOptions
options: IncOptions,
profiler: InvalidationProfiler = InvalidationProfiler.empty
)(implicit equivS: Equiv[XStamp]): (Boolean, Analysis) = {
val previous = previous0 match { case a: Analysis => a }
val incremental: IncrementalCommon =
new IncrementalNameHashing(log, options)
val initialChanges = incremental.changedInitial(sources, previous, current, lookup)
val runProfiler = profiler.profileRun
val incremental: IncrementalCommon = new IncrementalNameHashing(log, options, runProfiler)
val initialChanges = incremental.detectInitialChanges(sources, previous, current, lookup)
val binaryChanges = new DependencyChanges {
val modifiedBinaries = initialChanges.binaryDeps.toArray
val modifiedClasses = initialChanges.external.allModified.toArray
Expand Down Expand Up @@ -118,15 +120,11 @@ object Incremental {
private[inc] def apiDebug(options: IncOptions): Boolean =
options.apiDebug || java.lang.Boolean.getBoolean(apiDebugProp)

private[sbt] def prune(invalidatedSrcs: Set[File], previous: CompileAnalysis): Analysis =
prune(invalidatedSrcs, previous, ClassFileManager.deleteImmediately)

private[sbt] def prune(invalidatedSrcs: Set[File],
previous0: CompileAnalysis,
classfileManager: XClassFileManager): Analysis = {
val previous = previous0 match { case a: Analysis => a }
classfileManager.delete(invalidatedSrcs.flatMap(previous.relations.products).toArray)
previous -- invalidatedSrcs
private[sbt] def prune(invalidatedSrcs: Set[File], previous0: CompileAnalysis): Analysis = {
val previous = previous0.asInstanceOf[Analysis]
IncrementalCommon.pruneClassFilesOfInvalidations(invalidatedSrcs,
previous,
ClassFileManager.deleteImmediately)
}

private[this] def manageClassfiles[T](options: IncOptions)(run: XClassFileManager => T): T = {
Expand Down
Loading

0 comments on commit b125d82

Please sign in to comment.