diff --git a/internal/compiler-bridge/src/main/scala/xsbt/Dependency.scala b/internal/compiler-bridge/src/main/scala/xsbt/Dependency.scala
index dfb7b6fa9d..9dc01dbe46 100644
--- a/internal/compiler-bridge/src/main/scala/xsbt/Dependency.scala
+++ b/internal/compiler-bridge/src/main/scala/xsbt/Dependency.scala
@@ -144,7 +144,7 @@ final class Dependency(val global: CallbackGlobal) extends LocateClassFile with
 
   private case class ClassDependency(from: Symbol, to: Symbol)
 
-  private class DependencyTraverser(processor: DependencyProcessor) extends Traverser {
+  private final class DependencyTraverser(processor: DependencyProcessor) extends Traverser {
     // are we traversing an Import node at the moment?
     private var inImportNode = false
 
@@ -255,13 +255,6 @@ final class Dependency(val global: CallbackGlobal) extends LocateClassFile with
       ()
     }
 
-    def addTypeDependencies(tpe: Type): Unit = {
-      // Defined in GlobalHelpers.scala
-      object TypeDependencyTraverser extends TypeDependencyTraverser(addDependency)
-      TypeDependencyTraverser.traverse(tpe)
-      TypeDependencyTraverser.reinitializeVisited()
-    }
-
     private def addDependency(dep: Symbol): Unit = {
       val fromClass = resolveDependencySource
       if (ignoredSymbol(fromClass) || fromClass.hasPackageFlag) {
@@ -272,6 +265,48 @@ final class Dependency(val global: CallbackGlobal) extends LocateClassFile with
       }
     }
 
+    /** Define a type traverser to keep track of the type dependencies. */
+    object TypeDependencyTraverser extends TypeDependencyTraverser {
+      type Handler = Symbol => Unit
+      // Type dependencies are always added to member references
+      val memberRefHandler = processor.memberRef
+      def createHandler(fromClass: Symbol): Handler = { (dep: Symbol) =>
+        if (ignoredSymbol(fromClass) || fromClass.hasPackageFlag) {
+          if (inImportNode) addTopLevelImportDependency(dep)
+          else devWarning(Feedback.missingEnclosingClass(dep, currentOwner))
+        } else {
+          addClassDependency(_memberRefCache, memberRefHandler, fromClass, dep)
+        }
+      }
+
+      val cache = scala.collection.mutable.Map.empty[Symbol, (Handler, scala.collection.mutable.HashSet[Type])]
+      private var handler: Handler = _
+      private var visitedOwner: Symbol = _
+      def setOwner(owner: Symbol) = {
+        if (visitedOwner != owner) {
+          cache.get(owner) match {
+            case Some((h, ts)) =>
+              visited = ts
+              handler = h
+            case None =>
+              val newVisited = scala.collection.mutable.HashSet.empty[Type]
+              handler = createHandler(owner)
+              cache += owner -> (handler -> newVisited)
+              visited = newVisited
+              visitedOwner = owner
+          }
+        }
+      }
+
+      override def addDependency(symbol: global.Symbol) = handler(symbol)
+    }
+
+    def addTypeDependencies(tpe: Type): Unit = {
+      val fromClass = resolveDependencySource
+      TypeDependencyTraverser.setOwner(fromClass)
+      TypeDependencyTraverser.traverse(tpe)
+    }
+
     private def addInheritanceDependency(dep: Symbol): Unit = {
       val fromClass = resolveDependencySource
       if (_isLocalSource) {
diff --git a/internal/compiler-bridge/src/main/scala/xsbt/ExtractUsedNames.scala b/internal/compiler-bridge/src/main/scala/xsbt/ExtractUsedNames.scala
index 24e1a42a69..b13082d6b5 100644
--- a/internal/compiler-bridge/src/main/scala/xsbt/ExtractUsedNames.scala
+++ b/internal/compiler-bridge/src/main/scala/xsbt/ExtractUsedNames.scala
@@ -91,9 +91,8 @@ class ExtractUsedNames[GlobalType <: CallbackGlobal](val global: GlobalType) ext
       super.traverse(tree)
     }
 
-    val addSymbol: Symbol => Unit = {
-      symbol =>
-        val names = getNamesOfEnclosingScope
+    val addSymbol = {
+      (names: mutable.Set[Name], symbol: Symbol) =>
         if (!ignoredSymbol(symbol)) {
           val name = symbol.name
           // Synthetic names are no longer included. See https://github.com/sbt/sbt/issues/2537
@@ -131,7 +130,29 @@ class ExtractUsedNames[GlobalType <: CallbackGlobal](val global: GlobalType) ext
       }
     }
 
-    object TypeDependencyTraverser extends TypeDependencyTraverser(addSymbol)
+    private object TypeDependencyTraverser extends TypeDependencyTraverser {
+      private var ownersCache = mutable.Map.empty[Symbol, mutable.HashSet[Type]]
+      private var nameCache: mutable.Set[Name] = _
+      private var ownerVisited: Symbol = _
+
+      def setCacheAndOwner(cache: mutable.Set[Name], owner: Symbol) = {
+        if (ownerVisited != owner) {
+          ownersCache.get(owner) match {
+            case Some(ts) =>
+              visited = ts
+            case None =>
+              val newVisited = mutable.HashSet.empty[Type]
+              visited = newVisited
+              ownersCache += owner -> newVisited
+          }
+          nameCache = cache
+          ownerVisited = owner
+        }
+      }
+
+      override def addDependency(symbol: global.Symbol) =
+        addSymbol(nameCache, symbol)
+    }
 
     private def handleClassicTreeNode(tree: Tree): Unit = tree match {
       case _: DefTree | _: Template => ()
@@ -158,11 +179,13 @@ class ExtractUsedNames[GlobalType <: CallbackGlobal](val global: GlobalType) ext
           original.foreach(traverse)
         }
       case t if t.hasSymbolField =>
-        addSymbol(t.symbol)
+        addSymbol(getNamesOfEnclosingScope, t.symbol)
         val tpe = t.tpe
         if (!ignoredType(tpe)) {
+          // Initialize _currentOwner if it's not
+          val cache = getNamesOfEnclosingScope
+          TypeDependencyTraverser.setCacheAndOwner(cache, _currentOwner)
           TypeDependencyTraverser.traverse(tpe)
-          TypeDependencyTraverser.reinitializeVisited()
         }
       case _ =>
     }
diff --git a/internal/compiler-bridge/src/main/scala/xsbt/GlobalHelpers.scala b/internal/compiler-bridge/src/main/scala/xsbt/GlobalHelpers.scala
index 11f207910b..6130a577ff 100644
--- a/internal/compiler-bridge/src/main/scala/xsbt/GlobalHelpers.scala
+++ b/internal/compiler-bridge/src/main/scala/xsbt/GlobalHelpers.scala
@@ -36,18 +36,8 @@ trait GlobalHelpers {
     }
   }
 
-  /** Apply `op` on every type symbol which doesn't represent a package. */
-  def foreachNotPackageSymbolInType(tpe: Type)(op: Symbol => Unit): Unit = {
-    new ForEachTypeTraverser(_ match {
-      case null =>
-      case tpe =>
-        val sym = tpe.typeSymbolDirect
-        if (sym != NoSymbol && !sym.hasPackageFlag) op(sym)
-    }).traverse(tpe)
-  }
-
-  private[xsbt] class TypeDependencyTraverser(addDependency: Symbol => Unit)
-    extends TypeTraverser {
+  private[xsbt] abstract class TypeDependencyTraverser extends TypeTraverser {
+    def addDependency(symbol: Symbol): Unit
 
     /** Add type dependency ignoring packages and inheritance info from classes. */
     @inline private def addTypeSymbolDependency(symbol: Symbol): Unit = {
@@ -67,10 +57,10 @@ trait GlobalHelpers {
     }
 
     // Define cache and populate it with known types at initialization time
-    private val visited = scala.collection.mutable.HashSet.empty[Type]
+    protected var visited = scala.collection.mutable.HashSet.empty[Type]
 
     /** Clear the cache after every `traverse` invocation at the call-site. */
-    private[xsbt] def reinitializeVisited(): Unit = visited.clear()
+    protected def reinitializeVisited(): Unit = visited.clear()
 
     /**
      * Traverse the type and its info to track all type dependencies.