Backport "Refactor hasOnlyLocalInstantiation to use isLocalToCompilat…

…ionUnit" to LTS (#20993)

Backports #19886 to the LTS branch.

PR submitted by the release tooling.
[skip ci]

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1200,6 +1200,11 @@ object SymDenotations {
     final def isEffectivelySealed(using Context): Boolean =
       isOneOf(FinalOrSealed) || isClass && !isOneOf(EffectivelyOpenFlags)
 
+    final def isLocalToCompilationUnit(using Context): Boolean =
+      is(Private)
+      || owner.ownersIterator.takeWhile(!_.isStaticOwner).exists(_.isTerm)
+      || accessBoundary(defn.RootClass).isProperlyContainedIn(symbol.topLevelClass)
+
     final def isTransparentClass(using Context): Boolean =
       is(TransparentType)
       || defn.isAssumedTransparent(symbol)

compiler/src/dotty/tools/dotc/transform/ExplicitOuter.scala

@@ -218,18 +218,17 @@ object ExplicitOuter {
    */
   private def needsOuterAlways(cls: ClassSymbol)(using Context): Boolean =
     needsOuterIfReferenced(cls) &&
-    (!hasLocalInstantiation(cls) || // needs outer because we might not know whether outer is referenced or not
+    (!hasOnlyLocalInstantiation(cls) || // needs outer because we might not know whether outer is referenced or not
      cls.mixins.exists(needsOuterIfReferenced) || // needs outer for parent traits
      cls.info.parents.exists(parent => // needs outer to potentially pass along to parent
        needsOuterIfReferenced(parent.classSymbol.asClass)))
 
   /** Class is only instantiated in the compilation unit where it is defined */
-  private def hasLocalInstantiation(cls: ClassSymbol)(using Context): Boolean =
+  private def hasOnlyLocalInstantiation(cls: ClassSymbol)(using Context): Boolean =
     // Modules are normally locally instantiated, except if they are declared in a trait,
     // in which case they will be instantiated in the classes that mix in the trait.
-    cls.owner.ownersIterator.takeWhile(!_.isStaticOwner).exists(_.isTerm)
-    || cls.is(Private, butNot = Module)
-    || cls.is(Module) && !cls.owner.is(Trait)
+    if cls.is(Module) then !cls.owner.is(Trait)
+    else cls.isLocalToCompilationUnit
 
   /** The outer parameter accessor of cass `cls` */
   private def outerParamAccessor(cls: ClassSymbol)(using Context): TermSymbol =

tests/generic-java-signatures/outer-ref-elimination.check

@@ -7,3 +7,5 @@ List(public T6$$anon$3$C6())
 List(public T7$C7$1())
 List(public T8$$anon$4$C8())
 List(public T9$C9$1(T9))
+List(public T10$C10(T10))
+List(public T11$D11$C11())

tests/generic-java-signatures/outer-ref-elimination.scala

@@ -26,6 +26,10 @@ class T8 { def t8(): Unit = new AnyRef { class C8; test(classOf[C8]) } }
 // the field x.
 class T9 { var x = 451; def t9(): Unit = { class C9 {def getX = x}; test(classOf[C9])} }
 
+class T10 { private[T10] class C10; test(classOf[C10]) }
+
+class T11 { class D11 { private[D11] class C11; test(classOf[C11]) } }
+
 object Test {
   def main(args: Array[String]): Unit = {
     T1
@@ -37,5 +41,7 @@ object Test {
     new T7().t7()
     new T8().t8()
     new T9().t9()
+    new T10()
+    val t11 = new T11(); new t11.D11()
   }
 }