Streamline tryNormalize with underlyingMatchType (#20268)

The overall goal was to have a more uniform treatment of `tryNormalize`
rather than the three overrides, making the logic easier to follow.

It also now reuses `underlyingMatchType` for it, which not only has a
caching benefit but also ensures consistent results between them. In
particular, making `tryNormalize.exists` imply
`underlyingMatchType.exists`, which one might assume as true but did not
hold in general previously.

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -490,14 +490,7 @@ object Types extends TypeUtils {
       case _ => false
 
     /** Does this application expand to a match type? */
-    def isMatchAlias(using Context): Boolean = underlyingMatchType.exists
-
-    def underlyingMatchType(using Context): Type = stripped match {
-      case tp: MatchType => tp
-      case tp: HKTypeLambda => tp.resType.underlyingMatchType
-      case tp: AppliedType => tp.underlyingMatchType
-      case _ => NoType
-    }
+    def isMatchAlias(using Context): Boolean = underlyingNormalizable.isMatch
 
     /** Is this a higher-kinded type lambda with given parameter variances?
      *  These lambdas are used as the RHS of higher-kinded abstract types or
@@ -1550,19 +1543,24 @@ object Types extends TypeUtils {
       }
       deskolemizer(this)
 
-    /** The result of normalization using `tryNormalize`, or the type itself if
-     *  tryNormlize yields NoType
+    /** The result of normalization, or the type itself if none apply. */
+    final def normalized(using Context): Type = tryNormalize.orElse(this)
+
+    /** If this type has an underlying match type or applied compiletime.ops,
+     *  then the result after applying all toplevel normalizations, otherwise NoType.
      */
-    final def normalized(using Context): Type = {
-      val normed = tryNormalize
-      if (normed.exists) normed else this
-    }
+    def tryNormalize(using Context): Type = underlyingNormalizable match
+      case mt: MatchType => mt.reduced.normalized
+      case tp: AppliedType => tp.tryCompiletimeConstantFold
+      case _ => NoType
 
-    /** If this type can be normalized at the top-level by rewriting match types
-     *  of S[n] types, the result after applying all toplevel normalizations,
-     *  otherwise NoType
+    /** Perform successive strippings, and beta-reductions of applied types until
+     *  a match type or applied compiletime.ops is reached, if any, otherwise NoType.
      */
-    def tryNormalize(using Context): Type = NoType
+    def underlyingNormalizable(using Context): Type = stripped.stripLazyRef match
+      case tp: MatchType => tp
+      case tp: AppliedType => tp.underlyingNormalizable
+      case _ => NoType
 
     private def widenDealias1(keep: AnnotatedType => Context ?=> Boolean)(using Context): Type = {
       val res = this.widen.dealias1(keep, keepOpaques = false)
@@ -3258,8 +3256,6 @@ object Types extends TypeUtils {
     private var myRef: Type | Null = null
     private var computed = false
 
-    override def tryNormalize(using Context): Type = ref.tryNormalize
-
     def ref(using Context): Type =
       if computed then
         if myRef == null then
@@ -4625,8 +4621,8 @@ object Types extends TypeUtils {
     private var myEvalRunId: RunId = NoRunId
     private var myEvalued: Type = uninitialized
 
-    private var validUnderlyingMatch: Period = Nowhere
-    private var cachedUnderlyingMatch: Type = uninitialized
+    private var validUnderlyingNormalizable: Period = Nowhere
+    private var cachedUnderlyingNormalizable: Type = uninitialized
 
     def isGround(acc: TypeAccumulator[Boolean])(using Context): Boolean =
       if myGround == 0 then myGround = if acc.foldOver(true, this) then 1 else -1
@@ -4690,37 +4686,25 @@ object Types extends TypeUtils {
         case nil => x
       foldArgs(op(x, tycon), args)
 
-    /** Exists if the tycon is a TypeRef of an alias with an underlying match type.
-     *  Anything else should have already been reduced in `appliedTo` by the TypeAssigner.
+    /** Exists if the tycon is a TypeRef of an alias with an underlying match type,
+     *  or a compiletime applied type. Anything else should have already been
+     *  reduced in `appliedTo` by the TypeAssigner. This may reduce several
+     *  HKTypeLambda applications before the underlying normalizable type is reached.
      */
-    override def underlyingMatchType(using Context): Type =
-      if ctx.period != validUnderlyingMatch then
-        cachedUnderlyingMatch = superType.underlyingMatchType
-        validUnderlyingMatch = validSuper
-      cachedUnderlyingMatch
+    override def underlyingNormalizable(using Context): Type =
+      if ctx.period != validUnderlyingNormalizable then tycon match
+        case tycon: TypeRef if defn.isCompiletimeAppliedType(tycon.symbol) =>
+          cachedUnderlyingNormalizable = this
+          validUnderlyingNormalizable = ctx.period
+        case _ =>
+          cachedUnderlyingNormalizable = superType.underlyingNormalizable
+          validUnderlyingNormalizable = validSuper
+      cachedUnderlyingNormalizable
 
-    override def tryNormalize(using Context): Type = tycon.stripTypeVar match {
-      case tycon: TypeRef =>
-        def tryMatchAlias = tycon.info match
-          case AliasingBounds(alias) if isMatchAlias =>
-            trace(i"normalize $this", typr, show = true) {
-              MatchTypeTrace.recurseWith(this) {
-                alias.applyIfParameterized(args.map(_.normalized)).tryNormalize
-                /* `applyIfParameterized` may reduce several HKTypeLambda applications
-                 * before the underlying MatchType is reached.
-                 * Even if they do not involve any match type normalizations yet,
-                 * we still want to record these reductions in the MatchTypeTrace.
-                 * They should however only be attempted if they eventually expand
-                 * to a match type, which is ensured by the `isMatchAlias` guard.
-                 */
-              }
-            }
-          case _ =>
-            NoType
-        tryCompiletimeConstantFold.orElse(tryMatchAlias)
-      case _ =>
-        NoType
-    }
+    override def tryNormalize(using Context): Type =
+      if isMatchAlias && MatchTypeTrace.isRecording then
+        MatchTypeTrace.recurseWith(this)(superType.tryNormalize)
+      else super.tryNormalize
 
     /** Is this an unreducible application to wildcard arguments?
      *  This is the case if tycon is higher-kinded. This means
@@ -5183,13 +5167,6 @@ object Types extends TypeUtils {
     private var myReduced: Type | Null = null
     private var reductionContext: util.MutableMap[Type, Type] | Null = null
 
-    override def tryNormalize(using Context): Type =
-      try
-        reduced.normalized
-      catch
-        case ex: Throwable =>
-          handleRecursive("normalizing", s"${scrutinee.show} match ..." , ex)
-
     private def thisMatchType = this
 
     def reduced(using Context): Type = atPhaseNoLater(elimOpaquePhase) {
@@ -5292,7 +5269,7 @@ object Types extends TypeUtils {
     def apply(bound: Type, scrutinee: Type, cases: List[Type])(using Context): MatchType =
       unique(new CachedMatchType(bound, scrutinee, cases))
 
-    def thatReducesUsingGadt(tp: Type)(using Context): Boolean = tp.underlyingMatchType match
+    def thatReducesUsingGadt(tp: Type)(using Context): Boolean = tp.underlyingNormalizable match
       case mt: MatchType => mt.reducesUsingGadt
       case _ => false
 
@@ -5741,7 +5718,8 @@ object Types extends TypeUtils {
   /** Common supertype of `TypeAlias` and `MatchAlias` */
   abstract class AliasingBounds(val alias: Type) extends TypeBounds(alias, alias) {
 
-    def derivedAlias(alias: Type)(using Context): AliasingBounds
+    def derivedAlias(alias: Type)(using Context): AliasingBounds =
+      if alias eq this.alias then this else AliasingBounds(alias)
 
     override def computeHash(bs: Binders): Int = doHash(bs, alias)
     override def hashIsStable: Boolean = alias.hashIsStable
@@ -5763,10 +5741,7 @@ object Types extends TypeUtils {
 
   /**    = T
    */
-  class TypeAlias(alias: Type) extends AliasingBounds(alias) {
-    def derivedAlias(alias: Type)(using Context): AliasingBounds =
-      if (alias eq this.alias) this else TypeAlias(alias)
-  }
+  class TypeAlias(alias: Type) extends AliasingBounds(alias)
 
   /**    = T     where `T` is a `MatchType`
    *
@@ -5775,10 +5750,7 @@ object Types extends TypeUtils {
    *  If we assumed full substitutivity, we would have to reject all recursive match
    *  aliases (or else take the jump and allow full recursive types).
    */
-  class MatchAlias(alias: Type) extends AliasingBounds(alias) {
-    def derivedAlias(alias: Type)(using Context): AliasingBounds =
-      if (alias eq this.alias) this else MatchAlias(alias)
-  }
+  class MatchAlias(alias: Type) extends AliasingBounds(alias)
 
   object TypeBounds {
     def apply(lo: Type, hi: Type)(using Context): TypeBounds =

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -2044,7 +2044,7 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
               case _ => false
             }
 
-        val result = pt.underlyingMatchType match {
+        val result = pt.underlyingNormalizable match {
           case mt: MatchType if isMatchTypeShaped(mt) =>
             typedDependentMatchFinish(tree, sel1, selType, tree.cases, mt)
           case _ =>

compiler/test/dotc/neg-best-effort-pickling.blacklist

@@ -15,6 +15,7 @@ illegal-match-types.scala
 i13780-1.scala
 i20317a.scala
 i11226.scala
+i974.scala
 
 # semantic db generation fails in the first compilation
 i1642.scala

tests/neg/i12049d.check

@@ -0,0 +1,14 @@
+-- [E007] Type Mismatch Error: tests/neg/i12049d.scala:14:52 -----------------------------------------------------------
+14 |val x: M[NotRelevant[Nothing], Relevant[Nothing]] = 2 // error
+   |                                                    ^
+   |                                                  Found:    (2 : Int)
+   |                                                  Required: M[NotRelevant[Nothing], Relevant[Nothing]]
+   |
+   |                                                  Note: a match type could not be fully reduced:
+   |
+   |                                                    trying to reduce  M[NotRelevant[Nothing], Relevant[Nothing]]
+   |                                                    trying to reduce  Relevant[Nothing]
+   |                                                    failed since selector Nothing
+   |                                                    is uninhabited (there are no values of that type).
+   |
+   | longer explanation available when compiling with `-explain`

tests/neg/i12049d.scala

@@ -0,0 +1,14 @@
+
+trait A
+trait B
+
+type M[X, Y] = Y match
+  case A => Int
+  case B => String
+
+type Relevant[Z] = Z match
+  case A => B
+type NotRelevant[Z] = Z match
+  case B => A
+
+val x: M[NotRelevant[Nothing], Relevant[Nothing]] = 2 // error

tests/pos/i20482.scala

@@ -0,0 +1,16 @@
+trait WrapperType[A]
+
+case class Foo[A]()
+
+case class Bar[A]()
+
+type FooToBar[D[_]] = [A] =>> D[Unit] match {
+  case Foo[Unit] => Bar[A]
+}
+
+case class Test()
+object Test {
+  implicit val wrapperType: WrapperType[Bar[Test]] = new WrapperType[Bar[Test]] {}
+}
+
+val test = summon[WrapperType[FooToBar[Foo][Test]]]

tests/pos/matchtype-unusedArg/A_1.scala

@@ -0,0 +1,8 @@
+
+type Rec[X] = X match
+  case Int => Rec[X]
+
+type M[Unused, Y] = Y match
+  case String => Double
+
+def foo[X](d: M[Rec[X], "hi"]) = ???

tests/pos/matchtype-unusedArg/B_2.scala

@@ -0,0 +1,2 @@
+
+def Test = foo[Int](3d) // crash before changes