Implement match type amendment: extractors follow aliases and singletons

This implements the change proposed in
scala/improvement-proposals#84.

The added pos test case presents motivating examples, the added neg test cases
demonstrate that errors are correctly reported when cycles are present. The
potential for cycle is no worse than with the existing extraction logic as
demonstrated by the existing test in `tests/neg/mt-deskolemize.scala`.

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -3518,20 +3518,75 @@ class MatchReducer(initctx: Context) extends TypeComparer(initctx) {
                 false
 
           case MatchTypeCasePattern.TypeMemberExtractor(typeMemberName, capture) =>
+            /** Try to remove references to `skolem` from a type in accordance with the spec.
+             *
+             *  If any reference to `skolem` remains in the result type,
+             *  `refersToSkolem` is set to true.
+             */
+            class DropSkolemMap(skolem: SkolemType) extends TypeMap:
+              var refersToSkolem = false
+              def apply(tp: Type): Type =
+                tp match
+                  case `skolem` =>
+                    refersToSkolem = true
+                    tp
+                  case tp: NamedType =>
+                    var savedRefersToSkolem = refersToSkolem
+                    refersToSkolem = false
+                    try
+                      val pre1 = apply(tp.prefix)
+                      if refersToSkolem then
+                        tp match
+                          case tp: TermRef => tp.info.widenExpr.dealias match
+                            case info: SingletonType =>
+                              refersToSkolem = false
+                              apply(info)
+                            case _ =>
+                              tp.derivedSelect(pre1)
+                          case tp: TypeRef => tp.info match
+                            case info: AliasingBounds =>
+                              refersToSkolem = false
+                              apply(info.alias)
+                            case _ =>
+                              tp.derivedSelect(pre1)
+                      else
+                        tp.derivedSelect(pre1)
+                    finally
+                      refersToSkolem |= savedRefersToSkolem
+                  case tp: LazyRef =>
+                    // By default, TypeMap maps LazyRefs lazily. We need to
+                    // force it for `refersToSkolem` to be correctly set.
+                    apply(tp.ref)
+                  case _ =>
+                    mapOver(tp)
+            end DropSkolemMap
+            /** Try to remove references to `skolem` from `u` in accordance with the spec.
+             *
+             *  If any reference to `skolem` remains in the result type, return
+             *  NoType instead.
+             */
+            def dropSkolem(u: Type, skolem: SkolemType): Type =
+              val dmap = DropSkolemMap(skolem)
+              val res = dmap(u)
+              if dmap.refersToSkolem then NoType else res
+
             val stableScrut: SingletonType = scrut match
               case scrut: SingletonType => scrut
               case _                    => SkolemType(scrut)
+
             stableScrut.member(typeMemberName) match
               case denot: SingleDenotation if denot.exists =>
                 val info = denot.info match
                   case alias: AliasingBounds           => alias.alias        // Extract the alias
                   case ClassInfo(prefix, cls, _, _, _) => prefix.select(cls) // Re-select the class from the prefix
                   case info => info // Notably, RealTypeBounds, which will eventually give a MatchResult.NoInstances
-                val infoRefersToSkolem = stableScrut.isInstanceOf[SkolemType] && stableScrut.occursIn(info)
-                val info1 = info match
-                  case info: TypeBounds        => info                       // Will already trigger a MatchResult.NoInstances
-                  case _ if infoRefersToSkolem => RealTypeBounds(info, info) // Explicitly trigger a MatchResult.NoInstances
-                  case _                       => info                       // We have a match
+                val info1 = stableScrut match
+                  case skolem: SkolemType =>
+                    dropSkolem(info, skolem).orElse:
+                      info match
+                        case info: TypeBounds  => info                       // Will already trigger a MatchResult.NoInstances
+                        case _                 => RealTypeBounds(info, info) // Explicitly trigger a MatchResult.NoInstances
+                  case _ => info
                 rec(capture, info1, variance = 0, scrutIsWidenedAbstract)
               case _ =>
                 false

tests/neg/mt-deskolemize.scala

@@ -14,3 +14,40 @@ class SimpleLoop2 extends Expr:
 
 object Test1:
   val x: ExtractValue[SimpleLoop1] = 1 // error
+
+trait Description:
+  type Elem <: Tuple
+
+class PrimBroken extends Expr:
+  type Value = Alias
+  type Alias = Value // error
+
+class Prim extends Expr:
+  type Value = BigInt
+
+class VecExpr[E <: Expr] extends Expr:
+  type Value = Vector[ExtractValue[E]]
+
+trait ProdExpr extends Expr:
+  val description: Description
+  type Value = Tuple.Map[description.Elem, [X] =>> ExtractValue[X & Expr]]
+
+
+class MyExpr1 extends ProdExpr:
+  final val description = new Description:
+    type Elem = (VecExpr[Prim], MyExpr2)
+
+class MyExpr2 extends ProdExpr:
+  final val description = new Description:
+    type Elem = (VecExpr[VecExpr[MyExpr1]], Prim)
+
+object Test2:
+  def fromLiteral[E <: Expr](v: ExtractValue[E]): E = ???
+  val x1: ExtractValue[PrimBroken] = 1 // error
+
+  val x: ExtractValue[SimpleLoop1] = 1 // error
+
+  val foo: MyExpr2 = new MyExpr2
+  val v: foo.Value = (Vector(Vector()), 1) // error: Recursion limit exceeded
+  val c: MyExpr2 = fromLiteral:
+    (Vector(Vector()), 1) // error: Recursion limit exceeded

tests/pos/mt-deskolemize.scala

@@ -0,0 +1,49 @@
+trait Expr:
+  type Value
+
+object Expr:
+  type Of[V] = Expr { type Value = V }
+  type ExtractValue[F <: Expr] = F match
+    case Expr.Of[v] => v
+import Expr.ExtractValue
+
+class Prim extends Expr:
+  type Value = Alias
+  type Alias = BigInt
+
+class VecExpr[E <: Expr] extends Expr:
+  type Value = Vector[ExtractValue[E]]
+
+trait Description:
+  type Elem <: Tuple
+
+trait ProdExpr extends Expr:
+  val description: Description
+  type Value = Tuple.Map[description.Elem, [X] =>> ExtractValue[X & Expr]]
+
+class MyExpr1 extends ProdExpr:
+  final val description = new Description:
+    type Elem = (VecExpr[Prim], Prim)
+
+class MyExpr2 extends ProdExpr:
+  final val description = new Description:
+    type Elem = (VecExpr[VecExpr[MyExpr1]], Prim)
+
+trait ProdExprAlt[T <: Tuple] extends Expr:
+  type Value = Tuple.Map[T, [X] =>> ExtractValue[X & Expr]]
+
+class MyExpr3 extends ProdExprAlt[(Prim, VecExpr[Prim], Prim)]
+
+object Test:
+  def fromLiteral[E <: Expr](v: ExtractValue[E]): E = ???
+  val a: Prim = fromLiteral(1)
+  val b: VecExpr[Prim] = fromLiteral(Vector(1))
+  val c: MyExpr1 = fromLiteral((Vector(1), 1))
+  val d: MyExpr2 = fromLiteral(Vector(Vector((Vector(1), 1))), 2)
+  val e: MyExpr3 = fromLiteral((1, Vector(1), 1))
+  val f: ProdExprAlt[(MyExpr1, VecExpr[MyExpr3])] = fromLiteral:
+    (
+      (Vector(1), 1),
+      Vector((1, Vector(1), 1), (2, Vector(1), 2))
+    )
+  val g: Expr { type Alias = Int; type Value = Alias } = fromLiteral(1)