Fix scala#3324: introduce IsInstanceOfChecker

add check for runtime realizability of type test

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -81,6 +81,7 @@ class Compiler {
          new CrossCastAnd,           // Normalize selections involving intersection types.
          new Splitter) ::            // Expand selections involving union types into conditionals
     List(new ErasedDecls,            // Removes all erased defs and vals decls (except for parameters)
+         new IsInstanceOfChecker,    // check runtime realisability for `isInstanceOf`
          new VCInlineMethods,        // Inlines calls to value class methods
          new SeqLiterals,            // Express vararg arguments as arrays
          new InterceptedMethods,     // Special handling of `==`, `|=`, `getClass` methods

compiler/src/dotty/tools/dotc/transform/IsInstanceOfChecker.scala

@@ -0,0 +1,84 @@
+package dotty.tools.dotc
+package transform
+
+import util.Positions._
+import MegaPhase.MiniPhase
+import core._
+import Contexts.Context, Types._, Decorators._, Symbols._, typer._
+import TypeUtils._, Flags._
+import config.Printers.{ transforms => debug }
+
+/** check runtime realizability of type test
+ */
+class IsInstanceOfChecker extends MiniPhase {
+
+  import ast.tpd._
+
+  val phaseName = "isInstanceOfChecker"
+
+  override def transformTypeApply(tree: TypeApply)(implicit ctx: Context): Tree = {
+    def ensureCheckable(qual: Tree, pt: Tree): Tree = {
+      if (!Checkable.checkable(qual.tpe, pt.tpe))
+         ctx.warning(
+           s"the type test for ${pt.show} cannot be checked at runtime",
+           tree.pos
+         )
+
+      tree
+    }
+
+    tree.fun match {
+      case fn: Select if fn.symbol == defn.Any_typeTest =>
+        ensureCheckable(fn.qualifier, tree.args.head)
+      case fn: Select if fn.symbol == defn.Any_isInstanceOf =>
+        ensureCheckable(fn.qualifier, tree.args.head)
+      case _ => tree
+    }
+  }
+}
+
+object Checkable {
+  import Inferencing._
+  import ProtoTypes._
+
+  /** Whether `(x:X).isInstanceOf[P]` can be checked at runtime?
+   *
+   *  The following cases are not checkable at runtime:
+   *
+   *  1. if `P` refers to an abstract type member
+   *  2. if `P` is `pre.F[Ts]` and `pre.F` refers to a class:
+   *     (a) replace `Ts` with fresh type variables `Xs`
+   *     (b) instantiate `Xs` with the constraint `pre.F[Xs] <:< X`
+   *     (c) `pre.F[Xs] <:< P` doesn't hold
+   *  3. if `P = T1 | T2` or `P = T1 & T2`, checkable(X, T1) && checkable(X, T2).
+   */
+  def checkable(X: Type, P: Type)(implicit ctx: Context): Boolean = {
+    def Psym = P.dealias.typeSymbol
+
+    def isAbstract = !Psym.isClass
+
+    def isClassDetermined(tpe: AppliedType) = {
+      val AppliedType(tycon, args) = tpe
+      val tvars = tycon.typeParams.map { tparam => newTypeVar(tparam.paramInfo.bounds) }
+      val P2 = tycon.appliedTo(tvars)
+
+      debug.println("P2 : " + P2)
+      debug.println("X : " + X)
+
+      !(P2 <:< X.widen) || {
+        val syms = maximizeType(P2, Psym.pos, fromScala2x = false)
+        val res = P2 <:< P
+        debug.println("P2: " + P2.show)
+        debug.println("P2 <:< P = " + res)
+        res
+      }
+    }
+
+    P match {
+      case tpe: AppliedType  => !isAbstract && isClassDetermined(tpe)
+      case AndType(tp1, tp2) => checkable(X, tp1) && checkable(X, tp2)
+      case OrType(tp1, tp2)  => checkable(X, tp1) && checkable(X, tp2)
+      case _                 => !isAbstract
+    }
+  }
+}

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -103,6 +103,7 @@ class CompilationTests extends ParallelTesting {
     compileFilesInDir("tests/pos-no-optimise", defaultOptions) +
     compileFilesInDir("tests/pos-deep-subtype", allowDeepSubtypes) +
     compileDir("tests/pos/i1137-1", defaultOptions and "-Yemit-tasty") +
+    compileDir("tests/neg-custom-args/isInstanceOf", defaultOptions and "-Xfatal-warnings") +
     compileFile(
       // succeeds despite -Xfatal-warnings because of -nowarn
       "tests/neg-custom-args/fatal-warnings/xfatalWarnings.scala",

tests/neg-custom-args/isInstanceOf/3324b.scala

@@ -0,0 +1,9 @@
+class C[T] {
+  val x: Any = ???
+  if (x.isInstanceOf[List[String]]) // error: unchecked
+    if (x.isInstanceOf[T])          // error: unchecked
+      x match {
+        case x: List[String] =>     // error: unchecked
+        case x: T =>                // error: unchecked
+      }
+}

tests/neg-custom-args/isInstanceOf/3324c.scala

@@ -0,0 +1,10 @@
+sealed trait A[T]
+class B[T] extends A[T]
+
+class Test {
+  def f(x: B[Int]) = x match { case _: A[Int] if true => }
+
+  def g(x: A[Int]) = x match { case _: B[Int] => }
+
+  def foo(x: Any) = x.isInstanceOf[List[String]]  // error
+}

tests/neg-custom-args/isInstanceOf/i3324.scala

@@ -0,0 +1,4 @@
+class Foo {
+  def foo(x: Any): Boolean =
+    x.isInstanceOf[List[String]]  // error
+}