Make JVM version configurable per-module (#3716)

Fixes #3480.

Changes:

- adds `coursier-jvm` as a dependency to the `util` module, this library
is for fetching JVMs using coursier and the coursier
[jvm-index](https://github.com/coursier/jvm-index)

- add a `def javaHome: Task[Option[PathRef]]` task to
`ZincWorkerModule`. This defaults to `None`, which is the existing
behavior of using mill's java home.

- Users who want to use a custom JVM need to define a custom
`ZincWorkerModule` and override `def jvmId`, optionally `def
jvmIndexVersion`

- updates the `mockito` and `commons-io` examples to use the new
configuration options. Now these examples should run even when mill
itself is not running on java 11.

- This also required adding `-encoding utf-8` to the projects' javac
options.

- update the `run` and `test` tasks to use the `zincWorker`'s java home
if a different one specified

Notes:

* `JavaModule#compile` forks a new JVM for each compilation to support
custom JVM versions, and `ScalaModule#compile` does not support custom
JVM versions at all. This would require a deeper refactoring of
`ZincWorkerModule` to fix
*

---------

Co-authored-by: Li Haoyi <[email protected]>

build.mill

@@ -116,8 +116,10 @@ object Deps {
   val asmTree = ivy"org.ow2.asm:asm-tree:9.7.1"
   val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5"
 
-  val coursier = ivy"io.get-coursier::coursier:2.1.16"
+  val coursierVersion = "2.1.16"
+  val coursier = ivy"io.get-coursier::coursier:$coursierVersion"
   val coursierInterface = ivy"io.get-coursier:interface:1.0.22"
+  val coursierJvm = ivy"io.get-coursier::coursier-jvm:$coursierVersion"
 
   val cask = ivy"com.lihaoyi::cask:0.9.4"
   val castor = ivy"com.lihaoyi::castor:0.3.0"
@@ -192,6 +194,7 @@ object Deps {
   val mavenResolverTransportFile = ivy"org.apache.maven.resolver:maven-resolver-transport-file:$mavenResolverVersion"
   val mavenResolverTransportHttp = ivy"org.apache.maven.resolver:maven-resolver-transport-http:$mavenResolverVersion"
   val mavenResolverTransportWagon = ivy"org.apache.maven.resolver:maven-resolver-transport-wagon:$mavenResolverVersion"
+  val coursierJvmIndexVersion = "0.0.4-70-51469f"
 
   object RuntimeDeps {
     val dokkaVersion = "1.9.20"
@@ -567,6 +570,13 @@ trait MillStableScalaModule extends MillPublishScalaModule with Mima {
     // https://github.com/com-lihaoyi/mill/pull/3503
     ProblemFilter.exclude[ReversedMissingMethodProblem](
       "mill.scalalib.ScalaModule#ScalaTests.mill$scalalib$ScalaModule$ScalaTests$$super$mandatoryScalacOptions"
+    ),
+    // Not sure why Mima is complaining when these are internal and private
+    ProblemFilter.exclude[Problem](
+      "*.bspJvmBuildTarget"
+    ),
+    ProblemFilter.exclude[Problem](
+      "mill.scalalib.RunModule#RunnerImpl.*"
     )
   )
   def mimaPreviousVersions: T[Seq[String]] = Settings.mimaBaseVersions

contrib/jmh/src/mill/contrib/jmh/JmhModule.scala

@@ -43,7 +43,8 @@ trait JmhModule extends JavaModule {
         classPath = (runClasspath() ++ generatorDeps()).map(_.path) ++
           Seq(compileGeneratedSources().path, resources),
         mainArgs = args,
-        workingDir = T.ctx().dest
+        workingDir = T.ctx().dest,
+        javaHome = zincWorker().javaHome().map(_.path)
       )
     }
 
@@ -90,6 +91,7 @@ trait JmhModule extends JavaModule {
           resourcesDir.toString,
           "default"
         ),
+        javaHome = zincWorker().javaHome().map(_.path),
         jvmArgs = forkedArgs
       )
 

docs/modules/ROOT/nav.adoc

@@ -11,6 +11,7 @@
 ** xref:javalib/publishing.adoc[]
 ** xref:javalib/build-examples.adoc[]
 ** xref:javalib/web-examples.adoc[]
+
 * xref:scalalib/intro.adoc[]
 ** xref:scalalib/module-config.adoc[]
 ** xref:scalalib/dependencies.adoc[]
@@ -56,6 +57,7 @@
 ** xref:fundamentals/library-deps.adoc[]
 ** xref:fundamentals/cross-builds.adoc[]
 ** xref:fundamentals/bundled-libraries.adoc[]
+** xref:fundamentals/configuring-jvm-versions.adoc[]
 // This section talks about Mill plugins. While it could theoretically fit in
 // either section above, it is probably an important enough topic it is worth
 // breaking out on its own

docs/modules/ROOT/pages/fundamentals/configuring-jvm-versions.adoc

@@ -0,0 +1,8 @@
+= Configuring JVM Versions
+
+By default, Mill uses the same JVM that it itself is running on to compile/test/run
+Java/Scala/Kotlin modules. This page goes into more detail about downloading
+and using a custom Java home on a per-module basis.
+
+include::partial$example/depth/javahome/1-custom-jvms.adoc[]
+

example/depth/javahome/1-custom-jvms/bar/src/bar/Bar.scala

@@ -0,0 +1,7 @@
+package bar
+
+object Bar {
+  def main(args: Array[String]): Unit = {
+    println(s"Bar running on Java ${sys.props("java.version")}")
+  }
+}

example/depth/javahome/1-custom-jvms/build.mill

@@ -0,0 +1,125 @@
+// :link-jvm-indices: https://github.com/coursier/jvm-index/blob/master/indices/linux-arm64.json
+// To override the default Java home, you can create a custom
+// `ZincWorkerModule` and override the `zincWorker` method in your
+// `JavaModule` or `ScalaModule` by pointing it to that custom object:
+
+// == Downloading Jvm By Version
+//
+// The `ZincWorkerModule.ForJvm` class takes a Jvm dependency string as its
+// contructor argument and configures the module to fetch and cache the Jvm
+// using coursier and use it for the compile, run, and test tasks.
+//
+// The string has the form:
+//
+// ----
+// "{name}:{version}"
+// ----
+//
+// To see what Jvms are available for download look at the index for your os
+// and architecture {link-jvm-indices}[here].
+import mill._, javalib._
+import mill.define.ModuleRef
+
+object ZincWorkerJava11 extends ZincWorkerModule {
+  def jvmId = "temurin:18.0.2"
+}
+
+object foo extends JavaModule {
+  def zincWorker = ModuleRef(ZincWorkerJava11)
+
+  object test extends JavaTests with TestModule.Junit4
+}
+
+/** Usage
+
+> mill foo.run
+Foo running on Java 18.0.2
+
+> mill foo.test
+Testing with JVM version: 18.0.2
+Test foo.FooTest.testSimple finished...
+
+*/
+
+// Selecting a custom JVM via `ZincWorkerModule.ForJvm` means that JVM is used for
+// compiling, testing, and running that module via Mill. Note that `.assembly` is not
+// affected, as JVM assembly jars do not bundle a JVM and have to be run using a
+// JVM installed on the target host machine. Configuration is done via `ZincWorkerModule`
+// as the https://github.com/sbt/zinc[Zinc] incremental compiler is used for compiling
+// Java and Scala sources.
+//
+// == Selecting JVM Index Versions
+//
+
+//
+// By default, Mill comes bundled with a version of the JVM index that was published when
+// each version of Mill is released. This ensures that the JVM versions you pick are stable,
+// but means that the latest JVM versions may not be available. You can pass in the JVM
+// index version explicitly via `def jvmIndexVersion` below, choosing a published
+// index version from the Maven Central (https://repo1.maven.org/maven2/io/get-coursier/jvm/indices/index-linux-arm64/[link])
+//
+
+import scalalib._
+
+object ZincWorkerJava11Latest extends ZincWorkerModule {
+  def jvmId = "temurin:23.0.1"
+  def jvmIndexVersion = "latest.release"
+}
+
+object bar extends ScalaModule {
+  def scalaVersion = "2.13.12"
+  def zincWorker = ModuleRef(ZincWorkerJava11Latest)
+
+}
+
+/** Usage
+
+> mill bar.run
+Bar running on Java 23.0.1
+
+*/
+
+// == Explicit JVM Download URLs
+//
+// You can also pass in the JVM download URL explicitly. Note that if you do so, you need
+// to ensure yourself that you are downloading the appropriate JVM distribution for your
+// operating system and CPU architecture. In the example below we switch between Mac/ARM and
+// Linux/X64, but you may have additional cases if you need to support Windows or other
+// OS/CPU combinations
+
+import kotlinlib._
+
+object ZincWorkerJava11Url extends ZincWorkerModule {
+  def jvmId =
+    if (sys.props("os.name") == "Mac OS X") {
+      "https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_aarch64_mac_hotspot_22.0.2_9.tar.gz"
+    } else {
+      "https://github.com/adoptium/temurin22-binaries/releases/download/jdk-22.0.2%2B9/OpenJDK22U-jdk_x64_linux_hotspot_22.0.2_9.tar.gz"
+    }
+
+}
+
+object qux extends KotlinModule {
+  def kotlinVersion = "2.0.20"
+  def zincWorker = ModuleRef(ZincWorkerJava11Url)
+}
+
+/** Usage
+
+> mill qux.run
+Qux running on Java 22.0.2
+
+*/
+
+// == Locally-Installed JVMs
+//
+// Lastly, you can point Mill at any JVM distribution installed locally on disk via:
+
+object ZincWorkerLocalJvm extends ZincWorkerModule {
+  def javaHome = Some(PathRef(os.Path("/my/java/home"), quick = true))
+}
+
+object baz extends JavaModule {
+  def zincWorker = ModuleRef(ZincWorkerLocalJvm)
+
+}

example/depth/javahome/1-custom-jvms/foo/src/foo/Foo.java

@@ -0,0 +1,8 @@
+package foo;
+
+public class Foo {
+
+  public static void main(String[] args) throws Exception {
+    System.out.println("Foo running on Java " + System.getProperty("java.version"));
+  }
+}

example/depth/javahome/1-custom-jvms/foo/test/src/foo/FooTest.java

@@ -0,0 +1,13 @@
+package foo;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class FooTest {
+  @Test
+  public void testSimple() {
+    System.out.println("Testing with JVM version: " + System.getProperty("java.version"));
+    assertEquals(System.getProperty("java.version"), "18.0.2");
+  }
+}

example/depth/javahome/1-custom-jvms/qux/src/qux/Qux.java

@@ -0,0 +1,8 @@
+package qux;
+
+public class Qux {
+
+  public static void main(String[] args) throws Exception {
+    System.out.println("Qux running on Java " + System.getProperty("java.version"));
+  }
+}

example/package.mill

@@ -87,7 +87,7 @@ object `package` extends RootModule with Module {
     object large extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "large"))
 
     object sandbox extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "sandbox"))
-
+    object javahome extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "javahome"))
   }
 
   object extending extends Module {

example/thirdparty/commons-io/build.mill

@@ -1,9 +1,17 @@
 package build
 import mill._, javalib._, publish._
+import mill.define.ModuleRef
 import $ivy.`com.lihaoyi::mill-contrib-jmh:$MILL_VERSION`
 import contrib.jmh.JmhModule
 
 object `package` extends RootModule with PublishModule with MavenModule {
+
+  object ZincWorkerJava11 extends ZincWorkerModule {
+    def jvmId = "temurin:11.0.24"
+  }
+
+  override def zincWorker = ModuleRef(ZincWorkerJava11)
+  def javacOptions = Seq("-encoding", "UTF-8")
   def publishVersion = "2.17.0-SNAPSHOT"
 
   def pomSettings = PomSettings(

example/thirdparty/mockito/build.mill

@@ -1,5 +1,6 @@
 package build
 import mill._, javalib._
+import mill.define.ModuleRef
 
 object libraries {
 
@@ -37,6 +38,8 @@ object libraries {
 }
 
 trait MockitoModule extends MavenModule {
+  override def zincWorker = ModuleRef(build.ZincWorkerJava11)
+  def javacOptions = Seq("-encoding", "UTF-8")
   def testModuleDeps: Seq[JavaModule] = Nil
   def testIvyDeps: T[Agg[Dep]] = Agg.empty[Dep]
   def testRuntimeIvyDeps: T[Agg[Dep]] = Agg.empty[Dep]
@@ -68,6 +71,11 @@ trait MockitoModule extends MavenModule {
 }
 
 object `package` extends RootModule with MockitoModule {
+
+  object ZincWorkerJava11 extends ZincWorkerModule {
+    def jvmId = "temurin:11.0.24"
+  }
+
   def compileIvyDeps = Agg(
     libraries.hamcrest,
     libraries.junit4,

main/api/src/mill/api/JsonFormatters.scala

@@ -21,8 +21,14 @@ trait JsonFormatters {
 
   /**
    * Additional [[mainargs.TokensReader]] instance to teach it how to read Ammonite paths
+   *
+   * Should be replaced by `PathTokensReader2` but kept for binary compatibility
    */
-  implicit def PathTokensReader: mainargs.TokensReader[os.Path] = JsonFormatters.PathTokensReader0
+  implicit def PathTokensReader: mainargs.TokensReader[os.Path] =
+    JsonFormatters.PathTokensReader0
+
+  def PathTokensReader2: mainargs.TokensReader.Simple[os.Path] =
+    JsonFormatters.PathTokensReader0
 
   implicit val pathReadWrite: RW[os.Path] = upickle.default.readwriter[String]
     .bimap[os.Path](

main/package.mill

@@ -100,7 +100,11 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI
 
   object util extends build.MillStableScalaModule {
     def moduleDeps = Seq(api, client)
-    def ivyDeps = Agg(build.Deps.coursier, build.Deps.jline)
+    def ivyDeps = Agg(
+      build.Deps.coursier,
+      build.Deps.coursierJvm,
+      build.Deps.jline
+    )
   }
 
   object define extends build.MillStableScalaModule {