Pass Mill source JARs to Bloop

contrib/bloop/src/mill/contrib/bloop/Bloop.scala

@@ -8,5 +8,6 @@ import mill.api.WorkspaceRoot
  */
 object Bloop extends BloopImpl(
       () => Evaluator.allBootstrapEvaluators.value.value,
-      WorkspaceRoot.workspaceRoot
+      WorkspaceRoot.workspaceRoot,
+      addMillSources = None
     )

contrib/bloop/src/mill/contrib/bloop/BloopImpl.scala

@@ -5,19 +5,26 @@ import mill._
 import mill.api.Result
 import mill.define.{Discover, ExternalModule, Module => MillModule}
 import mill.eval.Evaluator
+import mill.main.BuildInfo
 import mill.scalalib.internal.JavaModuleUtils
 import mill.scalajslib.ScalaJSModule
 import mill.scalajslib.api.{JsEnvConfig, ModuleKind}
 import mill.scalalib._
 import mill.scalanativelib.ScalaNativeModule
 import mill.scalanativelib.api.ReleaseMode
 
+import java.nio.file.{FileSystemNotFoundException, Files, Paths}
+
 /**
  * Implementation of the Bloop related tasks. Inherited by the
  * `mill.contrib.bloop.Bloop` object, and usable in tests by passing
  * a custom evaluator.
  */
-class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
+class BloopImpl(
+    evs: () => Seq[Evaluator],
+    wd: os.Path,
+    addMillSources: Option[Boolean]
+) extends ExternalModule {
   outer =>
   import BloopFormats._
 
@@ -28,7 +35,10 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
    * under pwd/.bloop.
    */
   def install() = Task.Command {
-    val res = Task.traverse(computeModules)(_.bloop.writeConfigFile())()
+    val res = Task.traverse(computeModules0) {
+      case (mod, isRoot) =>
+        mod.bloop.writeConfigFile(isRoot)
+    }()
     val written = res.map(_._2).map(_.path)
     // Make bloopDir if it doesn't exists
     if (!os.exists(bloopDir)) {
@@ -62,7 +72,7 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
 
     object bloop extends MillModule {
       def config = Task {
-        new BloopOps(self).bloop.config()
+        bloopConfig(self, false)
       }
     }
 
@@ -85,19 +95,20 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
     override def millOuterCtx = jm.millOuterCtx
 
     object bloop extends MillModule {
-      def config = Task { outer.bloopConfig(jm) }
+      @deprecated("", "")
+      def config = Task.Anon { outer.bloopConfig(jm, false) }
 
-      def writeConfigFile(): Command[(String, PathRef)] = Task.Command {
+      def writeConfigFile(isRootModule: Boolean): Command[(String, PathRef)] = Task.Command {
         os.makeDir.all(bloopDir)
         val path = bloopConfigPath(jm)
-        _root_.bloop.config.write(config(), path.toNIO)
+        _root_.bloop.config.write(outer.bloopConfig(jm, isRootModule)(), path.toNIO)
         Task.log.info(s"Wrote $path")
         name(jm) -> PathRef(path)
       }
 
       @deprecated("Use writeConfigFile instead.", "Mill after 0.10.9")
       def writeConfig: T[(String, PathRef)] = Task {
-        writeConfigFile()()
+        writeConfigFile(isRootModule = false)()
       }
     }
 
@@ -116,16 +127,23 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
     JavaModuleUtils.transitiveModules(mod, accept)
   }
 
-  protected def computeModules: Seq[JavaModule] = {
-    val evals = evs()
-    evals.flatMap { eval =>
-      if (eval != null)
-        JavaModuleUtils.transitiveModules(eval.rootModule, accept)
+  protected def computeModules: Seq[JavaModule] =
+    computeModules0.map(_._1)
+
+  /**
+   * All modules that are meant to be imported in Bloop
+   *
+   * @return sequence of modules and a boolean indicating whether the module is a root one
+   */
+  protected def computeModules0: Seq[(JavaModule, Boolean)] =
+    evs()
+      .filter(_ != null)
+      .flatMap { eval =>
+        val rootModule = eval.rootModule
+        JavaModuleUtils.transitiveModules(rootModule, accept)
           .collect { case jm: JavaModule => jm }
-      else
-        Seq.empty
-    }
-  }
+          .map(mod => (mod, mod == rootModule))
+      }
 
   // class-based pattern matching against path-dependant types doesn't seem to work.
   private def accept(module: MillModule): Boolean =
@@ -156,7 +174,7 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
   // Computation of the bloop configuration for a specific module
   // ////////////////////////////////////////////////////////////////////////////
 
-  def bloopConfig(module: JavaModule): Task[BloopConfig.File] = {
+  def bloopConfig(module: JavaModule, isRootModule: Boolean): Task[BloopConfig.File] = {
     import _root_.bloop.config.Config
     def out(m: JavaModule) = bloopDir / "out" / name(m)
     def classes(m: JavaModule) = out(m) / "classes"
@@ -212,11 +230,20 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
         module.localCompileClasspath().map(_.path)
     }
 
-    val runtimeClasspath = Task.Anon {
-      module.transitiveModuleDeps.map(classes) ++
-        module.resolvedRunIvyDeps().map(_.path) ++
-        module.unmanagedClasspath().map(_.path)
+    val isTestModule = module match {
+      case _: TestModule => true
+      case _ => false
     }
+    val runtimeClasspathOpt =
+      if (isTestModule)
+        Task.Anon(None)
+      else
+        Task.Anon {
+          val cp = module.transitiveModuleDeps.map(classes) ++
+            module.resolvedRunIvyDeps().map(_.path) ++
+            module.unmanagedClasspath().map(_.path)
+          Some(cp.map(_.toNIO).toList)
+        }
 
     val compileResources =
       Task.Anon(module.compileResources().map(_.path.toNIO).toList)
@@ -283,10 +310,7 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
             ),
             mainClass = module.mainClass(),
             runtimeConfig = None,
-            classpath = module match {
-              case _: TestModule => None
-              case _ => Some(runtimeClasspath().map(_.toNIO).toList)
-            },
+            classpath = runtimeClasspathOpt(),
             resources = Some(runtimeResources())
           )
         }
@@ -389,15 +413,95 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
       gatherTask.unsafeRun()
     }
 
-    val bloopResolution: Task[BloopConfig.Resolution] = Task.Anon {
+    val millBuildDependencies: Task[List[BloopConfig.Module]] = Task.Anon {
+
+      val result = module.defaultResolver().getArtifacts(
+        BuildInfo.millEmbeddedDeps
+          .split(',')
+          .filter(_.nonEmpty)
+          .map { str =>
+            str.split(":", 3) match {
+              case Array(org, name, ver) =>
+                val module =
+                  coursier.Module(coursier.Organization(org), coursier.ModuleName(name), Map.empty)
+                coursier.Dependency(module, ver)
+              case other =>
+                sys.error(
+                  s"Unexpected misshapen entry in BuildInfo.millEmbeddedDeps ('$str', expected 'org:name')"
+                )
+            }
+          },
+        sources = true
+      )
+
+      def moduleOf(dep: coursier.Dependency): BloopConfig.Module =
+        BloopConfig.Module(
+          dep.module.organization.value,
+          dep.module.name.value,
+          dep.version,
+          Some(dep.configuration.value).filter(_.nonEmpty),
+          Nil
+        )
+
+      val indices = result.fullDetailedArtifacts
+        .map {
+          case (dep, _, _, _) =>
+            moduleOf(dep)
+        }
+        .zipWithIndex
+        .reverseIterator
+        .toMap
+
+      result.fullDetailedArtifacts
+        .groupBy {
+          case (dep, _, _, _) =>
+            moduleOf(dep)
+        }
+        .toList
+        .sortBy {
+          case (mod, _) =>
+            indices(mod)
+        }
+        .map {
+          case (mod, artifacts) =>
+            mod.copy(
+              artifacts = artifacts.toList.collect {
+                case (_, pub, art, Some(file)) =>
+                  BloopConfig.Artifact(
+                    pub.name,
+                    Some(pub.classifier.value).filter(_.nonEmpty),
+                    None,
+                    file.toPath
+                  )
+              }
+            )
+        }
+    }
+
+    val bloopDependencies: Task[List[BloopConfig.Module]] = Task.Anon {
       val repos = module.allRepositories()
       // same as input of resolvedIvyDeps
       val coursierDeps = Seq(
         module.coursierDependency.withConfiguration(coursier.core.Configuration.provided),
         module.coursierDependency
       )
-      BloopConfig.Resolution(artifacts(repos, coursierDeps))
+      artifacts(repos, coursierDeps)
+    }
+
+    val addMillSources0 = addMillSources.getOrElse {
+      // We only try to resolve Mill's dependencies to get their source JARs
+      // if we're not running from sources. If Mill is running purely from its sources
+      // it's not is published in ~/.ivy2/local, so we can't get its source JARs.
+      !BloopImpl.isMillRunningFromSources
     }
+    val allBloopDependencies: Task[List[BloopConfig.Module]] =
+      // Add Mill source JARs to root modules
+      if (isRootModule && addMillSources0)
+        Task.Anon {
+          bloopDependencies() ::: millBuildDependencies()
+        }
+      else
+        bloopDependencies
 
     // //////////////////////////////////////////////////////////////////////////
     //  Tying up
@@ -432,7 +536,7 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
         sbt = None,
         test = testConfig(),
         platform = Some(platform()),
-        resolution = Some(bloopResolution()),
+        resolution = Some(BloopConfig.Resolution(allBloopDependencies())),
         tags = Some(tags),
         sourceGenerators = None // TODO: are we supposed to hook generated sources here?
       )
@@ -448,3 +552,20 @@ class BloopImpl(evs: () => Seq[Evaluator], wd: os.Path) extends ExternalModule {
 
   lazy val millDiscover: Discover = Discover[this.type]
 }
+
+object BloopImpl {
+  // If the class of BloopImpl is loaded from a directory rather than a JAR,
+  // then we're running from sources (the directory should be something
+  // like mill-repo/out/contrib/bloop/compile.dest/classes).
+  private lazy val isMillRunningFromSources =
+    Option(classOf[BloopImpl].getProtectionDomain.getCodeSource)
+      .flatMap(s => Option(s.getLocation))
+      .flatMap { url =>
+        try Some(Paths.get(url.toURI))
+        catch {
+          case _: FileSystemNotFoundException => None
+          case _: IllegalArgumentException => None
+        }
+      }
+      .exists(Files.isDirectory(_))
+}

contrib/bloop/test/src/mill/contrib/bloop/BloopTests.scala

@@ -18,7 +18,11 @@ object BloopTests extends TestSuite {
 
   val unitTester = UnitTester(build, null)
   val workdir = unitTester.evaluator.rootModule.millSourcePath
-  val testBloop = new BloopImpl(() => Seq(unitTester.evaluator), workdir)
+  val testBloop = new BloopImpl(
+    () => Seq(unitTester.evaluator),
+    workdir,
+    addMillSources = None
+  )
 
   object build extends TestBaseModule {
     object scalaModule extends scalalib.ScalaModule with testBloop.Module {

developer.adoc

@@ -168,7 +168,7 @@ To test bootstrapping of Mill's own Mill build using a version of Mill built fro
 ci/patch-mill-bootstrap.sh
 ----
 
-This creates a standalone assembly at `target/mill-release` you can use, which references jars
+This creates a standalone assembly at `mill-assembly.jar` you can use, which references jars
 published locally in your `~/.ivy2/local` cache, and applies any necessary patches to
 `build.mill` to deal with changes in Mill between the version specified in `.config/mill-version`
 that is normally used to build Mill and the `HEAD` version your assembly was created from.

integration/ide/bloop/src/BloopTests.scala

@@ -15,13 +15,6 @@ object BloopTests extends UtestIntegrationTestSuite {
         assert(installResult)
         assert(os.exists(workspacePath / ".bloop/root-module.json"))
       }
-      test("mill-build module bloop config should be created") - integrationTest { tester =>
-        import tester._
-        val installResult: Boolean = eval("mill.contrib.bloop.Bloop/install").isSuccess
-        val millBuildJsonFile = workspacePath / ".bloop/mill-build-.json"
-        assert(installResult)
-        assert(os.exists(millBuildJsonFile))
-      }
 
       test("mill-build config should contain build.mill source") - integrationTest { tester =>
         import tester._
@@ -32,6 +25,25 @@ object BloopTests extends UtestIntegrationTestSuite {
         assert(config("project")("sources").arr.exists(path =>
           os.Path(path.str).last == "build.mill"
         ))
+
+        if (isPackagedLauncher) {
+          val modules = config("project")("resolution")("modules").arr
+
+          def sourceJars(org: String, namePrefix: String) = modules
+            .filter(mod => mod("organization").str == org && mod("name").str.startsWith(namePrefix))
+            .flatMap(_("artifacts").arr)
+            .filter(_("classifier").strOpt.contains("sources"))
+            .flatMap(_("path").strOpt)
+            .filter(_.endsWith("-sources.jar"))
+            .distinct
+
+          // Look for one of Mill's own source JARs
+          val millScalaLibSourceJars = sourceJars("com.lihaoyi", "mill-scalalib_")
+          assert(millScalaLibSourceJars.nonEmpty)
+          // Look for a source JAR of a dependency
+          val coursierCacheSourceJars = sourceJars("io.get-coursier", "coursier-cache_")
+          assert(coursierCacheSourceJars.nonEmpty)
+        }
       }
     }
   }

integration/package.mill

@@ -49,13 +49,17 @@ object `package` extends RootModule {
           IntegrationTestModule.this.forkEnv() ++
           Map(
             "MILL_INTEGRATION_SERVER_MODE" -> (mode == "local" || mode == "server").toString,
+            "MILL_INTEGRATION_IS_PACKAGED_LAUNCHER" -> millIntegrationIsPackagedLauncher().toString,
             "MILL_LAUNCHER" -> build.dist.bootstrapLauncher().path.toString,
             "MILL_LAUNCHER_BAT" -> build.dist.bootstrapLauncherBat().path.toString,
             "MILL_INTEGRATION_LAUNCHER" -> millIntegrationLauncher().path.toString
           )
 
       def millIntegrationLauncher: T[PathRef]
 
+      /** Whether the Mill JARs are published locally alongside this Mill launcher */
+      def millIntegrationIsPackagedLauncher: Task[Boolean]
+
       def forkArgs = Task { super.forkArgs() ++ build.dist.forkArgs() }
 
       def resources = IntegrationTestModule.this.resources()
@@ -72,20 +76,28 @@ object `package` extends RootModule {
 
     object local extends IntegrationLauncherModule {
       def millIntegrationLauncher = build.dist.launcher()
+      def millIntegrationIsPackagedLauncher = Task(false)
     }
     object packaged extends IntegrationLauncherModule {
       def millIntegrationLauncher = build.dist.executable()
+      def millIntegrationIsPackagedLauncher = Task(true)
     }
     object native extends IntegrationLauncherModule {
       def millIntegrationLauncher = build.dist.native.executable()
+      def millIntegrationIsPackagedLauncher = Task(true)
     }
     trait IntegrationLauncherModule extends Module {
       def millIntegrationLauncher: T[PathRef]
+      def millIntegrationIsPackagedLauncher: Task[Boolean]
       object fork extends ModeModule {
         def millIntegrationLauncher = IntegrationLauncherModule.this.millIntegrationLauncher
+        def millIntegrationIsPackagedLauncher =
+          IntegrationLauncherModule.this.millIntegrationIsPackagedLauncher
       }
       object server extends ModeModule {
         def millIntegrationLauncher = IntegrationLauncherModule.this.millIntegrationLauncher
+        def millIntegrationIsPackagedLauncher =
+          IntegrationLauncherModule.this.millIntegrationIsPackagedLauncher
       }
     }
   }