Merge pull request #10812 from romanowski/scala3doc/before-m3

Scala3doc/before m3

Author: abgruszecki

project/Build.scala

@@ -1528,14 +1528,9 @@ object Build {
       def generateDocumentation(targets: String, name: String, outDir: String, ref: String, params: String = "") = Def.taskDyn {
           val projectVersion = version.value
           IO.createDirectory(file(outDir))
-          val managedSources =
-            (`stdlib-bootstrapped`/Compile/sourceManaged).value / "scala-library-src"
-          val projectRoot = (ThisBuild/baseDirectory).value.toPath
-          val stdLibRoot = projectRoot.relativize(managedSources.toPath.normalize())
-          val scalaSourceLink =
-             s"$stdLibRoot=github://scala/scala/v${stdlibVersion(Bootstrapped)}#src/library"
-          val sourcesAndRevision = s"-source-links $scalaSourceLink,github://lampepfl/dotty  -revision $ref -project-version $projectVersion"
-          val cmd = s""" -d $outDir -project "$name" $sourcesAndRevision $params $targets"""
+          val sourceLinks = "-source-links:github://lampepfl/dotty "
+          val revision = s"-revision $ref -project-version $projectVersion"
+          val cmd = s""" -d $outDir -project "$name" $sourceLinks $revision $params $targets"""
           run.in(Compile).toTask(cmd)
       }
 
@@ -1575,12 +1570,9 @@ object Build {
               classDirectory.in(Compile).value.getAbsolutePath,
               "scala3doc", "scala3doc/output/self", VersionUtil.gitHash,
               "-siteroot scala3doc/documentation -project-logo scala3doc/documentation/logo.svg " +
-              "-external-mappings " + raw".*scala.*" + "::" +
-                "scala3doc" + "::" +
-                "http://dotty.epfl.ch/api/" + ":::" +
-                raw".*java.*" + "::" +
-                "javadoc" + "::" +
-                "https://docs.oracle.com/javase/8/docs/api/"
+              "-external-mappings:" +
+                ".*scala.*::scala3doc::http://dotty.epfl.ch/api/," +
+                ".*java.*::javadoc::https://docs.oracle.com/javase/8/docs/api/"
             )
           }.value,
 
@@ -1597,6 +1589,12 @@ object Build {
 
             val roots = joinProducts(dottyJars)
 
+            val managedSources =
+              (`stdlib-bootstrapped`/Compile/sourceManaged).value / "scala-library-src"
+            val projectRoot = (ThisBuild/baseDirectory).value.toPath
+            val stdLibRoot = projectRoot.relativize(managedSources.toPath.normalize())
+            val docRootFile = stdLibRoot.resolve("rootdoc.txt")
+
             if (dottyJars.isEmpty) Def.task { streams.value.log.error("Dotty lib wasn't found") }
             else Def.task{
               IO.write(dest / "versions" / "latest-nightly-base", majorVersion)
@@ -1615,9 +1613,9 @@ object Build {
               "-skip-by-regex:.+\\.internal($|\\..+) " +
               "-skip-by-regex:.+\\.impl($|\\..+) " +
               "-comment-syntax wiki -siteroot scala3doc/scala3-docs -project-logo scala3doc/scala3-docs/logo.svg " +
-              "-external-mappings " + raw".*java.*" + "::" +
-                "javadoc" + "::" +
-                "https://docs.oracle.com/javase/8/docs/api/"
+              "-external-mappings:.*java.*::javadoc::https://docs.oracle.com/javase/8/docs/api/ " +
+              s"-source-links:$stdLibRoot=github://scala/scala/v${stdlibVersion(Bootstrapped)}#src/library " +
+              s"-doc-root-content $docRootFile"
               ))
           }.evaluated,
 

scala3doc/src/dotty/dokka/DocContext.scala

@@ -69,6 +69,9 @@ extension (r: report.type)
   def warn(m: String, f: File)(using CompilerContext): Unit =
     r.warning(createMessage(m, f, null), sourcePostionFor(f))
 
+  def warn(m: String, e: Throwable)(using CompilerContext): Unit =
+    r.warning(s"$m: ${throwableToString(e)}")
+
 case class NavigationNode(name: String, dri: DRI, nested: Seq[NavigationNode])
 
 case class DocContext(args: Scala3doc.Args, compilerContext: CompilerContext)
@@ -99,51 +102,7 @@ case class DocContext(args: Scala3doc.Args, compilerContext: CompilerContext)
         sourceLinks
       )(using compilerContext))
 
-    def parseDocTool(docTool: String) = docTool match {
-      case "scaladoc" => Some(DocumentationKind.Scaladoc)
-      case "scala3doc" => Some(DocumentationKind.Scala3doc)
-      case "javadoc" => Some(DocumentationKind.Javadoc)
-      case other => None
-    }
-    val externalDocumentationLinks: List[Scala3docExternalDocumentationLink] = args.externalMappings.filter(_.size >= 3).flatMap { mapping =>
-      val regexStr = mapping(0)
-      val docTool = mapping(1)
-      val urlStr = mapping(2)
-      val packageListUrlStr = if mapping.size > 3 then Some(mapping(3)) else None
-      val regex = Try(regexStr.r).toOption
-      val url = Try(URL(urlStr)).toOption
-      val packageListUrl = Try(packageListUrlStr.map(URL(_)))
-        .fold(
-          e => {
-          logger.warn(s"Wrong packageListUrl parameter in external mapping. Found '$packageListUrlStr'. " +
-            s"Package list url will be omitted")
-          None},
-          res => res
-        )
-
-      val parsedDocTool = parseDocTool(docTool)
-      val res = if regexStr.isEmpty then
-        logger.warn(s"Wrong regex parameter in external mapping. Found '$regexStr'. Mapping will be omitted")
-        None
-      else if url.isEmpty then
-        logger.warn(s"Wrong url parameter in external mapping. Found '$urlStr'. Mapping will be omitted")
-        None
-      else if parsedDocTool.isEmpty then
-        logger.warn(s"Wrong doc-tool parameter in external mapping. " +
-          s"Expected one of: 'scaladoc', 'scala3doc', 'javadoc'. Found:'$docTool'.  Mapping will be omitted "
-        )
-        None
-      else
-        Some(
-          Scala3docExternalDocumentationLink(
-            List(regexStr.r),
-            URL(urlStr),
-            parsedDocTool.get,
-            packageListUrlStr.map(URL(_))
-          )
-        )
-      res
-    }
+    val externalDocumentationLinks = args.externalMappings
 
     override def getPluginsConfiguration: JList[DokkaConfiguration.PluginConfiguration] =
       JNil

scala3doc/src/dotty/dokka/ExternalDocLink.scala

@@ -0,0 +1,57 @@
+package dotty.dokka
+
+import java.net.URL
+import scala.util.matching._
+import scala.util.Try
+
+case class ExternalDocLink(
+  originRegexes: List[Regex],
+  documentationUrl: URL,
+  kind: DocumentationKind,
+  packageListUrl: Option[URL] = None
+):
+  def withPackageList(url: URL): ExternalDocLink = copy(packageListUrl = Some(url))
+
+enum DocumentationKind:
+  case Javadoc extends DocumentationKind
+  case Scaladoc extends DocumentationKind
+  case Scala3doc extends DocumentationKind
+
+object ExternalDocLink:
+  def parse(mapping: String)(using CompilerContext): Option[ExternalDocLink] =
+    def fail(msg: String) =
+      report.warning(s"Unable to parocess external mapping $mapping. $msg")
+      None
+
+    def tryParse[T](descr: String)(op: => T): Option[T] = try Some(op) catch
+      case e: RuntimeException =>
+        report.warn(s"Unable to parse $descr", e)
+        None
+
+    def parsePackageList(elements: List[String]) = elements match
+     case List(urlStr) => tryParse("packageList")(Option(URL(urlStr)))
+     case Nil => Some(None)
+     case other => fail(s"Provided multiple package lists: $other")
+
+    def doctoolByName(name: String) = name match
+      case "javadoc" => Some(DocumentationKind.Javadoc)
+      case "scaladoc" => Some(DocumentationKind.Scaladoc)
+      case "scala3doc" => Some(DocumentationKind.Scala3doc)
+      case other => fail(s"Unknown doctool: $other")
+
+
+    mapping.split("::").toList match
+      case regexStr :: docToolStr :: urlStr :: rest =>
+        for {
+          regex <- tryParse("regex")(regexStr.r)
+          url <- tryParse("url")(URL(urlStr))
+          doctool <- doctoolByName(docToolStr)
+          packageList <- parsePackageList(rest)
+        } yield ExternalDocLink(
+          List(regex),
+          url,
+          doctool,
+          packageList
+        )
+      case _ =>
+        fail("Accepted format: `regexStr::docToolStr::urlStr[::rest]`")

scala3doc/src/dotty/dokka/IO.java

@@ -43,4 +43,8 @@ public FileVisitResult visitFile(
   public static String read(Path path) throws IOException {
     return new String(Files.readAllBytes(path), Charset.defaultCharset());
   }
+
+  public static String read(String path) throws IOException {
+    return read(Paths.get(path));
+  }
 }

scala3doc/src/dotty/dokka/Scala3doc.scala

@@ -65,9 +65,10 @@ object Scala3doc:
     defaultSyntax: CommentSyntax = CommentSyntax.Markdown,
     sourceLinks: List[String] = Nil,
     revision: Option[String] = None,
-    externalMappings: List[List[String]] = List.empty,
+    externalMappings: List[ExternalDocLink] = Nil,
     identifiersToSkip: List[String] = Nil,
     regexesToSkip: List[String] = Nil,
+    rootDocPath: Option[String] = None
   )
 
   def run(args: Array[String], rootContext: CompilerContext): Reporter =

scala3doc/src/dotty/dokka/Scala3docArgs.scala

@@ -25,25 +25,34 @@ class Scala3docArgs extends SettingGroup with CommonScalaSettings:
     sourcepath, sourceroot
   )
 
-  val sourceLinks: Setting[String] =
-    StringSetting("-source-links", "sources", SourceLinks.usage, "")
+  val sourceLinks: Setting[List[String]] =
+    MultiStringSetting("-source-links", "sources", SourceLinks.usage)
 
   val syntax: Setting[String] =
     StringSetting("-comment-syntax", "syntax", "Syntax of the comment used", "")
 
   val revision: Setting[String] =
     StringSetting("-revision", "revision", "Revision (branch or ref) used to build project project", "")
 
-  val externalDocumentationMappings: Setting[String] =
-    StringSetting("-external-mappings", "external-mappings", "Mapping between regex matching class file and external documentation", "")
+  val externalDocumentationMappings: Setting[List[String]] =
+    MultiStringSetting("-external-mappings", "external-mappings",
+      "Mapping between regexes matching classpath entries and external documentation. " +
+        "'regex::[scaladoc|scala3doc|javadoc]::path' syntax is used")
+
+  val deprecatedSkipPackages: Setting[List[String]] =
+    MultiStringSetting("-skip-packages", "packages", "Deprecated, please use `-skip-by-id` or `-skip-by-regex`")
 
   val skipById: Setting[List[String]] =
     MultiStringSetting("-skip-by-id", "package or class identifier", "Identifiers of packages or top-level classes to skip when generating documentation")
 
   val skipByRegex: Setting[List[String]] =
     MultiStringSetting("-skip-by-regex", "regex", "Regexes that match fully qualified names of packages or top-level classes to skip when generating documentation")
 
-  def scala3docSpecificSettings: Set[Setting[_]] = Set(sourceLinks, syntax, revision, externalDocumentationMappings, skipById, skipByRegex)
+  val docRootContent: Setting[String] =
+    StringSetting("-doc-root-content", "path", "The file from which the root package documentation should be imported.", "")
+
+  def scala3docSpecificSettings: Set[Setting[_]] =
+    Set(sourceLinks, syntax, revision, externalDocumentationMappings, skipById, skipByRegex, deprecatedSkipPackages, docRootContent)
 
 object Scala3docArgs:
   def extract(args: List[String], rootCtx: CompilerContext):(Scala3doc.Args, CompilerContext) =
@@ -104,7 +113,8 @@ object Scala3docArgs:
         CommentSyntax.default
       }
     }
-    val externalMappings = externalDocumentationMappings.get.split(":::").map(_.split("::").toList).toList
+    val externalMappings =
+      externalDocumentationMappings.get.flatMap(ExternalDocLink.parse)
 
     unsupportedSettings.filter(s => s.get != s.default).foreach { s =>
       report.warning(s"Setting ${s.name} is currently not supported.")
@@ -114,6 +124,8 @@ object Scala3docArgs:
     report.inform(
       s"Generating documenation $printableProjectName in $destFile")
 
+    if deprecatedSkipPackages.get.nonEmpty then report.warning(deprecatedSkipPackages.description)
+
     val docArgs = Args(
       projectName.withDefault("root"),
       dirs,
@@ -124,10 +136,11 @@ object Scala3docArgs:
       projectVersion.nonDefault,
       projectLogo.nonDefault,
       parseSyntax,
-      sourceLinks.nonDefault.fold(Nil)(_.split(",").toList),
+      sourceLinks.get,
       revision.nonDefault,
       externalMappings,
-      skipById.get,
+      skipById.get ++ deprecatedSkipPackages.get,
       skipByRegex.get,
+      docRootContent.nonDefault
     )
     (docArgs, newContext)

scala3doc/src/dotty/dokka/ScalaModuleCreator.scala

@@ -16,7 +16,7 @@ import kotlin.coroutines.Continuation
 
 class ScalaModuleProvider(using ctx: DocContext) extends SourceToDocumentableTranslator:
    override def invoke(sourceSet: DokkaSourceSet, cxt: DokkaContext, unused: Continuation[? >: DModule]) =
-    val result = DokkaTastyInspector(new MarkdownParser(_ => null)).result()
+    val (result, rootDoc) = DokkaTastyInspector(new MarkdownParser(_ => null)).result()
     val (rootPck, rest) = result.partition(_.name == "<empty>")
     val packageMembers = (rest ++ rootPck.flatMap(_.allMembers)).sortBy(_.name)
 
@@ -32,7 +32,7 @@ class ScalaModuleProvider(using ctx: DocContext) extends SourceToDocumentableTra
       null,
       JSet(ctx.sourceSet),
       PropertyContainer.Companion.empty()
-    ).withNewMembers(packageMembers).withKind(Kind.RootPackage).asInstanceOf[DPackage]
+    ).withNewMembers(packageMembers).withKind(Kind.RootPackage).withDocs(rootDoc).asInstanceOf[DPackage]
 
     new DModule(
       sourceSet.getDisplayName,

scala3doc/src/dotty/dokka/externalDocumentationLinks.scala

@@ -79,6 +79,10 @@ extension (member: Member)
     val ext = MemberExtension.getFrom(member).getOrElse(MemberExtension.empty).copy(kind = kind)
     putInMember(ext)
 
+  def withDocs(docs: Option[Comment]): Member =
+    val ext = MemberExtension.getFrom(member).getOrElse(MemberExtension.empty).copy(rawDoc = docs)
+    putInMember(ext)
+
   def withMembers(newMembers: Seq[Member]): Member =
     val original = member.compositeMemberExt.getOrElse(CompositeMemberExtension())
     val newExt = original.copy(members = newMembers)

scala3doc/src/dotty/dokka/model/api/internalExtensions.scala

@@ -29,16 +29,10 @@ trait BasicSupport:
     Annotation(dri, params)
 
   extension (sym: Symbol)
-    def documentation = sym.docstring match
-      case Some(docstring) =>
-          Map(ctx.sourceSet -> parseComment(docstring, sym.tree))
-      case None =>
-          Map.empty
-
-    def documentation2 = sym.docstring.map(preparseComment(_, sym.tree))
+    def documentation = sym.docstring.map(parseComment(_, sym.tree))
 
     def source(using Quotes) =
-      val path = sym.pos.filter(isValidPos(_)).map(_.sourceFile.jpath).filter(_ != null).map(_.toAbsolutePath).map(_.toString)
+      val path = sym.pos.map(_.sourceFile.jpath).filter(_ != null).map(_.toAbsolutePath).map(_.toString)
       path.map(TastyDocumentableSource(_, sym.pos.get.startLine))
 
     def getAnnotations(): List[Annotation] =

scala3doc/src/dotty/dokka/tasty/BasicSupport.scala

@@ -465,7 +465,7 @@ trait ClassLikeSupport:
         placeholderModifier,
         ctx.sourceSet.toSet,
         /*isExpectActual =*/ false,
-        PropertyContainer.Companion.empty().plus(member.copy(rawDoc = symbol.documentation2)).plus(compositeExt)
+        PropertyContainer.Companion.empty().plus(member.copy(rawDoc = symbol.documentation)).plus(compositeExt)
     )
 
   private def isUsingModifier(parameters: Seq[ValDef]): Boolean =