Add e2e tests for sourcelinks of standard library of scala3

Author: BarkingBad

.github/workflows/scaladoc.yaml

@@ -109,3 +109,26 @@ jobs:
             echo uplading docs to https://scala3doc.virtuslab.com/$DOC_DEST
             az storage container create --name $DOC_DEST --account-name scala3docstorage --public-access container
             az storage blob upload-batch -s community-build/docsOutput -d  $DOC_DEST --account-name scala3docstorage
+
+  stdlib-sourcelinks-test:
+    runs-on: ubuntu-latest
+    if: "(    github.event_name == 'pull_request'
+           && !contains(github.event.pull_request.body, '[skip ci]')
+           && !contains(github.event.pull_request.body, '[skip docs]')
+         )
+         || contains(github.event.ref, 'scaladoc')
+         || contains(github.event.ref, 'scala3doc')
+         || contains(github.event.ref, 'master')"
+
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 8
+
+      - name: Test sourcelinks to stdlib
+        run: ./project/scripts/sbt scaladoc/sourceLinksIntegrationTest:test
+

project/Build.scala

@@ -1162,7 +1162,17 @@ object Build {
   // Note: the two tasks below should be one, but a bug in Tasty prevents that
   val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
   val generateTestcasesDocumentation  = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
-  lazy val `scaladoc` = project.in(file("scaladoc")).asScaladoc
+
+  val SourceLinksIntegrationTest = config("sourceLinksIntegrationTest") extend Test
+
+  lazy val `scaladoc` = project.in(file("scaladoc"))
+    .configs(SourceLinksIntegrationTest)
+    .asScaladoc
+    .settings(inConfig(SourceLinksIntegrationTest)(Defaults.testSettings))
+    .settings(
+      scalaSource in SourceLinksIntegrationTest := baseDirectory.value / "test-source-links",
+      test in SourceLinksIntegrationTest := ((test in SourceLinksIntegrationTest) dependsOn generateScalaDocumentation.toTask("")).value,
+    )
   lazy val `scaladoc-nonBootstrapped` = project.in(file("scaladoc")).scaladocBasic(NonBootstrapped).settings(
     // Unit tests in scaladoc depends on scaladoc-testcases
     // we do not want to just cross compile them with bootstrapped and unbootstrapped compiler

scaladoc-testcases/src/toplevel.scala

@@ -1,3 +1,3 @@
 def toplevelDef = 123
 
-class ToplevelClass
+class ToplevelClass

scaladoc/src/dotty/tools/scaladoc/renderers/WikiDocRenderer.scala

@@ -46,7 +46,7 @@ class DocRender(signatureRenderer: SignatureRenderer)(using DocContext):
           case 5 => h5(content)
           case 6 => h6(content)
     case Paragraph(text) => p(renderElement(text))
-    case Code(data: String) => code(raw(data)) // TODO add classes
+    case Code(data: String) => pre(code(data)) // TODO add classes
     case HorizontalRule => hr
 
     case UnorderedList(items) => ul(listItems(items))

scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala

@@ -17,10 +17,9 @@ trait NameNormalizer { self: TastyParser =>
   private val ignoredKeywords: Set[String] = Set("this")
 
   private def escapedName(name: String) =
-    val simpleIdentifierRegex = raw"(?:\w+_[^\[\(\s_]+)|\w+|[^\[\(\s\w_]+".r
+    val simpleIdentifierRegex = "([([{}]) ]|[^A-Za-z0-9$]_)".r
     name match
       case n if ignoredKeywords(n) => n
-      case n if keywords(termName(n)) => s"`$n`"
-      case simpleIdentifierRegex() => name
-      case n => s"`$n`"
+      case n if keywords(termName(n)) || simpleIdentifierRegex.findFirstIn(n).isDefined => s"`$n`"
+      case _ => name
 }

scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala

@@ -114,4 +114,3 @@ trait SyntheticsSupport:
     typeForClass(c).asInstanceOf[dotc.core.Types.Type]
       .memberInfo(symbol.asInstanceOf[dotc.core.Symbols.Symbol])
       .asInstanceOf[TypeRepr]
-

scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala

@@ -66,4 +66,4 @@ case class DocFlexmarkRenderer(renderLink: (DocLink, String) => String)
 object DocFlexmarkRenderer:
   def render(node: Node)(renderLink: (DocLink, String) => String) =
     val opts = MarkdownParser.mkMarkdownOptions(Seq(DocFlexmarkRenderer(renderLink)))
-    HtmlRenderer.builder(opts).build().render(node)
+    HtmlRenderer.builder(opts).escapeHtml(true).build().render(node)

scaladoc/src/dotty/tools/scaladoc/util/html.scala

@@ -11,7 +11,17 @@ object HTML:
   case class Tag(name: String):
     def apply(tags: TagArg*): AppliedTag = apply()(tags:_*)
     def apply(first: AttrArg, rest: AttrArg*): AppliedTag = apply((first +: rest):_*)()
-    def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag = {
+    def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag =
+      def unpackTags(tags: TagArg*)(using sb: StringBuilder): StringBuilder =
+        tags.foreach {
+          case t: AppliedTag =>
+            sb.append(t)
+          case s: String =>
+            sb.append(s.escapeReservedTokens)
+          case s: Seq[AppliedTag | String] =>
+            unpackTags(s:_*)
+        }
+        sb
       val sb = StringBuilder()
       sb.append(s"<$name")
       attrs.filter(_ != Nil).foreach{
@@ -21,22 +31,9 @@ object HTML:
           sb.append(" ").append(e)
       }
       sb.append(">")
-      tags.foreach{
-        case t: AppliedTag =>
-          sb.append(t)
-        case s: String =>
-          sb.append(s.escapeReservedTokens)
-        case s: Seq[AppliedTag | String] =>
-          s.foreach{
-            case a: AppliedTag =>
-              sb.append(a)
-            case s: String =>
-              sb.append(s.escapeReservedTokens)
-          }
-      }
+      unpackTags(tags:_*)(using sb)
       sb.append(s"</$name>")
       sb
-    }
 
   extension (s: String) private def escapeReservedTokens: String =
     s.replace("&", "&amp;")
@@ -83,7 +80,7 @@ object HTML:
   val ol = Tag("ol")
   val li = Tag("li")
   val code = Tag("code")
-
+  val pre = Tag("pre")
 
   val cls = Attr("class")
   val href = Attr("href")

scaladoc/test-source-links/dotty/tools/scaladoc/source-links/RemoteLinksTest.scala

@@ -0,0 +1,66 @@
+package dotty.tools.scaladoc
+package sourcelinks
+
+import scala.io.Source
+import scala.jdk.CollectionConverters._
+import scala.util.matching.Regex
+import dotty.tools.scaladoc.test.BuildInfo
+import java.nio.file.Path
+import java.nio.file.Paths
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import util.IO
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+class RemoteLinksTest:
+
+  @Test
+  def runTest =
+    val mtsl = membersToSourceLinks(using testDocContext())
+    val pageToMtsl: Map[String, List[(String, String)]] = mtsl.groupMap(_._2.split("#L").head)(v => (v._1, v._2.split("#L").last))
+    pageToMtsl.foreach { case (link, members) =>
+      try
+        val doc = getDocumentFromUrl(link)
+        members.foreach { (member, line) =>
+          if !member.startsWith("given_") then // TODO: handle synthetic givens, for now we disable them from testing
+            val loc = doc.select(s"#LC$line").text
+            val memberToMatch = member.replace("`", "")
+            assertTrue(s"Expected to find $memberToMatch at $link at line $line", loc.contains(memberToMatch))
+        }
+      catch
+        case e: java.lang.IllegalArgumentException =>
+          report.error(s"Could not open link for $link - invalid URL")(using testContext)
+        case e: org.jsoup.HttpStatusException => e.getStatusCode match
+          case 404 => throw AssertionError(s"Page $link does not exists")
+          case n   => report.warning(s"Could not open link for $link, return code $n")(using testContext)
+    }
+    assertNoErrors(testContext.reportedDiagnostics)
+
+  private def getDocumentFromUrl(link: String): Document =
+    try
+      Jsoup.connect(link).get
+    catch
+      case e: org.jsoup.HttpStatusException => e.getStatusCode match
+          case 429 =>
+            Thread.sleep(10)
+            getDocumentFromUrl(link)
+          case n =>
+            throw e
+
+  private def membersToSourceLinks(using DocContext): List[(String, String)] =
+    val output = Paths.get("scaladoc", "output", "scala3", "api").toAbsolutePath
+    val mtsl = List.newBuilder[(String, String)]
+    def processFile(path: Path): Unit =
+      val document = Jsoup.parse(IO.read(path))
+      if document.select("span.kind").first.text == "package" then
+        document.select(".documentableElement").forEach { dElem =>
+          if dElem.select("span.kind").first.text != "package" then
+            dElem.select("dt").forEach { elem =>
+              val content = elem.text
+              if content == "Source" then
+                mtsl += dElem.select(".documentableName").first.text -> elem.nextSibling.childNode(0).attr("href")
+            }
+        }
+    IO.foreachFileIn(output, processFile)
+    mtsl.result