Add e2e tests for sourcelinks of standard library of scala3

BarkingBad · BarkingBad · commit 50d1a5975a76 · 2021-03-04T17:18:22.000+01:00
diff --git a/.github/workflows/scaladoc.yaml b/.github/workflows/scaladoc.yaml
@@ -109,3 +109,26 @@ jobs:
             echo uplading docs to https://scala3doc.virtuslab.com/$DOC_DEST
             az storage container create --name $DOC_DEST --account-name scala3docstorage --public-access container
             az storage blob upload-batch -s community-build/docsOutput -d  $DOC_DEST --account-name scala3docstorage
+
+  stdlib-sourcelinks-test:
+    runs-on: ubuntu-latest
+    if: "(    github.event_name == 'pull_request'
+           && !contains(github.event.pull_request.body, '[skip ci]')
+           && !contains(github.event.pull_request.body, '[skip docs]')
+         )
+         || contains(github.event.ref, 'scaladoc')
+         || contains(github.event.ref, 'scala3doc')
+         || contains(github.event.ref, 'master')"
+
+    steps:
+      - name: Git Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up JDK 8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 8
+
+      - name: Test sourcelinks to stdlib
+        run: ./project/scripts/sbt scaladoc/sourceLinksIntegrationTest:test
+
diff --git a/project/Build.scala b/project/Build.scala
@@ -1162,7 +1162,17 @@ object Build {
   // Note: the two tasks below should be one, but a bug in Tasty prevents that
   val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib")
   val generateTestcasesDocumentation  = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests")
-  lazy val `scaladoc` = project.in(file("scaladoc")).asScaladoc
+
+  val SourceLinksIntegrationTest = config("sourceLinksIntegrationTest") extend Test
+
+  lazy val `scaladoc` = project.in(file("scaladoc"))
+    .configs(SourceLinksIntegrationTest)
+    .asScaladoc
+    .settings(inConfig(SourceLinksIntegrationTest)(Defaults.testSettings))
+    .settings(
+      scalaSource in SourceLinksIntegrationTest := baseDirectory.value / "test-source-links",
+      test in SourceLinksIntegrationTest := ((test in SourceLinksIntegrationTest) dependsOn generateScalaDocumentation.toTask("")).value,
+    )
   lazy val `scaladoc-nonBootstrapped` = project.in(file("scaladoc")).scaladocBasic(NonBootstrapped).settings(
     // Unit tests in scaladoc depends on scaladoc-testcases
     // we do not want to just cross compile them with bootstrapped and unbootstrapped compiler
diff --git a/scaladoc-testcases/src/toplevel.scala b/scaladoc-testcases/src/toplevel.scala
@@ -1,3 +1,3 @@
 def toplevelDef = 123
 
-class ToplevelClass
+class ToplevelClass
diff --git a/scaladoc/src/dotty/tools/scaladoc/renderers/WikiDocRenderer.scala b/scaladoc/src/dotty/tools/scaladoc/renderers/WikiDocRenderer.scala
@@ -46,7 +46,7 @@ class DocRender(signatureRenderer: SignatureRenderer)(using DocContext):
           case 5 => h5(content)
           case 6 => h6(content)
     case Paragraph(text) => p(renderElement(text))
-    case Code(data: String) => code(raw(data)) // TODO add classes
+    case Code(data: String) => pre(code(data)) // TODO add classes
     case HorizontalRule => hr
 
     case UnorderedList(items) => ul(listItems(items))
diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala
@@ -15,12 +15,11 @@ trait NameNormalizer { self: TastyParser =>
   }
 
   private val ignoredKeywords: Set[String] = Set("this")
+  private val reservedCharacters: Set[Char] = Set(' ', '(', '{', '[', ']', '}', ')')
 
   private def escapedName(name: String) =
-    val simpleIdentifierRegex = raw"(?:\w+_[^\[\(\s_]+)|\w+|[^\[\(\s\w_]+".r
     name match
       case n if ignoredKeywords(n) => n
-      case n if keywords(termName(n)) => s"`$n`"
-      case simpleIdentifierRegex() => name
-      case n => s"`$n`"
+      case n if keywords(termName(n)) || n.exists(reservedCharacters.contains) => s"`$n`"
+      case _ => name
 }
diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala
@@ -114,4 +114,3 @@ trait SyntheticsSupport:
     typeForClass(c).asInstanceOf[dotc.core.Types.Type]
       .memberInfo(symbol.asInstanceOf[dotc.core.Symbols.Symbol])
       .asInstanceOf[TypeRepr]
-
diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala
@@ -66,4 +66,4 @@ case class DocFlexmarkRenderer(renderLink: (DocLink, String) => String)
 object DocFlexmarkRenderer:
   def render(node: Node)(renderLink: (DocLink, String) => String) =
     val opts = MarkdownParser.mkMarkdownOptions(Seq(DocFlexmarkRenderer(renderLink)))
-    HtmlRenderer.builder(opts).build().render(node)
+    HtmlRenderer.builder(opts).escapeHtml(true).build().render(node)
diff --git a/scaladoc/src/dotty/tools/scaladoc/util/html.scala b/scaladoc/src/dotty/tools/scaladoc/util/html.scala
@@ -11,7 +11,17 @@ object HTML:
   case class Tag(name: String):
     def apply(tags: TagArg*): AppliedTag = apply()(tags:_*)
     def apply(first: AttrArg, rest: AttrArg*): AppliedTag = apply((first +: rest):_*)()
-    def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag = {
+    def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag =
+      def unpackTags(tags: TagArg*)(using sb: StringBuilder): StringBuilder =
+        tags.foreach {
+          case t: AppliedTag =>
+            sb.append(t)
+          case s: String =>
+            sb.append(s.escapeReservedTokens)
+          case s: Seq[AppliedTag | String] =>
+            unpackTags(s:_*)
+        }
+        sb
       val sb = StringBuilder()
       sb.append(s"<$name")
       attrs.filter(_ != Nil).foreach{
@@ -21,22 +31,9 @@ object HTML:
           sb.append(" ").append(e)
       }
       sb.append(">")
-      tags.foreach{
-        case t: AppliedTag =>
-          sb.append(t)
-        case s: String =>
-          sb.append(s.escapeReservedTokens)
-        case s: Seq[AppliedTag | String] =>
-          s.foreach{
-            case a: AppliedTag =>
-              sb.append(a)
-            case s: String =>
-              sb.append(s.escapeReservedTokens)
-          }
-      }
+      unpackTags(tags:_*)(using sb)
       sb.append(s"</$name>")
       sb
-    }
 
   extension (s: String) private def escapeReservedTokens: String =
     s.replace("&", "&amp;")
@@ -83,7 +80,7 @@ object HTML:
   val ol = Tag("ol")
   val li = Tag("li")
   val code = Tag("code")
-
+  val pre = Tag("pre")
 
   val cls = Attr("class")
   val href = Attr("href")
diff --git a/scaladoc/test-source-links/dotty/tools/scaladoc/source-links/RemoteLinksTest.scala b/scaladoc/test-source-links/dotty/tools/scaladoc/source-links/RemoteLinksTest.scala
@@ -5,33 +5,51 @@ import scala.io.Source
 import scala.jdk.CollectionConverters._
 import scala.util.matching.Regex
 import dotty.tools.scaladoc.test.BuildInfo
-import java.nio.file.Path;
+import java.nio.file.Path
+import java.nio.file.Paths
 import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
 import util.IO
 import org.junit.Assert.assertTrue
+import org.junit.Test
 
-class RemoteLinksTest extends ScaladocTest(""): // empty since it is unused in our case
+class RemoteLinksTest:
 
-  def runTest = afterRendering {
-    val mtsl = membersToSourceLinks
+  @Test
+  def runTest =
+    val mtsl = membersToSourceLinks(using testDocContext())
     val pageToMtsl: Map[String, List[(String, String)]] = mtsl.groupMap(_._2.split("#L").head)(v => (v._1, v._2.split("#L").last))
     pageToMtsl.foreach { case (link, members) =>
       try
-        val doc = Jsoup.connect(link).get
+        val doc = getDocumentFromUrl(link)
         members.foreach { (member, line) =>
           if !member.startsWith("given_") then // TODO: handle synthetic givens, for now we disable them from testing
             val loc = doc.select(s"#LC$line").text
-            assertTrue(s"Expected to find $member at $link at line $line", loc.contains(member))
+            val memberToMatch = member.replace("`", "")
+            assertTrue(s"Expected to find $memberToMatch at $link at line $line", loc.contains(memberToMatch))
         }
       catch
+        case e: java.lang.IllegalArgumentException =>
+          report.error(s"Could not open link for $link - invalid URL")(using testContext)
         case e: org.jsoup.HttpStatusException => e.getStatusCode match
           case 404 => throw AssertionError(s"Page $link does not exists")
-          case n   => report.warning(s"Could not open link for $link, return code $n")
+          case n   => report.warning(s"Could not open link for $link, return code $n")(using testContext)
     }
-  }
+    assertNoErrors(testContext.reportedDiagnostics)
+
+  private def getDocumentFromUrl(link: String): Document =
+    try
+      Jsoup.connect(link).get
+    catch
+      case e: org.jsoup.HttpStatusException => e.getStatusCode match
+          case 429 =>
+            Thread.sleep(10)
+            getDocumentFromUrl(link)
+          case n =>
+            throw e
 
   private def membersToSourceLinks(using DocContext): List[(String, String)] =
-    val output = summon[DocContext].args.output.toPath.resolve("api")
+    val output = Paths.get("scaladoc", "output", "scala3", "api").toAbsolutePath
     val mtsl = List.newBuilder[(String, String)]
     def processFile(path: Path): Unit =
       val document = Jsoup.parse(IO.read(path))
@@ -43,5 +61,3 @@ class RemoteLinksTest extends ScaladocTest(""): // empty since it is unused in o
       }
     IO.foreachFileIn(output, processFile)
     mtsl.result
-
-

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,3 @@`
`1`	`1`	`def toplevelDef = 123`
`2`	`2`
`3`		`-class ToplevelClass`
	`3`	`+class ToplevelClass`
Original file line number	Diff line number	Diff line change
`@@ -15,12 +15,11 @@ trait NameNormalizer { self: TastyParser =>`
`15`	`15`	`}`
`16`	`16`
`17`	`17`	`private val ignoredKeywords: Set[String] = Set("this")`
	`18`	`+ private val reservedCharacters: Set[Char] = Set(' ', '(', '{', '[', ']', '}', ')')`
`18`	`19`
`19`	`20`	`private def escapedName(name: String) =`
`20`		`- val simpleIdentifierRegex = raw"(?:\w+_[^\[\(\s_]+)\|\w+\|[^\[\(\s\w_]+".r`
`21`	`21`	`name match`
`22`	`22`	`case n if ignoredKeywords(n) => n`
`23`		- case n if keywords(termName(n)) => s"`$n`"
`24`		`- case simpleIdentifierRegex() => name`
`25`		- case n => s"`$n`"
	`23`	+ case n if keywords(termName(n)) \|\| n.exists(reservedCharacters.contains) => s"`$n`"
	`24`	`+ case _ => name`
`26`	`25`	`}`