diff --git a/.github/workflows/scaladoc.yaml b/.github/workflows/scaladoc.yaml index 4a1fbf0f8ec1..a58f4c23c2a8 100644 --- a/.github/workflows/scaladoc.yaml +++ b/.github/workflows/scaladoc.yaml @@ -70,3 +70,25 @@ jobs: echo uplading docs to https://scala3doc.virtuslab.com/$DOC_DEST az storage container create --name $DOC_DEST --account-name scala3docstorage --public-access container az storage blob upload-batch -s scaladoc/output -d $DOC_DEST --account-name scala3docstorage + + stdlib-sourcelinks-test: + runs-on: ubuntu-latest + if: "( github.event_name == 'pull_request' + && !contains(github.event.pull_request.body, '[skip ci]') + && !contains(github.event.pull_request.body, '[skip docs]') + ) + || contains(github.event.ref, 'scaladoc') + || contains(github.event.ref, 'scala3doc') + || contains(github.event.ref, 'master')" + + steps: + - name: Git Checkout + uses: actions/checkout@v2 + + - name: Set up JDK 8 + uses: actions/setup-java@v1 + with: + java-version: 8 + + - name: Test sourcelinks to stdlib + run: ./project/scripts/sbt scaladoc/sourceLinksIntegrationTest:test diff --git a/project/Build.scala b/project/Build.scala index 4bc7ae23cc29..61b4d5076486 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -1182,6 +1182,7 @@ object Build { // Note: the two tasks below should be one, but a bug in Tasty prevents that val generateScalaDocumentation = inputKey[Unit]("Generate documentation for dotty lib") val generateTestcasesDocumentation = taskKey[Unit]("Generate documentation for testcases, usefull for debugging tests") + lazy val `scaladoc-testcases` = project.in(file("scaladoc-testcases")). dependsOn(`scala3-compiler-bootstrapped`). settings(commonBootstrappedSettings) @@ -1217,10 +1218,18 @@ object Build { Def.task((s"$distLocation/bin/scaladoc" +: cmd).!) } + val SourceLinksIntegrationTest = config("sourceLinksIntegrationTest") extend Test + lazy val scaladoc = project.in(file("scaladoc")). + configs(SourceLinksIntegrationTest). settings(commonBootstrappedSettings). dependsOn(`scala3-compiler-bootstrapped`). dependsOn(`scala3-tasty-inspector`). + settings(inConfig(SourceLinksIntegrationTest)(Defaults.testSettings)). + settings( + SourceLinksIntegrationTest / scalaSource := baseDirectory.value / "test-source-links", + SourceLinksIntegrationTest / test:= ((SourceLinksIntegrationTest / test) dependsOn generateScalaDocumentation.toTask("")).value, + ). settings( Compile / resourceGenerators += Def.task { val jsDestinationFile = (Compile / resourceManaged).value / "dotty_res" / "scripts" / "searchbar.js" diff --git a/scaladoc-testcases/src/toplevel.scala b/scaladoc-testcases/src/toplevel.scala index 764e9cbaf9cf..b82b5980263c 100644 --- a/scaladoc-testcases/src/toplevel.scala +++ b/scaladoc-testcases/src/toplevel.scala @@ -1,3 +1,3 @@ def toplevelDef = 123 -class ToplevelClass \ No newline at end of file +class ToplevelClass diff --git a/scaladoc/src/dotty/tools/scaladoc/SourceLinks.scala b/scaladoc/src/dotty/tools/scaladoc/SourceLinks.scala index 64000c00d872..307d5b05c100 100644 --- a/scaladoc/src/dotty/tools/scaladoc/SourceLinks.scala +++ b/scaladoc/src/dotty/tools/scaladoc/SourceLinks.scala @@ -140,4 +140,3 @@ object SourceLinks: ) SourceLinks(sourceLinks) } - diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala index 932d437353ab..6ba4e3f3ff36 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/NameNormalizer.scala @@ -21,10 +21,9 @@ object NameNormalizer { private val ignoredKeywords: Set[String] = Set("this") private def escapedName(name: String) = - val simpleIdentifierRegex = raw"(?:\w+_[^\[\(\s_]+)|\w+|[^\[\(\s\w_]+".r + val complexIdentifierRegex = """([([{}]) ]|[^A-Za-z0-9$]_)""".r name match case n if ignoredKeywords(n) => n - case n if keywords(termName(n)) => s"`$n`" - case simpleIdentifierRegex() => name - case n => s"`$n`" + case n if keywords(termName(n)) || complexIdentifierRegex.findFirstIn(n).isDefined => s"`$n`" + case _ => name } diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala index fe8b94817e3e..efc7909dbe96 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/SyntheticSupport.scala @@ -119,4 +119,3 @@ object SyntheticsSupport: typeForClass(c).asInstanceOf[dotc.core.Types.Type] .memberInfo(symbol.asInstanceOf[dotc.core.Symbols.Symbol]) .asInstanceOf[TypeRepr] - diff --git a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala index 9cd0e0653898..ad5e4ac20e2d 100644 --- a/scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala +++ b/scaladoc/src/dotty/tools/scaladoc/tasty/comments/markdown/DocFlexmarkExtension.scala @@ -66,4 +66,4 @@ case class DocFlexmarkRenderer(renderLink: (DocLink, String) => String) object DocFlexmarkRenderer: def render(node: Node)(renderLink: (DocLink, String) => String) = val opts = MarkdownParser.mkMarkdownOptions(Seq(DocFlexmarkRenderer(renderLink))) - HtmlRenderer.builder(opts).build().render(node) \ No newline at end of file + HtmlRenderer.builder(opts).escapeHtml(true).build().render(node) diff --git a/scaladoc/src/dotty/tools/scaladoc/util/html.scala b/scaladoc/src/dotty/tools/scaladoc/util/html.scala index 3c07222341b6..8033f39bb52b 100644 --- a/scaladoc/src/dotty/tools/scaladoc/util/html.scala +++ b/scaladoc/src/dotty/tools/scaladoc/util/html.scala @@ -11,7 +11,17 @@ object HTML: case class Tag(name: String): def apply(tags: TagArg*): AppliedTag = apply()(tags:_*) def apply(first: AttrArg, rest: AttrArg*): AppliedTag = apply((first +: rest):_*)() - def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag = { + def apply(attrs: AttrArg*)(tags: TagArg*): AppliedTag = + def unpackTags(tags: TagArg*)(using sb: StringBuilder): StringBuilder = + tags.foreach { + case t: AppliedTag => + sb.append(t) + case s: String => + sb.append(s.escapeReservedTokens) + case s: Seq[AppliedTag | String] => + unpackTags(s:_*) + } + sb val sb = StringBuilder() sb.append(s"<$name") attrs.filter(_ != Nil).foreach{ @@ -21,22 +31,9 @@ object HTML: sb.append(" ").append(e) } sb.append(">") - tags.foreach{ - case t: AppliedTag => - sb.append(t) - case s: String => - sb.append(s.escapeReservedTokens) - case s: Seq[AppliedTag | String] => - s.foreach{ - case a: AppliedTag => - sb.append(a) - case s: String => - sb.append(s.escapeReservedTokens) - } - } + unpackTags(tags:_*)(using sb) sb.append(s"") sb - } extension (s: String) private def escapeReservedTokens: String = s.replace("&", "&") diff --git a/scaladoc/test-source-links/dotty/tools/scaladoc/source-links/RemoteLinksTest.scala b/scaladoc/test-source-links/dotty/tools/scaladoc/source-links/RemoteLinksTest.scala new file mode 100644 index 000000000000..9fc8bc7eb17a --- /dev/null +++ b/scaladoc/test-source-links/dotty/tools/scaladoc/source-links/RemoteLinksTest.scala @@ -0,0 +1,84 @@ +package dotty.tools.scaladoc +package sourcelinks + +import scala.util.Random +import scala.io.Source +import scala.jdk.CollectionConverters._ +import scala.util.matching.Regex +import dotty.tools.scaladoc.test.BuildInfo +import java.nio.file.Path +import java.nio.file.Paths +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import util.IO +import org.junit.Assert.assertTrue +import org.junit.Test + +class RemoteLinksTest: + + class TimeoutException extends Exception + + val randomGenerator = new Random(125L) + val mtslAll = membersToSourceLinks(using testDocContext()) + + @Test + def scala213XSourceLink = + assertTrue(mtslAll.find((k, _) => k == "AbstractMap").isDefined) // source link to Scala2.13.X stdlib class + + @Test + def scala3SourceLink = + assertTrue(mtslAll.find((k, _) => k == "PolyFunction").isDefined) // source link to Scala3 stdlib class + + @Test + def runTest = + assertTrue(mtslAll.nonEmpty) + val mtsl = randomGenerator.shuffle(mtslAll).take(20) // take 20 random entries + val pageToMtsl: Map[String, List[(String, String)]] = mtsl.groupMap(_._2.split("#L").head)(v => (v._1, v._2.split("#L").last)) + pageToMtsl.foreach { case (link, members) => + try + val doc = getDocumentFromUrl(link) + members.foreach { (member, line) => + if !member.startsWith("given_") then // TODO: handle synthetic givens, for now we disable them from testing + val loc = doc.select(s"#LC$line").text + val memberToMatch = member.replace("`", "") + assertTrue(s"Expected to find $memberToMatch at $link at line $line", loc.contains(memberToMatch)) + } + catch + case e: java.lang.IllegalArgumentException => + report.error(s"Could not open link for $link - invalid URL")(using testContext) + case e: TimeoutException => + report.error(s"Tried to open link $link 16 times but with no avail")(using testContext) + case e: org.jsoup.HttpStatusException => e.getStatusCode match + case 404 => throw AssertionError(s"Page $link does not exists") + case n => report.warning(s"Could not open link for $link, return code $n")(using testContext) + } + assertNoErrors(testContext.reportedDiagnostics) + + private def getDocumentFromUrl(link: String, retries: Int = 16): Document = + try + if retries == 0 then throw TimeoutException() + Jsoup.connect(link).get + catch + case e: org.jsoup.HttpStatusException => e.getStatusCode match + case 429 => + Thread.sleep(10) + getDocumentFromUrl(link, retries - 1) + case n => + throw e + + private def membersToSourceLinks(using DocContext): List[(String, String)] = + val output = Paths.get("scaladoc", "output", "scala3", "api").toAbsolutePath + val mtsl = List.newBuilder[(String, String)] + def processFile(path: Path): Unit = + val document = Jsoup.parse(IO.read(path)) + if document.select("span.kind").first.text == "package" then + document.select(".documentableElement").forEach { dElem => + if dElem.select("span.kind").first.text != "package" then + dElem.select("dt").forEach { elem => + val content = elem.text + if content == "Source" then + mtsl += dElem.select(".documentableName").first.text -> elem.nextSibling.childNode(0).attr("href") + } + } + IO.foreachFileIn(output, processFile) + mtsl.result diff --git a/scaladoc/test/dotty/tools/scaladoc/ScaladocTest.scala b/scaladoc/test/dotty/tools/scaladoc/ScaladocTest.scala index 8d7574adf13d..1ca6dcd04367 100644 --- a/scaladoc/test/dotty/tools/scaladoc/ScaladocTest.scala +++ b/scaladoc/test/dotty/tools/scaladoc/ScaladocTest.scala @@ -25,7 +25,8 @@ abstract class ScaladocTest(val name: String): name = "test", tastyFiles = tastyFiles(name), output = getTempDir().getRoot, - projectVersion = Some("1.0") + projectVersion = Some("1.0"), + sourceLinks = List("github://lampepfl/dotty/master") ) @Test diff --git a/scaladoc/test/dotty/tools/scaladoc/SourceLinksTests.scala b/scaladoc/test/dotty/tools/scaladoc/source-links/SourceLinksTest.scala similarity index 99% rename from scaladoc/test/dotty/tools/scaladoc/SourceLinksTests.scala rename to scaladoc/test/dotty/tools/scaladoc/source-links/SourceLinksTest.scala index f5721fe90c03..00f5ba538d1b 100644 --- a/scaladoc/test/dotty/tools/scaladoc/SourceLinksTests.scala +++ b/scaladoc/test/dotty/tools/scaladoc/source-links/SourceLinksTest.scala @@ -1,4 +1,5 @@ package dotty.tools.scaladoc +package sourcelinks import java.nio.file._ import org.junit.Assert._ @@ -155,4 +156,4 @@ class SourceLinksTest: ("src/lib/core.scala", 33, edit) -> "https://gitlab.com/lampepfl/dotty/-/edit/develop/lib/core.scala#L33", ("src/generated.scala", 33, edit) -> "https://gitlab.com/lampepfl/dotty/-/edit/develop/generated.scala#L33", ("src/generated/template.scala", 1, edit) -> "/template.scala#1" - ) \ No newline at end of file + ) diff --git a/scaladoc/test/dotty/tools/scaladoc/testUtils.scala b/scaladoc/test/dotty/tools/scaladoc/testUtils.scala index cfab383c9ba8..b5c3d2abd2bc 100644 --- a/scaladoc/test/dotty/tools/scaladoc/testUtils.scala +++ b/scaladoc/test/dotty/tools/scaladoc/testUtils.scala @@ -53,7 +53,7 @@ class TestReporter extends ConsoleReporter: def testArgs(files: Seq[File] = Nil, dest: File = new File("notUsed")) = Scaladoc.Args( name = "Test Project Name", output = dest, - tastyFiles = files, + tastyFiles = files ) def testContext = (new ContextBase).initialCtx.fresh.setReporter(new TestReporter)