Serialize linkable entities incrementally for performance

Author: d-ronnqvist

Sources/SwiftDocC/Infrastructure/ConvertActionConverter.swift

@@ -77,7 +77,6 @@ package enum ConvertActionConverter {
 
         // Arrays to gather additional metadata if `emitDigest` is `true`.
         var indexingRecords = [IndexingRecord]()
-        var linkSummaries = [LinkDestinationSummary]()
         var assets = [RenderReferenceType : [RenderReference]]()
         var coverageInfo = [CoverageDataEntry]()
         let coverageFilterClosure = documentationCoverageOptions.generateFilterClosure()
@@ -142,16 +141,18 @@ package enum ConvertActionConverter {
                         let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: true)
                         let nodeIndexingRecords = try renderNode.indexingRecords(onPage: identifier)
 
+                        for linkSummary in nodeLinkSummaries {
+                            try outputConsumer.consumeIncremental(linkableElementSummary: linkSummary)
+                        }
                         resultsGroup.async(queue: resultsSyncQueue) {
                             assets.merge(renderNode.assetReferences, uniquingKeysWith: +)
-                            linkSummaries.append(contentsOf: nodeLinkSummaries)
                             indexingRecords.append(contentsOf: nodeIndexingRecords)
                         }
                     } else if FeatureFlags.current.isLinkHierarchySerializationEnabled {
                         let nodeLinkSummaries = entity.externallyLinkableElementSummaries(context: context, renderNode: renderNode, includeTaskGroups: false)
 
-                        resultsGroup.async(queue: resultsSyncQueue) {
-                            linkSummaries.append(contentsOf: nodeLinkSummaries)
+                        for linkSummary in nodeLinkSummaries {
+                            try outputConsumer.consumeIncremental(linkableElementSummary: linkSummary)
                         }
                     }
                 } catch {
@@ -171,7 +172,7 @@ package enum ConvertActionConverter {
         if emitDigest {
             signposter.withIntervalSignpost("Emit digest", id: signposter.makeSignpostID()) {
                 do {
-                    try outputConsumer.consume(linkableElementSummaries: linkSummaries)
+                    try outputConsumer.finishedConsumingLinkElementSummaries()
                     try outputConsumer.consume(indexingRecords: indexingRecords)
                     try outputConsumer.consume(assets: assets)
                 } catch {
@@ -187,7 +188,7 @@ package enum ConvertActionConverter {
                     try outputConsumer.consume(linkResolutionInformation: serializableLinkInformation)
 
                     if !emitDigest {
-                        try outputConsumer.consume(linkableElementSummaries: linkSummaries)
+                        try outputConsumer.finishedConsumingLinkElementSummaries()
                     }
                 } catch {
                     recordProblem(from: error, in: &conversionProblems, withIdentifier: "link-resolver")

Sources/SwiftDocC/Infrastructure/ConvertOutputConsumer.swift

@@ -26,6 +26,12 @@ public protocol ConvertOutputConsumer {
     func consume(assetsInBundle bundle: DocumentationBundle) throws
 
     /// Consumes the linkable element summaries produced during a conversion.
+    /// > Warning: This method might be called concurrently.
+    func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws
+    /// Consumes the linkable element summaries produced during a conversion.
+    func finishedConsumingLinkElementSummaries() throws
+    
+    @available(*, deprecated, renamed: "consume(linkableElementSummary:)", message: "Use 'consume(linkableElementSummary:)' instead. This deprecated API will be removed after 6.2 is released")
     func consume(linkableElementSummaries: [LinkDestinationSummary]) throws
 
     /// Consumes the indexing records produced during a conversion.
@@ -58,3 +64,9 @@ public extension ConvertOutputConsumer {
     func consume(buildMetadata: BuildMetadata) throws {}
     func consume(linkResolutionInformation: SerializableLinkResolutionInformation) throws {}
 }
+
+// Default implementations to avoid a source breaking change from introducing new protocol requirements
+public extension ConvertOutputConsumer {
+    func consume(linkableElementSummary: LinkDestinationSummary) throws {}
+    func finishedConsumingLinkElementSummaries() throws {}
+}

Sources/SwiftDocCUtilities/Action/Actions/Convert/ConvertFileWritingConsumer.swift

@@ -147,6 +147,33 @@ struct ConvertFileWritingConsumer: ConvertOutputConsumer {
         }
     }
 
+    private var linkableElementsData = Synchronized(Data())
+    
+    /// Consumes one  linkable element summary produced during a conversion.
+    func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws {
+        let data = try encode(linkableElementSummary)
+        linkableElementsData.sync {
+            if !$0.isEmpty {
+                $0.append(Data(",".utf8))
+            }
+            $0.append(data)
+        }
+    }
+    
+    /// Finishes consuming the linkable element summaries produced during a conversion.
+    func finishedConsumingLinkElementSummaries() throws {
+        let linkableElementsURL = targetFolder.appendingPathComponent(Self.linkableEntitiesFileName, isDirectory: false)
+        let data = linkableElementsData.sync { accumulatedData in
+            var data = Data()
+            swap(&data, &accumulatedData)
+            data.insert(UTF8.CodeUnit(ascii: "["), at: 0)
+            data.append(UTF8.CodeUnit(ascii: "]"))
+            
+            return data
+        }
+        try fileManager.createFile(at: linkableElementsURL, contents: data)
+    }
+    
     func consume(linkableElementSummaries summaries: [LinkDestinationSummary]) throws {
         let linkableElementsURL = targetFolder.appendingPathComponent(Self.linkableEntitiesFileName, isDirectory: false)
         let data = try encode(summaries)

Tests/SwiftDocCTests/Converter/DocumentationConverterTests.swift

@@ -22,6 +22,8 @@ class DocumentationConverterTests: XCTestCase {
         func consume(problems: [Problem]) throws { }
         func consume(assetsInBundle bundle: DocumentationBundle) throws {}
         func consume(linkableElementSummaries: [LinkDestinationSummary]) throws {}
+        func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws {}
+        func finishedConsumingLinkElementSummaries() throws {}
         func consume(indexingRecords: [IndexingRecord]) throws {}
         func consume(assets: [RenderReferenceType: [RenderReference]]) throws {}
         func consume(benchmarks: Benchmark) throws {}

Tests/SwiftDocCTests/TestRenderNodeOutputConsumer.swift

@@ -24,6 +24,8 @@ class TestRenderNodeOutputConsumer: ConvertOutputConsumer {
     func consume(problems: [Problem]) throws { }
     func consume(assetsInBundle bundle: DocumentationBundle) throws { }
     func consume(linkableElementSummaries: [LinkDestinationSummary]) throws { }
+    func consumeIncremental(linkableElementSummary: LinkDestinationSummary) throws { }
+    func finishedConsumingLinkElementSummaries() throws { }
     func consume(indexingRecords: [IndexingRecord]) throws { }
     func consume(assets: [RenderReferenceType: [RenderReference]]) throws { }
     func consume(benchmarks: Benchmark) throws { }