diff --git a/src/dag-builder/file/trickle.js b/src/dag-builder/file/trickle.js index 4995606..ca7db2d 100644 --- a/src/dag-builder/file/trickle.js +++ b/src/dag-builder/file/trickle.js @@ -7,100 +7,144 @@ module.exports = async function * trickleReduceToRoot (source, reduce, options) } async function trickleStream (source, reduce, options) { - let root = { - children: [] - } - let node = root + let root + let iteration = 0 let maxDepth = 1 - let currentDepth = 1 - let layerSize = 0 + let subTree = root = new Root(options.layerRepeat) for await (const layer of batch(source, options.maxChildrenPerNode)) { - node.data = layer - - let parent = node.parent || root - const nextNode = { - children: [] - } + if (subTree.isFull()) { + if (subTree !== root) { + root.addChild(await subTree.reduce(reduce)) + } - if (currentDepth < maxDepth) { - // the current layer can't have more children - // but we can descend a layer - node.children.push(nextNode) - nextNode.parent = node - node = nextNode - currentDepth++ - } else if (parent.children.length < options.layerRepeat) { - // the current layer can have more children - parent.children.push(nextNode) - nextNode.parent = parent - node = nextNode - } else if (currentDepth === maxDepth) { - // hit the bottom of the current iteration, can we find a sibling? - parent = findNext(root, 0, maxDepth, options) - - if (parent) { - nextNode.parent = parent - parent.children.push(nextNode) - node = nextNode - } else { - if (layerSize === 0) { - maxDepth++ - } - - layerSize++ - - if (layerSize === options.layerRepeat) { - layerSize = 0 - } - - nextNode.parent = root - root.children.push(nextNode) - node = nextNode - - currentDepth = 1 + if (iteration && iteration % options.layerRepeat === 0) { + maxDepth++ } + + subTree = new SubTree(maxDepth, options.layerRepeat, iteration) + + iteration++ } + + subTree.append(layer) } - // reduce to root - return walk(root, reduce) + if (subTree && subTree !== root) { + root.addChild(await subTree.reduce(reduce)) + } + + return root.reduce(reduce) } -const walk = async (node, reduce) => { - let children = [] +class SubTree { + constructor (maxDepth, layerRepeat, iteration) { + this.maxDepth = maxDepth + this.layerRepeat = layerRepeat + this.currentDepth = 1 + this.iteration = iteration + + this.root = this.node = this.parent = { + children: [], + depth: this.currentDepth, + maxDepth, + maxChildren: (this.maxDepth - this.currentDepth) * this.layerRepeat + } + } + + isFull () { + if (!this.root.data) { + return false + } + + if (this.currentDepth < this.maxDepth && this.node.maxChildren) { + // can descend + this._addNextNodeToParent(this.node) + + return false + } + + // try to find new node from node.parent + const distantRelative = this._findParent(this.node, this.currentDepth) + + if (distantRelative) { + this._addNextNodeToParent(distantRelative) + + return false + } - if (node.children.length) { - children = await Promise.all( - node.children - .filter(child => child.data) - .map(child => walk(child, reduce)) - ) + return true } - return reduce(node.data.concat(children)) -} + _addNextNodeToParent (parent) { + this.parent = parent + + // find site for new node + const nextNode = { + children: [], + depth: parent.depth + 1, + parent, + maxDepth: this.maxDepth, + maxChildren: Math.floor(parent.children.length / this.layerRepeat) * this.layerRepeat + } + + parent.children.push(nextNode) -const findNext = (node, depth, maxDepth, options) => { - if (depth === maxDepth) { - return + this.currentDepth = nextNode.depth + this.node = nextNode } - let nodeMatches = false + append (layer) { + this.node.data = layer + } + + reduce (reduce) { + return this._reduce(this.root, reduce) + } + + async _reduce (node, reduce) { + let children = [] + + if (node.children.length) { + children = await Promise.all( + node.children + .filter(child => child.data) + .map(child => this._reduce(child, reduce)) + ) + } - if (node.children.length < options.layerRepeat) { - nodeMatches = true + return reduce(node.data.concat(children)) } - if (node.children.length) { - const childMatches = findNext(node.children[node.children.length - 1], depth + 1, maxDepth, options) + _findParent (node, depth) { + const parent = node.parent + + if (!parent || parent.depth === 0) { + return + } - if (childMatches) { - return childMatches + if (parent.children.length === parent.maxChildren || !parent.maxChildren) { + // this layer is full, may be able to traverse to a different branch + return this._findParent(parent, depth) } + + return parent + } +} + +class Root extends SubTree { + constructor (layerRepeat) { + super(0, layerRepeat) + + this.root.depth = 0 + this.currentDepth = 1 + } + + addChild (child) { + this.root.children.push(child) } - if (nodeMatches) { - return node + reduce (reduce) { + return reduce(this.root.data.concat(this.root.children)) } } diff --git a/test/builder-trickle-dag.spec.js b/test/builder-trickle-dag.spec.js index 3744acd..8e625b4 100644 --- a/test/builder-trickle-dag.spec.js +++ b/test/builder-trickle-dag.spec.js @@ -68,7 +68,7 @@ describe('builder: trickle', () => { }]) }) - it('reduces 9 values correclty', async () => { + it('reduces 9 values correctly', async () => { const result = await all(builder(createValues(9), reduce, options)) expect(result).to.deep.equal([{ @@ -181,397 +181,394 @@ describe('builder: trickle', () => { }]) }) - it('reduces 68 values correclty', async () => { + it('reduces 68 values correctly', async () => { const result = await all(builder(createValues(68), reduce, options)) - expect(result).to.deep.equal([{ - children: [ - 0, - 1, - 2, - - { - children: [ - 3, - 4, - 5 - ] - }, - { - children: [ - 6, - 7, - 8 - ] - }, - - { - children: [ - 9, - 10, - 11, - { - children: [ - 12, - 13, - 14 - ] - }, - { - children: [ - 15, - 16, - 17 - ] - } - ] - }, - { - children: [ - 18, - 19, - 20, - { - children: [ - 21, - 22, - 23 - ] - }, - { - children: [ - 24, - 25, - 26 - ] - } - ] - }, - - { - children: [ - 27, - 28, - 29, - { - children: [ - 30, - 31, - 32, - { - children: [ - 33, - 34, - 35 - ] - }, - { - children: [ - 36, - 37, - 38 - ] - } - ] - }, - { - children: [ - 39, - 40, - 41, - { - children: [ - 42, - 43, - 44 - ] - }, - { - children: [ - 45, - 46, - 47 - ] - } - ] - } - ] - }, - { - children: [ - 48, - 49, - 50, - { - children: [ - 51, - 52, - 53, - { - children: [ - 54, - 55, - 56 - ] - }, - { - children: [ - 57, - 58, - 59 - ] - } - ] - }, - { - children: [ - 60, - 61, - 62, - { - children: [ - 63, - 64, - 65 - ] - }, - { - children: [ - 66, - 67 - ] - } - ] - } - ] - } - ] - }]) + expect(result).to.deep.equal([ + { + children: [ + 0, + 1, + 2, + { + children: [ + 3, + 4, + 5 + ] + }, + { + children: [ + 6, + 7, + 8 + ] + }, + { + children: [ + 9, + 10, + 11, + { + children: [ + 12, + 13, + 14 + ] + }, + { + children: [ + 15, + 16, + 17 + ] + } + ] + }, + { + children: [ + 18, + 19, + 20, + { + children: [ + 21, + 22, + 23 + ] + }, + { + children: [ + 24, + 25, + 26 + ] + } + ] + }, + { + children: [ + 27, + 28, + 29, + { + children: [ + 30, + 31, + 32 + ] + }, + { + children: [ + 33, + 34, + 35 + ] + }, + { + children: [ + 36, + 37, + 38, + { + children: [ + 39, + 40, + 41 + ] + }, + { + children: [ + 42, + 43, + 44 + ] + } + ] + }, + { + children: [ + 45, + 46, + 47, + { + children: [ + 48, + 49, + 50 + ] + }, + { + children: [ + 51, + 52, + 53 + ] + } + ] + } + ] + }, + { + children: [ + 54, + 55, + 56, + { + children: [ + 57, + 58, + 59 + ] + }, + { + children: [ + 60, + 61, + 62 + ] + }, + { + children: [ + 63, + 64, + 65, + { + children: [ + 66, + 67 + ] + } + ] + } + ] + } + ] + } + ]) }) - it('reduces 93 values correclty', async () => { + it('reduces 93 values correctly', async () => { const result = await all(builder(createValues(93), reduce, options)) - expect(result).to.deep.equal([{ - children: [ - 0, - 1, - 2, - - { - children: [ - 3, - 4, - 5 - ] - }, - { - children: [ - 6, - 7, - 8 - ] - }, - - { - children: [ - 9, - 10, - 11, - { - children: [ - 12, - 13, - 14 - ] - }, - { - children: [ - 15, - 16, - 17 - ] - } - ] - }, - { - children: [ - 18, - 19, - 20, - { - children: [ - 21, - 22, - 23 - ] - }, - { - children: [ - 24, - 25, - 26 - ] - } - ] - }, - - { - children: [ - 27, - 28, - 29, - { - children: [ - 30, - 31, - 32, - { - children: [ - 33, - 34, - 35 - ] - }, - { - children: [ - 36, - 37, - 38 - ] - } - ] - }, - { - children: [ - 39, - 40, - 41, - { - children: [ - 42, - 43, - 44 - ] - }, - { - children: [ - 45, - 46, - 47 - ] - } - ] - } - ] - }, - { - children: [ - 48, - 49, - 50, - { - children: [ - 51, - 52, - 53, - { - children: [ - 54, - 55, - 56 - ] - }, - { - children: [ - 57, - 58, - 59 - ] - } - ] - }, - { - children: [ - 60, - 61, - 62, - { - children: [ - 63, - 64, - 65 - ] - }, - { - children: [ - 66, - 67, - 68 - ] - } - ] - } - ] - }, - - { - children: [ - 69, - 70, - 71, - { - children: [ - 72, - 73, - 74, - { - children: [ - 75, - 76, - 77, - { - children: [ - 78, - 79, - 80 - ] - }, - { - children: [ - 81, - 82, - 83 - ] - } - ] - }, - { - children: [ - 84, - 85, - 86, - { - children: [ - 87, - 88, - 89 - ] - }, - { - children: [ - 90, - 91, - 92 - ] - } - ] - } - ] - } - ] - } - ] - }]) + expect(result).to.deep.equal([ + { + children: [ + 0, + 1, + 2, + { + children: [ + 3, + 4, + 5 + ] + }, + { + children: [ + 6, + 7, + 8 + ] + }, + { + children: [ + 9, + 10, + 11, + { + children: [ + 12, + 13, + 14 + ] + }, + { + children: [ + 15, + 16, + 17 + ] + } + ] + }, + { + children: [ + 18, + 19, + 20, + { + children: [ + 21, + 22, + 23 + ] + }, + { + children: [ + 24, + 25, + 26 + ] + } + ] + }, + { + children: [ + 27, + 28, + 29, + { + children: [ + 30, + 31, + 32 + ] + }, + { + children: [ + 33, + 34, + 35 + ] + }, + { + children: [ + 36, + 37, + 38, + { + children: [ + 39, + 40, + 41 + ] + }, + { + children: [ + 42, + 43, + 44 + ] + } + ] + }, + { + children: [ + 45, + 46, + 47, + { + children: [ + 48, + 49, + 50 + ] + }, + { + children: [ + 51, + 52, + 53 + ] + } + ] + } + ] + }, + { + children: [ + 54, + 55, + 56, + { + children: [ + 57, + 58, + 59 + ] + }, + { + children: [ + 60, + 61, + 62 + ] + }, + { + children: [ + 63, + 64, + 65, + { + children: [ + 66, + 67, + 68 + ] + }, + { + children: [ + 69, + 70, + 71 + ] + } + ] + }, + { + children: [ + 72, + 73, + 74, + { + children: [ + 75, + 76, + 77 + ] + }, + { + children: [ + 78, + 79, + 80 + ] + } + ] + } + ] + }, + { + children: [ + 81, + 82, + 83, + { + children: [ + 84, + 85, + 86 + ] + }, + { + children: [ + 87, + 88, + 89 + ] + }, + { + children: [ + 90, + 91, + 92 + ] + } + ] + } + ] + } + ]) }) })