fix: return correct chunks of streams, fixes #229

Author: achingbrain

src/exporter/extract-data-from-block.js

@@ -1,22 +1,23 @@
 'use strict'
 
-module.exports = function extractDataFromBlock (block, streamPosition, begin, end) {
+module.exports = function extractDataFromBlock (block, blockStart, requestedStart, requestedEnd) {
   const blockLength = block.length
+  const blockEnd = blockStart + blockLength
 
-  if (begin >= streamPosition + blockLength) {
-    // If begin is after the start of the block, return an empty block
-    // This can happen when internal nodes contain data
+  if (requestedStart >= blockEnd || requestedEnd < blockStart) {
+    // If we are looking for a byte range that is starts after the start of the block,
+    // return an empty block.  This can happen when internal nodes contain data
     return Buffer.alloc(0)
   }
 
-  if (end - streamPosition < blockLength) {
+  if (requestedEnd >= blockStart && requestedEnd < blockEnd) {
     // If the end byte is in the current block, truncate the block to the end byte
-    block = block.slice(0, end - streamPosition)
+    block = block.slice(0, requestedEnd - blockStart)
   }
 
-  if (begin > streamPosition && begin < (streamPosition + blockLength)) {
+  if (requestedStart >= blockStart && requestedStart < blockEnd) {
     // If the start byte is in the current block, skip to the start byte
-    block = block.slice(begin - streamPosition)
+    block = block.slice(requestedStart - blockStart)
   }
 
   return block

src/exporter/file.js

@@ -77,36 +77,25 @@ function streamBytes (dag, node, fileSize, offset, length) {
 
   const end = offset + length
 
-  function getData ({ node, start }) {
-    try {
-      if (Buffer.isBuffer(node)) {
-        // this is a raw node
-        return extractDataFromBlock(node, start, offset, end)
-      }
-
-      const file = UnixFS.unmarshal(node.data)
-
-      if (!file.data) {
-        if (file.blockSizes.length) {
-          return
-        }
-
-        return Buffer.alloc(0)
-      }
-
-      return extractDataFromBlock(file.data, start, offset, end)
-    } catch (error) {
-      throw new Error(`Failed to unmarshal node - ${error.message}`)
-    }
-  }
+  return pull(
+    traverse.depthFirst({
+      node,
+      start: 0,
+      end: fileSize
+    }, getChildren(dag, offset, end)),
+    pull.map(extractData(offset, end)),
+    pull.filter(Boolean)
+  )
+}
 
+function getChildren (dag, offset, end) {
   // as we step through the children, keep track of where we are in the stream
   // so we can filter out nodes we're not interested in
   let streamPosition = 0
 
-  function visitor ({ node }) {
+  return function visitor ({ node }) {
     if (Buffer.isBuffer(node)) {
-      // this is a raw node
+      // this is a leaf node, can't traverse any further
       return pull.empty()
     }
 
@@ -131,7 +120,8 @@ function streamBytes (dag, node, fileSize, offset, length) {
         const child = {
           link: link,
           start: streamPosition,
-          end: streamPosition + file.blockSizes[index]
+          end: streamPosition + file.blockSizes[index],
+          size: file.blockSizes[index]
         }
 
         streamPosition = child.end
@@ -161,14 +151,46 @@ function streamBytes (dag, node, fileSize, offset, length) {
       })
     )
   }
+}
 
-  return pull(
-    traverse.depthFirst({
-      node,
-      start: 0,
-      end: fileSize
-    }, visitor),
-    pull.map(getData),
-    pull.filter(Boolean)
-  )
+function extractData (requestedStart, requestedEnd) {
+  let streamPosition = -1
+
+  return function getData ({ node, start, end }) {
+    let block
+
+    if (Buffer.isBuffer(node)) {
+      block = node
+    } else {
+      try {
+        const file = UnixFS.unmarshal(node.data)
+
+        if (!file.data) {
+          if (file.blockSizes.length) {
+            return
+          }
+
+          return Buffer.alloc(0)
+        }
+
+        block = file.data
+      } catch (error) {
+        throw new Error(`Failed to unmarshal node - ${error.message}`)
+      }
+    }
+
+    if (block && block.length) {
+      if (streamPosition === -1) {
+        streamPosition = start
+      }
+
+      const output = extractDataFromBlock(block, streamPosition, requestedStart, requestedEnd)
+
+      streamPosition += block.length
+
+      return output
+    }
+
+    return Buffer.alloc(0)
+  }
 }

test/exporter.js

@@ -323,6 +323,33 @@ module.exports = (repo) => {
       )
     })
 
+    it('exports the right chunks of files when offsets are specified', function (done) {
+      this.timeout(30 * 1000)
+      const offset = 3
+      const data = Buffer.alloc(300 * 1024)
+
+      addAndReadTestFile({
+        file: data,
+        offset: 0
+      }, (err, fileWithNoOffset) => {
+        expect(err).to.not.exist()
+
+        addAndReadTestFile({
+          file: data,
+          offset
+        }, (err, fileWithOffset) => {
+          expect(err).to.not.exist()
+
+          expect(fileWithNoOffset.length).to.equal(data.length)
+          expect(fileWithNoOffset.length - fileWithOffset.length).to.equal(offset)
+          expect(fileWithOffset.length).to.equal(data.length - offset)
+          expect(fileWithNoOffset.length).to.equal(fileWithOffset.length + offset)
+
+          done()
+        })
+      })
+    })
+
     it('exports a zero length chunk of a large file', function (done) {
       this.timeout(30 * 1000)