diff --git a/src/chunker/index.js b/src/chunker/index.js index ec2c494..71ca417 100644 --- a/src/chunker/index.js +++ b/src/chunker/index.js @@ -8,7 +8,12 @@ const chunkers = { } module.exports = (type, source, options) => { - const chunker = chunkers[type] + let chunker + if (typeof type === 'function') { + chunker = type + } else { + chunker = chunkers[type] + } if (!chunker) { throw errCode(new Error(`Unknkown chunker named ${type}`), 'ERR_UNKNOWN_CHUNKER') diff --git a/src/dag-builder/file/index.js b/src/dag-builder/file/index.js index f9df4fd..d3f3514 100644 --- a/src/dag-builder/file/index.js +++ b/src/dag-builder/file/index.js @@ -27,23 +27,28 @@ async function * importBuffer (file, source, ipld, options) { ...options } - if (options.rawLeaves) { - node = buffer - - opts.codec = 'raw' - opts.cidVersion = 1 + let cid + if (options.fromParts) { + node = { length: buffer.size } + cid = buffer.cid } else { - unixfs = new UnixFS({ - type: options.leafType, - data: buffer, - mtime: file.mtime, - mode: file.mode - }) - - node = new DAGNode(unixfs.marshal()) - } + if (options.rawLeaves) { + node = buffer - const cid = await persist(node, ipld, opts) + opts.codec = 'raw' + opts.cidVersion = 1 + } else { + unixfs = new UnixFS({ + type: options.leafType, + data: buffer, + mtime: file.mtime, + mode: file.mode + }) + + node = new DAGNode(unixfs.marshal()) + } + cid = await persist(node, ipld, opts) + } return { cid: cid, diff --git a/src/dag-builder/index.js b/src/dag-builder/index.js index bc2f4b2..4e60a93 100644 --- a/src/dag-builder/index.js +++ b/src/dag-builder/index.js @@ -30,7 +30,14 @@ async function * dagBuilder (source, ipld, options) { } } - const chunker = createChunker(options.chunker, validateChunks(source), options) + if (options.fromParts) { + options.rawLeaves = true + options.chunker = source => source + options.reduceSingleLeafToSelf = false + } else { + source = validateChunks(source) + } + const chunker = createChunker(options.chunker, source, options) // item is a file yield () => fileBuilder(entry, chunker, ipld, options) diff --git a/test/chunker-custom.spec.js b/test/chunker-custom.spec.js new file mode 100644 index 0000000..350ce93 --- /dev/null +++ b/test/chunker-custom.spec.js @@ -0,0 +1,36 @@ +/* eslint-env mocha */ +'use strict' + +const importer = require('../src') + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const IPLD = require('ipld') +const inMemory = require('ipld-in-memory') + +// eslint bug https://github.com/eslint/eslint/issues/12459 +// eslint-disable-next-line require-await +const iter = async function * () { + yield Buffer.from('one') + yield Buffer.from('two') +} + +describe('custom chunker', function () { + it('keeps custom chunking', async () => { + const chunker = source => source + const content = iter() + const inmem = await inMemory(IPLD) + const sizes = [11, 11, 116] + const ipld = { + put: (node, ...args) => { + expect(node.toJSON().size).to.equal(sizes.shift()) + return inmem.put(node, ...args) + } + } + for await (const part of importer([{ path: 'test', content }], ipld, { chunker })) { + expect(part.size).to.equal(116) + } + expect(sizes).to.be.empty() + }) +}) diff --git a/test/fromParts.spec.js b/test/fromParts.spec.js new file mode 100644 index 0000000..98073bb --- /dev/null +++ b/test/fromParts.spec.js @@ -0,0 +1,43 @@ +/* eslint-env mocha */ +'use strict' + +const importer = require('../src') + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const IPLD = require('ipld') +const inMemory = require('ipld-in-memory') +const CID = require('cids') + +const fromPartsTest = (iter, size) => async () => { + const content = iter() + const inmem = await inMemory(IPLD) + const sizes = [size] + const ipld = { + put: (node, ...args) => { + expect(node.toJSON().size).to.equal(sizes.shift()) + return inmem.put(node, ...args) + } + } + for await (const part of importer([{ path: 'test', content }], ipld, { fromParts: true })) { + expect(part.size).to.equal(size) + } + expect(sizes).to.be.empty() +} + +describe('custom chunker', function () { + // eslint bug https://github.com/eslint/eslint/issues/12459 + // eslint-disable-next-line require-await + const multi = async function * () { + yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') } + yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') } + } + it('works with multiple parts', fromPartsTest(multi, 276312)) + + // eslint-disable-next-line require-await + const single = async function * () { + yield { size: 138102, cid: new CID('mAVUSIO7K3sMLqZPsJ/6SYMa5HiHBaj81xjniNYRUXbpKl/Ac') } + } + it('works with single part', fromPartsTest(single, 138160)) +})