diff --git a/package.json b/package.json index 9d22c322..a53f215a 100644 --- a/package.json +++ b/package.json @@ -45,13 +45,13 @@ "dirty-chai": "^2.0.1", "ipfs": "~0.26.0", "ipfs-block-service": "~0.13.0", - "ipfs-repo": "~0.18.2", + "ipfs-repo": "~0.18.3", "ncp": "^2.0.0", "pre-commit": "^1.2.2", "pull-generate": "^2.2.0", "pull-zip": "^2.0.1", "rimraf": "^2.6.2", - "sinon": "^4.1.1", + "sinon": "^4.1.2", "split": "^1.0.1" }, "dependencies": { diff --git a/test/browser.js b/test/browser.js index c2bc0c2d..6cf3280f 100644 --- a/test/browser.js +++ b/test/browser.js @@ -34,23 +34,35 @@ describe('IPFS data importing tests on the Browser', function () { ], done) }) - require('./test-builder')(repo) - require('./test-flat-builder') - require('./test-balanced-builder') - require('./test-trickle-builder') - require('./test-fixed-size-chunker') - - // relies on data in the repo - // require('./test-exporter')(repo) - - require('./test-consumable-buffer') - require('./test-consumable-hash') - require('./test-hamt') - require('./test-importer')(repo) - require('./test-importer-flush')(repo) - require('./test-import-export')(repo) - require('./test-hash-parity-with-go-ipfs')(repo) - require('./test-nested-dir-import-export')(repo) - require('./test-dirbuilder-sharding')(repo) - require('./test-builder-only-hash')(repo) + // HAMT + require('./hamt') + require('./hamt-consumable-buffer') + require('./hamt-consumable-hash') + + // Chunkers + require('./chunker-fixed-size') + + // Graph Builders + require('./builder')(repo) + require('./builder-flat') + require('./builder-balanced') + require('./builder-trickle-dag') + require('./builder-only-hash')(repo) + // TODO: make these tests not require data on the repo + // require('./builder-dir-sharding')(repo) + + // Importer + require('./importer')(repo) + require('./importer-flush')(repo) + + // Exporter + // TODO: make these tests not require data on the repo + // require('./exporter')(repo) + // require('./exporter-subtree')(repo) + + // Other + require('./import-export')(repo) + require('./import-export-nested-dir')(repo) + require('./hash-parity-with-go-ipfs')(repo) + // require('./with-dag-api') }) diff --git a/test/test-balanced-builder.js b/test/builder-balanced.js similarity index 98% rename from test/test-balanced-builder.js rename to test/builder-balanced.js index e88a568c..fea033a9 100644 --- a/test/test-balanced-builder.js +++ b/test/builder-balanced.js @@ -20,7 +20,7 @@ const options = { maxChildrenPerNode: 3 } -describe('balanced builder', () => { +describe('builder: balanced', () => { it('reduces one value into itself', (callback) => { pull( pull.values([1]), diff --git a/test/test-dirbuilder-sharding.js b/test/builder-dir-sharding.js similarity index 99% rename from test/test-dirbuilder-sharding.js rename to test/builder-dir-sharding.js index a9131a9c..42f029bd 100644 --- a/test/test-dirbuilder-sharding.js +++ b/test/builder-dir-sharding.js @@ -17,7 +17,7 @@ const setImmediate = require('async/setImmediate') const leftPad = require('left-pad') module.exports = (repo) => { - describe('dirbuilder sharding', function () { + describe('builder: directory sharding', function () { this.timeout(20 * 1000) let ipldResolver diff --git a/test/test-flat-builder.js b/test/builder-flat.js similarity index 88% rename from test/test-flat-builder.js rename to test/builder-flat.js index be30a2a8..45ade05d 100644 --- a/test/test-flat-builder.js +++ b/test/builder-flat.js @@ -16,7 +16,7 @@ function reduce (leaves, callback) { } } -describe('flat builder', () => { +describe('builder: flat', () => { it('reduces one value into itself', (callback) => { pull( pull.values([1]), @@ -35,9 +35,7 @@ describe('flat builder', () => { builder(reduce), pull.collect((err, result) => { expect(err).to.not.exist() - expect(result).to.be.eql([{ - children: [1, 2] - }]) + expect(result).to.eql([{ children: [1, 2] }]) callback() }) ) diff --git a/test/test-builder-only-hash.js b/test/builder-only-hash.js similarity index 97% rename from test/test-builder-only-hash.js rename to test/builder-only-hash.js index eb48de8e..4d613834 100644 --- a/test/test-builder-only-hash.js +++ b/test/builder-only-hash.js @@ -12,7 +12,7 @@ const createBuilder = require('../src/builder') const FixedSizeChunker = require('../src/chunker/fixed-size') module.exports = (repo) => { - describe('builder', () => { + describe('builder: onlyHash', () => { let ipldResolver before(() => { diff --git a/test/test-trickle-builder.js b/test/builder-trickle-dag.js similarity index 99% rename from test/test-trickle-builder.js rename to test/builder-trickle-dag.js index 483dbc18..d79e9ca5 100644 --- a/test/test-trickle-builder.js +++ b/test/builder-trickle-dag.js @@ -21,7 +21,7 @@ const options = { layerRepeat: 2 } -describe('trickle builder', () => { +describe('builder: trickle', () => { it('reduces one value into itself', callback => { pull( pull.values([1]), diff --git a/test/test-builder.js b/test/builder.js similarity index 100% rename from test/test-builder.js rename to test/builder.js diff --git a/test/test-fixed-size-chunker.js b/test/chunker-fixed-size.js similarity index 98% rename from test/test-fixed-size-chunker.js rename to test/chunker-fixed-size.js index 0fb8bbd7..75a305a7 100644 --- a/test/test-fixed-size-chunker.js +++ b/test/chunker-fixed-size.js @@ -52,6 +52,7 @@ describe('chunker: fixed size', () => { it('256 KiB chunks', (done) => { const KiB256 = 262144 + pull( pull.values(rawFile), chunker(KiB256), @@ -88,7 +89,7 @@ describe('chunker: fixed size', () => { } }) - expect(counter).to.be.eql(1) + expect(counter).to.equal(1) done() }) ) diff --git a/test/test-export-subtree.js b/test/exporter-subtree.js similarity index 100% rename from test/test-export-subtree.js rename to test/exporter-subtree.js diff --git a/test/test-exporter.js b/test/exporter.js similarity index 100% rename from test/test-exporter.js rename to test/exporter.js diff --git a/test/test-consumable-buffer.js b/test/hamt-consumable-buffer.js similarity index 98% rename from test/test-consumable-buffer.js rename to test/hamt-consumable-buffer.js index d2eb1d60..3fc47c2f 100644 --- a/test/test-consumable-buffer.js +++ b/test/hamt-consumable-buffer.js @@ -5,7 +5,7 @@ const expect = require('chai').expect const ConsumableBuffer = require('../src/hamt/consumable-buffer') -describe('consumable buffer', () => { +describe('HAMT: consumable buffer', () => { let buf it('can create an empty one', () => { diff --git a/test/test-consumable-hash.js b/test/hamt-consumable-hash.js similarity index 97% rename from test/test-consumable-hash.js rename to test/hamt-consumable-hash.js index 8eed3cb8..5f76aaf1 100644 --- a/test/test-consumable-hash.js +++ b/test/hamt-consumable-hash.js @@ -9,7 +9,7 @@ const whilst = require('async/whilst') const ConsumableHash = require('../src/hamt/consumable-hash') -describe('consumable hash', () => { +describe('HAMT: consumable hash', () => { let hash, h const maxIter = 100 const values = [] diff --git a/test/test-hamt.js b/test/hamt.js similarity index 100% rename from test/test-hamt.js rename to test/hamt.js diff --git a/test/test-hash-parity-with-go-ipfs.js b/test/hash-parity-with-go-ipfs.js similarity index 95% rename from test/test-hash-parity-with-go-ipfs.js rename to test/hash-parity-with-go-ipfs.js index 48e22b4a..e439fcc2 100644 --- a/test/test-hash-parity-with-go-ipfs.js +++ b/test/hash-parity-with-go-ipfs.js @@ -30,7 +30,7 @@ module.exports = (repo) => { strategy: strategy } - describe(strategy + ' importer', () => { + describe('go-ipfs interop using importer:' + strategy, () => { let ipldResolver before(() => { diff --git a/test/test-nested-dir-import-export.js b/test/import-export-nested-dir.js similarity index 98% rename from test/test-nested-dir-import-export.js rename to test/import-export-nested-dir.js index cdd6fd0c..da448abf 100644 --- a/test/test-nested-dir-import-export.js +++ b/test/import-export-nested-dir.js @@ -13,7 +13,7 @@ const map = require('async/map') const unixFSEngine = require('./../') module.exports = (repo) => { - describe('import adn export big nested dir', () => { + describe('import and export: directory', () => { const rootHash = 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK' let ipldResolver diff --git a/test/import-export.js b/test/import-export.js new file mode 100644 index 00000000..e09b87a0 --- /dev/null +++ b/test/import-export.js @@ -0,0 +1,69 @@ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect +const BlockService = require('ipfs-block-service') +const IPLDResolver = require('ipld-resolver') +const pull = require('pull-stream') +const loadFixture = require('aegir/fixtures') +const bigFile = loadFixture(__dirname, 'fixtures/1.2MiB.txt') + +const unixFSEngine = require('./../') +const exporter = unixFSEngine.exporter + +const strategies = [ + 'flat', + 'balanced', + 'trickle' +] + +function fileEql (f1, fileData, callback) { + pull( + f1.content, + pull.concat((err, data) => { + expect(err).to.not.exist() + // TODO: eql is super slow at comparing large buffers + // expect(data).to.eql(fileData) + callback() + }) + ) +} + +module.exports = (repo) => { + describe('import and export', () => { + strategies.forEach((strategy) => { + const importerOptions = { strategy: strategy } + + describe('using builder: ' + strategy, () => { + let ipldResolver + + before(() => { + const bs = new BlockService(repo) + ipldResolver = new IPLDResolver(bs) + }) + + it('import and export', (done) => { + const path = strategy + '-big.dat' + + pull( + pull.values([{ path: path, content: pull.values(bigFile) }]), + unixFSEngine.importer(ipldResolver, importerOptions), + pull.map((file) => { + expect(file.path).to.eql(path) + + return exporter(file.multihash, ipldResolver) + }), + pull.flatten(), + pull.collect((err, files) => { + expect(err).to.not.exist() + expect(files[0].size).to.eql(bigFile.length) + fileEql(files[0], bigFile, done) + }) + ) + }) + }) + }) + }) +} diff --git a/test/test-importer-flush.js b/test/importer-flush.js similarity index 99% rename from test/test-importer-flush.js rename to test/importer-flush.js index 418e30ad..46902bfb 100644 --- a/test/test-importer-flush.js +++ b/test/importer-flush.js @@ -12,7 +12,7 @@ const pull = require('pull-stream') const pushable = require('pull-pushable') module.exports = (repo) => { - describe('importer flush', () => { + describe('importer: flush', () => { let ipldResolver before(() => { diff --git a/test/test-importer.js b/test/importer.js similarity index 99% rename from test/test-importer.js rename to test/importer.js index eae309a8..e1edfe62 100644 --- a/test/test-importer.js +++ b/test/importer.js @@ -160,7 +160,7 @@ module.exports = (repo) => { const expected = extend({}, defaultResults, strategies[strategy]) - describe(strategy + ' importer', function () { + describe('importer: ' + strategy, function () { this.timeout(20 * 1000) let ipldResolver diff --git a/test/node.js b/test/node.js index 4790b05b..8bc82510 100644 --- a/test/node.js +++ b/test/node.js @@ -4,13 +4,14 @@ const ncp = require('ncp').ncp const rimraf = require('rimraf') const path = require('path') +const os = require('os') const IPFSRepo = require('ipfs-repo') const mkdirp = require('mkdirp') const series = require('async/series') describe('IPFS UnixFS Engine', () => { const repoExample = path.join(process.cwd(), '/test/test-repo') - const repoTests = path.join(process.cwd(), '/test/repo-tests' + Date.now()) + const repoTests = path.join(os.tmpdir(), '/unixfs-tests-' + Date.now()) const repo = new IPFSRepo(repoTests) @@ -36,22 +37,33 @@ describe('IPFS UnixFS Engine', () => { ], done) }) - require('./test-builder')(repo) - require('./test-flat-builder') - require('./test-balanced-builder') - require('./test-trickle-builder') - require('./test-fixed-size-chunker') - require('./test-consumable-buffer') - require('./test-consumable-hash') - require('./test-hamt') - require('./test-exporter')(repo) - require('./test-export-subtree')(repo) - require('./test-importer')(repo) - require('./test-importer-flush')(repo) - require('./test-import-export')(repo) - require('./test-hash-parity-with-go-ipfs')(repo) - require('./test-nested-dir-import-export')(repo) - require('./test-dirbuilder-sharding')(repo) - require('./test-dag-api') - require('./test-builder-only-hash')(repo) + // HAMT + require('./hamt') + require('./hamt-consumable-buffer') + require('./hamt-consumable-hash') + + // Chunkers + require('./chunker-fixed-size') + + // Graph Builders + require('./builder')(repo) + require('./builder-flat') + require('./builder-balanced') + require('./builder-trickle-dag') + require('./builder-only-hash')(repo) + require('./builder-dir-sharding')(repo) + + // Importer + require('./importer')(repo) + require('./importer-flush')(repo) + + // Exporter + require('./exporter')(repo) + require('./exporter-subtree')(repo) + + // Other + require('./import-export')(repo) + require('./import-export-nested-dir')(repo) + require('./hash-parity-with-go-ipfs')(repo) + require('./with-dag-api') }) diff --git a/test/test-import-export.js b/test/test-import-export.js deleted file mode 100644 index 107f887b..00000000 --- a/test/test-import-export.js +++ /dev/null @@ -1,103 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const chai = require('chai') -chai.use(require('dirty-chai')) -const expect = chai.expect -const BlockService = require('ipfs-block-service') -const IPLDResolver = require('ipld-resolver') -const pull = require('pull-stream') - -const randomByteStream = require('./helpers/finite-pseudorandom-byte-stream') -const unixFSEngine = require('./../') -const exporter = unixFSEngine.exporter - -const strategies = [ - 'flat', - 'balanced', - 'trickle' -] - -module.exports = (repo) => { - let bigFile - strategies.forEach((strategy) => { - const importerOptions = { - strategy: strategy - } - - describe('import export using ' + strategy + ' builder strategy', () => { - let ipldResolver - - before(() => { - const bs = new BlockService(repo) - ipldResolver = new IPLDResolver(bs) - }) - - before((done) => { - if (bigFile) { - return done() - } - - pull( - randomByteStream(50000000, 8372), - pull.collect((err, buffers) => { - if (err) { - done(err) - } else { - bigFile = buffers - done() - } - }) - ) - }) - - it('import and export', (done) => { - const path = strategy + '-big.dat' - pull( - pull.values([{ - path: path, - content: pull.values(bigFile) - }]), - unixFSEngine.importer(ipldResolver, importerOptions), - pull.map((file) => { - expect(file.path).to.be.eql(path) - - return exporter(file.multihash, ipldResolver) - }), - pull.flatten(), - pull.collect((err, files) => { - expect(err).to.not.exist() - expect(files[0].size).to.be.eql(bigFile.reduce(reduceLength, 0)) - fileEql(files[0], Buffer.concat(bigFile), done) - }) - ) - }) - }) - }) -} - -function fileEql (f1, f2, done) { - pull( - f1.content, - pull.collect((err, data) => { - if (err) { - return done(err) - } - - try { - if (f2) { - expect(Buffer.concat(data)).to.be.eql(f2) - } else { - expect(data).to.exist() - } - } catch (err) { - return done(err) - } - done() - }) - ) -} - -function reduceLength (acc, chunk) { - return acc + chunk.length -} diff --git a/test/test-repo/version b/test/test-repo/version index 7ed6ff82..1e8b3149 100644 --- a/test/test-repo/version +++ b/test/test-repo/version @@ -1 +1 @@ -5 +6 diff --git a/test/test-dag-api.js b/test/with-dag-api.js similarity index 99% rename from test/test-dag-api.js rename to test/with-dag-api.js index a04c9f90..d7ad29d6 100644 --- a/test/test-dag-api.js +++ b/test/with-dag-api.js @@ -158,7 +158,7 @@ describe('with dag-api', () => { const expected = extend({}, defaultResults, strategies[strategy]) - describe(strategy + ' importer', function () { + describe('importer: ' + strategy, function () { this.timeout(20 * 1000) let node