From cf2bc37d8c4582368cc9f8305f8b5e6861a47b1f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 23 Dec 2019 12:02:49 +0000 Subject: [PATCH 1/7] fix: support mtime as timespec --- package.json | 2 +- src/dag-builder/dir.js | 14 ++++------ src/dag-builder/file/index.js | 29 ++++++++------------- src/dir-flat.js | 14 ++++------ src/dir-sharded.js | 19 ++++++-------- src/index.js | 1 - src/tree-builder.js | 7 +++-- test/importer.spec.js | 49 +++++++++++++++++++++++------------ 8 files changed, 66 insertions(+), 69 deletions(-) diff --git a/package.json b/package.json index 5ffbd17..c7c5de8 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "it-all": "^1.0.1", "it-batch": "^1.0.3", "it-first": "^1.0.1", - "it-parallel-batch": "1.0.2", + "it-parallel-batch": "1.0.3", "merge-options": "^2.0.0", "multicodec": "~0.5.1", "multihashing-async": "^0.8.0", diff --git a/src/dag-builder/dir.js b/src/dag-builder/dir.js index 7ba8111..c3f8381 100644 --- a/src/dag-builder/dir.js +++ b/src/dag-builder/dir.js @@ -7,15 +7,11 @@ const { } = require('ipld-dag-pb') const dirBuilder = async (item, ipld, options) => { - const unixfs = new UnixFS('directory') - - if (item.mtime) { - unixfs.mtime = item.mtime - } - - if (item.mode) { - unixfs.mode = item.mode - } + const unixfs = new UnixFS({ + type: 'directory', + mtime: item.mtime, + mode: item.mode + }) const node = new DAGNode(unixfs.marshal(), []) const cid = await persist(node, ipld, options) diff --git a/src/dag-builder/file/index.js b/src/dag-builder/file/index.js index bfb71a6..f9df4fd 100644 --- a/src/dag-builder/file/index.js +++ b/src/dag-builder/file/index.js @@ -33,15 +33,12 @@ async function * importBuffer (file, source, ipld, options) { opts.codec = 'raw' opts.cidVersion = 1 } else { - unixfs = new UnixFS(options.leafType, buffer) - - if (file.mtime) { - unixfs.mtime = file.mtime - } - - if (file.mode) { - unixfs.mode = file.mode - } + unixfs = new UnixFS({ + type: options.leafType, + data: buffer, + mtime: file.mtime, + mode: file.mode + }) node = new DAGNode(unixfs.marshal()) } @@ -96,15 +93,11 @@ const reduce = (file, ipld, options) => { } // create a parent node and add all the leaves - const f = new UnixFS('file') - - if (file.mtime) { - f.mtime = file.mtime - } - - if (file.mode) { - f.mode = file.mode - } + const f = new UnixFS({ + type: 'file', + mtime: file.mtime, + mode: file.mode + }) const links = leaves .filter(leaf => { diff --git a/src/dir-flat.js b/src/dir-flat.js index cd8c10f..266901b 100644 --- a/src/dir-flat.js +++ b/src/dir-flat.js @@ -68,15 +68,11 @@ class DirFlat extends Dir { links.push(new DAGLink(children[i], child.node.length || child.node.size, child.cid)) } - const unixfs = new UnixFS('directory') - - if (this.mtime) { - unixfs.mtime = this.mtime - } - - if (this.mode) { - unixfs.mode = this.mode - } + const unixfs = new UnixFS({ + type: 'directory', + mtime: this.mtime, + mode: this.mode + }) const node = new DAGNode(unixfs.marshal(), links) const cid = await persist(node, ipld, this.options) diff --git a/src/dir-sharded.js b/src/dir-sharded.js index 4aa30b8..26065c7 100644 --- a/src/dir-sharded.js +++ b/src/dir-sharded.js @@ -141,17 +141,14 @@ async function * flush (path, bucket, ipld, shardRoot, options) { // go-ipfs uses little endian, that's why we have to // reverse the bit field before storing it const data = Buffer.from(children.bitField().reverse()) - const dir = new UnixFS('hamt-sharded-directory', data) - dir.fanout = bucket.tableSize() - dir.hashType = options.hamtHashFn.code - - if (shardRoot && shardRoot.mtime) { - dir.mtime = shardRoot.mtime - } - - if (shardRoot && shardRoot.mode) { - dir.mode = shardRoot.mode - } + const dir = new UnixFS({ + type: 'hamt-sharded-directory', + data, + fanout: bucket.tableSize(), + hashType: options.hamtHashFn.code, + mtime: shardRoot && shardRoot.mtime, + mode: shardRoot && shardRoot.mode + }) const node = new DAGNode(dir.marshal(), links) const cid = await persist(node, ipld, options) diff --git a/src/index.js b/src/index.js index fc09074..e1240e9 100644 --- a/src/index.js +++ b/src/index.js @@ -29,7 +29,6 @@ const defaultOptions = { wrapWithDirectory: false, pin: true, recursive: false, - ignore: null, // [] hidden: false, preload: true } diff --git a/src/tree-builder.js b/src/tree-builder.js index cd43170..9f36d42 100644 --- a/src/tree-builder.js +++ b/src/tree-builder.js @@ -64,6 +64,7 @@ async function * treeBuilder (source, ipld, options) { if (!entry) { continue } + tree = await addToTree(entry, tree, options) yield entry @@ -83,13 +84,11 @@ async function * treeBuilder (source, ipld, options) { tree = unwrapped.child } - if (!tree.dir) { + if (!(tree instanceof Dir)) { return } - for await (const entry of tree.flush(tree.path, ipld)) { - yield entry - } + yield * tree.flush(tree.path, ipld) } module.exports = treeBuilder diff --git a/test/importer.spec.js b/test/importer.spec.js index 321db3a..edc8f88 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -29,6 +29,16 @@ function stringifyMh (files) { }) } +function dateToTimespec (date) { + const ms = date.getTime() + const secs = Math.floor(ms / 1000) + + return { + secs, + nsecs: (ms - (secs * 1000)) * 1000 + } +} + const baseFiles = { '200Bytes.txt': { cid: 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8', @@ -681,7 +691,7 @@ strategies.forEach((strategy) => { const options = { rawLeaves: true } - const now = parseInt(Date.now() / 1000) + const now = new Date() for await (const file of importer([{ path: '1.2MiB.txt', @@ -690,14 +700,14 @@ strategies.forEach((strategy) => { }], ipld, options)) { const node = await exporter(file.cid, ipld) - expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) } }) it('supports passing mtime for directories', async () => { this.timeout(60 * 1000) - const now = parseInt(Date.now() / 1000) + const now = new Date() const entries = await all(importer([{ path: '/foo', @@ -705,13 +715,13 @@ strategies.forEach((strategy) => { }], ipld)) const node = await exporter(entries[0].cid, ipld) - expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) }) it('supports passing metadata for wrapping directories', async () => { this.timeout(60 * 1000) - const now = parseInt(Date.now() / 1000) + const now = new Date() const perms = parseInt('0777', 8) const entries = await all(importer([{ @@ -730,14 +740,14 @@ strategies.forEach((strategy) => { expect.fail('no directory found') } - expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) expect(node.unixfs.mode).to.equal(perms) }) it('supports passing metadata for intermediate directories', async () => { this.timeout(60 * 1000) - const now = parseInt(Date.now() / 1000) + const now = new Date() const perms = parseInt('0777', 8) const entries = await all(importer([{ @@ -756,14 +766,14 @@ strategies.forEach((strategy) => { expect.fail('no directory found') } - expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) expect(node.unixfs.mode).to.equal(perms) }) it('supports passing metadata for out of order intermediate directories', async () => { this.timeout(60 * 1000) - const now = parseInt(Date.now() / 1000) + const now = new Date() const perms = parseInt('0777', 8) const entries = await all(importer([{ @@ -787,14 +797,14 @@ strategies.forEach((strategy) => { expect.fail('no directory found') } - expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) expect(node.unixfs.mode).to.equal(perms) }) it('supports passing mtime for hamt-sharded-directories', async () => { this.timeout(60 * 1000) - const now = parseInt(Date.now() / 1000) + const now = new Date() const entries = await all(importer([{ path: '/foo', @@ -818,7 +828,7 @@ strategies.forEach((strategy) => { expect.fail('no hamt-sharded-directory found') } - expect(node.unixfs.mtime).to.equal(now) + expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) }) it('supports passing mode', async () => { @@ -907,12 +917,19 @@ strategies.forEach((strategy) => { }], ipld)) const node1 = await exporter(entries[0].cid, ipld) - expect(node1.unixfs.mode).to.equal(parseInt('0644', 8)) - expect(node1.unixfs.mtime).to.be.undefined() + expect(node1.unixfs).to.have.property('mode', parseInt('0644', 8)) + expect(node1.unixfs).to.have.deep.property('mtime', { + secs: 0, + nsecs: 0 + }) + expect(node1.unixfs.mtime.nsecs).to.equal(0) const node2 = await exporter(entries[1].cid, ipld) - expect(node2.unixfs.mode).to.equal(parseInt('0755', 8)) - expect(node2.unixfs.mtime).to.be.undefined() + expect(node2.unixfs).to.have.property('mode', parseInt('0755', 8)) + expect(node2.unixfs).to.have.deep.property('mtime', { + secs: 0, + nsecs: 0 + }) }) }) }) From 313a7fd4da823de46117d5bf319878df7ffe9248 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 23 Dec 2019 12:07:30 +0000 Subject: [PATCH 2/7] fix: use unixfs pr version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c7c5de8..eeccbec 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "bl": "^4.0.0", "err-code": "^2.0.0", "hamt-sharding": "~0.0.2", - "ipfs-unixfs": "^0.2.0", + "ipfs-unixfs": "ipfs/js-ipfs-unixfs#store-mtime-as-timespec", "ipld-dag-pb": "^0.18.0", "it-all": "^1.0.1", "it-batch": "^1.0.3", From 0e0219e9bfbb7d4c15a8c3ccb6807c004410d01e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 23 Dec 2019 14:58:57 +0000 Subject: [PATCH 3/7] fix: handle unwrapped directories --- src/tree-builder.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/tree-builder.js b/src/tree-builder.js index 9f36d42..feb9f42 100644 --- a/src/tree-builder.js +++ b/src/tree-builder.js @@ -67,7 +67,9 @@ async function * treeBuilder (source, ipld, options) { tree = await addToTree(entry, tree, options) - yield entry + if (!entry.unixfs || !entry.unixfs.isDirectory()) { + yield entry + } } if (!options.wrapWithDirectory) { @@ -85,6 +87,10 @@ async function * treeBuilder (source, ipld, options) { } if (!(tree instanceof Dir)) { + if (tree && tree.unixfs && tree.unixfs.isDirectory()) { + yield tree + } + return } From 0e4be3de41615f576cc7692a2e9314f7aa182828 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 8 Jan 2020 12:17:19 +0000 Subject: [PATCH 4/7] fix: update to support optional mtime --- test/importer.spec.js | 61 ++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/test/importer.spec.js b/test/importer.spec.js index edc8f88..50bef67 100644 --- a/test/importer.spec.js +++ b/test/importer.spec.js @@ -700,7 +700,7 @@ strategies.forEach((strategy) => { }], ipld, options)) { const node = await exporter(file.cid, ipld) - expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) + expect(node).to.have.nested.deep.property('unixfs.mtime', dateToTimespec(now)) } }) @@ -715,14 +715,14 @@ strategies.forEach((strategy) => { }], ipld)) const node = await exporter(entries[0].cid, ipld) - expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) + expect(node).to.have.nested.deep.property('unixfs.mtime', dateToTimespec(now)) }) it('supports passing metadata for wrapping directories', async () => { this.timeout(60 * 1000) const now = new Date() - const perms = parseInt('0777', 8) + const perms = 0o0777 const entries = await all(importer([{ path: '/foo', @@ -740,15 +740,15 @@ strategies.forEach((strategy) => { expect.fail('no directory found') } - expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) - expect(node.unixfs.mode).to.equal(perms) + expect(node).to.have.nested.deep.property('unixfs.mtime', dateToTimespec(now)) + expect(node).to.have.nested.property('unixfs.mode', perms) }) it('supports passing metadata for intermediate directories', async () => { this.timeout(60 * 1000) const now = new Date() - const perms = parseInt('0777', 8) + const perms = 0o0777 const entries = await all(importer([{ path: '/foo/bar', @@ -766,15 +766,15 @@ strategies.forEach((strategy) => { expect.fail('no directory found') } - expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) - expect(node.unixfs.mode).to.equal(perms) + expect(node).to.have.nested.deep.property('unixfs.mtime', dateToTimespec(now)) + expect(node).to.have.nested.property('unixfs.mode', perms) }) it('supports passing metadata for out of order intermediate directories', async () => { this.timeout(60 * 1000) const now = new Date() - const perms = parseInt('0777', 8) + const perms = 0o0777 const entries = await all(importer([{ path: '/foo/bar/qux.txt', @@ -797,8 +797,8 @@ strategies.forEach((strategy) => { expect.fail('no directory found') } - expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) - expect(node.unixfs.mode).to.equal(perms) + expect(node).to.have.nested.deep.property('unixfs.mtime', dateToTimespec(now)) + expect(node).to.have.nested.property('unixfs.mode', perms) }) it('supports passing mtime for hamt-sharded-directories', async () => { @@ -828,7 +828,7 @@ strategies.forEach((strategy) => { expect.fail('no hamt-sharded-directory found') } - expect(node.unixfs.mtime).to.deep.equal(dateToTimespec(now)) + expect(node).to.have.nested.deep.property('unixfs.mtime', dateToTimespec(now)) }) it('supports passing mode', async () => { @@ -837,7 +837,7 @@ strategies.forEach((strategy) => { const options = { rawLeaves: true } - const mode = parseInt('0111', 8) + const mode = 0o0111 for await (const file of importer([{ path: '1.2MiB.txt', @@ -846,14 +846,14 @@ strategies.forEach((strategy) => { }], ipld, options)) { const node = await exporter(file.cid, ipld) - expect(node.unixfs.mode).to.equal(mode) + expect(node).to.have.nested.property('unixfs.mode', mode) } }) it('supports passing mode for directories', async () => { this.timeout(60 * 1000) - const mode = parseInt('0111', 8) + const mode = 0o0111 const entries = await all(importer([{ path: '/foo', @@ -861,14 +861,14 @@ strategies.forEach((strategy) => { }], ipld)) const node = await exporter(entries[0].cid, ipld) - expect(node.unixfs.mode).to.equal(mode) + expect(node).to.have.nested.property('unixfs.mode', mode) }) it('supports passing different modes for different files', async () => { this.timeout(60 * 1000) - const mode1 = parseInt('0111', 8) - const mode2 = parseInt('0222', 8) + const mode1 = 0o0111 + const mode2 = 0o0222 const entries = await all(importer([{ path: '/foo/file1.txt', @@ -881,16 +881,16 @@ strategies.forEach((strategy) => { }], ipld)) const node1 = await exporter(entries[0].cid, ipld) - expect(node1.unixfs.mode).to.equal(mode1) + expect(node1).to.have.nested.property('unixfs.mode', mode1) const node2 = await exporter(entries[1].cid, ipld) - expect(node2.unixfs.mode).to.equal(mode2) + expect(node2).to.have.nested.property('unixfs.mode', mode2) }) it('supports deeply nested files do not inherit custom metadata', async () => { this.timeout(60 * 1000) - const mode = parseInt('0111', 8) + const mode = 0o0111 const entries = await all(importer([{ path: '/foo/file1.txt', @@ -902,13 +902,13 @@ strategies.forEach((strategy) => { }], ipld)) const node1 = await exporter(entries[0].cid, ipld) - expect(node1.unixfs.mode).to.equal(mode) + expect(node1).to.have.nested.property('unixfs.mode', mode) const node2 = await exporter(entries[1].cid, ipld) - expect(node2.unixfs.mode).to.not.equal(mode) + expect(node2).to.have.nested.property('unixfs.mode').that.does.not.equal(mode) }) - it('files and directories get default metadata if not specified', async () => { + it('files and directories get default mode if not specified', async () => { this.timeout(60 * 1000) const entries = await all(importer([{ @@ -917,19 +917,10 @@ strategies.forEach((strategy) => { }], ipld)) const node1 = await exporter(entries[0].cid, ipld) - expect(node1.unixfs).to.have.property('mode', parseInt('0644', 8)) - expect(node1.unixfs).to.have.deep.property('mtime', { - secs: 0, - nsecs: 0 - }) - expect(node1.unixfs.mtime.nsecs).to.equal(0) + expect(node1).to.have.nested.property('unixfs.mode', 0o0644) const node2 = await exporter(entries[1].cid, ipld) - expect(node2.unixfs).to.have.property('mode', parseInt('0755', 8)) - expect(node2.unixfs).to.have.deep.property('mtime', { - secs: 0, - nsecs: 0 - }) + expect(node2).to.have.nested.property('unixfs.mode', 0o0755) }) }) }) From f05609f70fd6fcda7b52d83d215b74e742e6fa75 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 8 Jan 2020 14:04:25 +0000 Subject: [PATCH 5/7] chore: update deps --- package.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index eeccbec..a93f1be 100644 --- a/package.json +++ b/package.json @@ -48,24 +48,24 @@ "ipld-in-memory": "^3.0.0", "it-buffer-stream": "^1.0.0", "it-last": "^1.0.0", - "multihashes": "~0.4.14", - "nyc": "^14.0.0", - "sinon": "^7.1.0" + "multihashes": "^0.4.14", + "nyc": "^15.0.0", + "sinon": "^8.0.4" }, "dependencies": { "bl": "^4.0.0", "err-code": "^2.0.0", - "hamt-sharding": "~0.0.2", - "ipfs-unixfs": "ipfs/js-ipfs-unixfs#store-mtime-as-timespec", + "hamt-sharding": "^0.0.2", + "ipfs-unixfs": "^0.3.0", "ipld-dag-pb": "^0.18.0", "it-all": "^1.0.1", "it-batch": "^1.0.3", "it-first": "^1.0.1", - "it-parallel-batch": "1.0.3", + "it-parallel-batch": "^1.0.3", "merge-options": "^2.0.0", - "multicodec": "~0.5.1", + "multicodec": "^1.0.0", "multihashing-async": "^0.8.0", - "rabin-wasm": "~0.0.8" + "rabin-wasm": "^0.0.8" }, "contributors": [ "Alan Shaw ", From d3b319e00655a3e6a3ed671b5b116a1daa11bed8 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 8 Jan 2020 14:07:32 +0000 Subject: [PATCH 6/7] fix: linting --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index a93f1be..90e305e 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "dependencies": { "bl": "^4.0.0", "err-code": "^2.0.0", - "hamt-sharding": "^0.0.2", + "hamt-sharding": "~0.0.2", "ipfs-unixfs": "^0.3.0", "ipld-dag-pb": "^0.18.0", "it-all": "^1.0.1", @@ -65,7 +65,7 @@ "merge-options": "^2.0.0", "multicodec": "^1.0.0", "multihashing-async": "^0.8.0", - "rabin-wasm": "^0.0.8" + "rabin-wasm": "~0.0.8" }, "contributors": [ "Alan Shaw ", From 76bc4734f6e69cee8a832fb39007e4f3b2e7f6fd Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 8 Jan 2020 15:10:20 +0000 Subject: [PATCH 7/7] chore: update deps --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 90e305e..ee1a956 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "deep-extend": "~0.6.0", "detect-node": "^2.0.4", "dirty-chai": "^2.0.1", - "ipfs-unixfs-exporter": "^0.39.0", + "ipfs-unixfs-exporter": "^0.40.0", "ipld": "^0.25.0", "ipld-in-memory": "^3.0.0", "it-buffer-stream": "^1.0.0", @@ -55,7 +55,7 @@ "dependencies": { "bl": "^4.0.0", "err-code": "^2.0.0", - "hamt-sharding": "~0.0.2", + "hamt-sharding": "^1.0.0", "ipfs-unixfs": "^0.3.0", "ipld-dag-pb": "^0.18.0", "it-all": "^1.0.1",