From 8fa7cb7f25d0fdd220b95fed78949aef3f204812 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 14 Aug 2019 16:57:53 +0100 Subject: [PATCH 1/3] fix: enable preload on MFS commands that accept IPFS paths fixes https://github.com/ipfs/js-ipfs/issues/2335 License: MIT Signed-off-by: Alan Shaw --- src/core/components/files-mfs.js | 60 ++++++++++++++++++++++---------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/src/core/components/files-mfs.js b/src/core/components/files-mfs.js index 3807e8bf16..826d4851e4 100644 --- a/src/core/components/files-mfs.js +++ b/src/core/components/files-mfs.js @@ -10,13 +10,15 @@ const callbackify = require('callbackify') const PassThrough = require('stream').PassThrough const pull = require('pull-stream/pull') const map = require('pull-stream/throughs/map') +const isIpfs = require('is-ipfs') +const { cidToString } = require('../../utils/cid') const mapLsFile = (options = {}) => { const long = options.long || options.l return (file) => { return { - hash: long ? file.cid.toBaseEncodedString(options.cidBase) : '', + hash: long ? cidToString(file.cid, { base: options.cidBase }) : '', name: file.name, type: long ? file.type : 0, size: long ? file.size || 0 : 0 @@ -32,15 +34,37 @@ module.exports = self => { repoOwner: self._options.repoOwner }) + const withPreload = fn => (...args) => { + const cids = args.reduce((cids, p) => { + if (isIpfs.ipfsPath(p)) { + cids.push(p.split('/')[2]) + } else if (isIpfs.cid(p)) { + cids.push(p) + } else if (isIpfs.cidPath(p)) { + cids.push(p.split('/')[0]) + } + return cids + }, []) + + if (cids.length) { + const options = args[args.length - 1] + if (options.preload !== false) { + cids.forEach(cid => self._preload(cid)) + } + } + + return fn(...args) + } + return { - cp: callbackify.variadic(methods.cp), + cp: callbackify.variadic(withPreload(methods.cp)), flush: callbackify.variadic(methods.flush), - ls: callbackify.variadic(async (path, options = {}) => { + ls: callbackify.variadic(withPreload(async (path, options = {}) => { const files = await all(methods.ls(path, options)) return files.map(mapLsFile(options)) - }), - lsReadableStream: (path, options = {}) => { + })), + lsReadableStream: withPreload((path, options = {}) => { const stream = toReadableStream.obj(methods.ls(path, options)) const through = new PassThrough({ objectMode: true @@ -60,33 +84,33 @@ module.exports = self => { }) return through - }, - lsPullStream: (path, options = {}) => { + }), + lsPullStream: withPreload((path, options = {}) => { return pull( toPullStream.source(methods.ls(path, options)), map(mapLsFile(options)) ) - }, + }), mkdir: callbackify.variadic(methods.mkdir), - mv: callbackify.variadic(methods.mv), - read: callbackify(async (path, options = {}) => { + mv: callbackify.variadic(withPreload(methods.mv)), + read: callbackify(withPreload(async (path, options = {}) => { return Buffer.concat(await all(methods.read(path, options))) - }), - readPullStream: (path, options = {}) => { + })), + readPullStream: withPreload((path, options = {}) => { return toPullStream.source(methods.read(path, options)) - }, - readReadableStream: (path, options = {}) => { + }), + readReadableStream: withPreload((path, options = {}) => { return toReadableStream(methods.read(path, options)) - }, + }), rm: callbackify.variadic(methods.rm), - stat: callbackify(async (path, options = {}) => { + stat: callbackify(withPreload(async (path, options = {}) => { const stats = await methods.stat(path, options) - stats.hash = stats.cid.toBaseEncodedString(options && options.cidBase) + stats.hash = cidToString(stats.cid, { base: options.cidBase }) delete stats.cid return stats - }), + })), write: callbackify.variadic(async (path, content, options = {}) => { if (isPullStream.isSource(content)) { content = pullStreamToAsyncIterator(content) From 0487765be74b717445d4119b59e6aa978f9f6f13 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 21 Aug 2019 12:00:26 +0100 Subject: [PATCH 2/3] test: add tests for preloading on MFS API License: MIT Signed-off-by: Alan Shaw --- src/core/components/files-mfs.js | 6 +- test/core/preload.spec.js | 138 +++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 4 deletions(-) diff --git a/src/core/components/files-mfs.js b/src/core/components/files-mfs.js index 826d4851e4..2f59e1a1aa 100644 --- a/src/core/components/files-mfs.js +++ b/src/core/components/files-mfs.js @@ -40,8 +40,6 @@ module.exports = self => { cids.push(p.split('/')[2]) } else if (isIpfs.cid(p)) { cids.push(p) - } else if (isIpfs.cidPath(p)) { - cids.push(p.split('/')[0]) } return cids }, []) @@ -93,7 +91,7 @@ module.exports = self => { }), mkdir: callbackify.variadic(methods.mkdir), mv: callbackify.variadic(withPreload(methods.mv)), - read: callbackify(withPreload(async (path, options = {}) => { + read: callbackify.variadic(withPreload(async (path, options = {}) => { return Buffer.concat(await all(methods.read(path, options))) })), readPullStream: withPreload((path, options = {}) => { @@ -103,7 +101,7 @@ module.exports = self => { return toReadableStream(methods.read(path, options)) }), rm: callbackify.variadic(methods.rm), - stat: callbackify(withPreload(async (path, options = {}) => { + stat: callbackify.variadic(withPreload(async (path, options = {}) => { const stats = await methods.stat(path, options) stats.hash = cidToString(stats.cid, { base: options.cidBase }) diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index aea0e11862..edb658b567 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -9,6 +9,8 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) +const pull = require('pull-stream') +const CID = require('cids') const MockPreloadNode = require('../utils/mock-preload-node') const IPFS = require('../../src') @@ -326,6 +328,142 @@ describe('preload', () => { }) }) }) + + it('should preload content retrieved with files.ls', done => { + let dirCid + + waterfall([ + cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb), + (res, cb) => { + dirCid = res[res.length - 1].hash + MockPreloadNode.waitForCids(dirCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => ipfs.files.ls(`/ipfs/${dirCid}`, err => cb(err)), + cb => MockPreloadNode.waitForCids(dirCid, cb) + ], done) + }) + + it('should preload content retrieved with files.ls by CID', done => { + let dirCid + + waterfall([ + cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb), + (res, cb) => { + dirCid = res[res.length - 1].hash + MockPreloadNode.waitForCids(dirCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => ipfs.files.ls(new CID(dirCid), err => cb(err)), + cb => MockPreloadNode.waitForCids(dirCid, cb) + ], done) + }) + + it('should preload content retrieved with files.lsReadableStream', done => { + let dirCid + + waterfall([ + cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb), + (res, cb) => { + dirCid = res[res.length - 1].hash + MockPreloadNode.waitForCids(dirCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => { + ipfs.files.lsReadableStream(`/ipfs/${dirCid}`) + .on('data', () => {}) + .on('error', cb) + .on('end', cb) + }, + cb => MockPreloadNode.waitForCids(dirCid, cb) + ], done) + }) + + it('should preload content retrieved with files.lsPullStream', done => { + let dirCid + + waterfall([ + cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb), + (res, cb) => { + dirCid = res[res.length - 1].hash + MockPreloadNode.waitForCids(dirCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => pull( + ipfs.files.lsPullStream(`/ipfs/${dirCid}`), + pull.onEnd(cb) + ), + cb => MockPreloadNode.waitForCids(dirCid, cb) + ], done) + }) + + it('should preload content retrieved with files.read', done => { + let fileCid + + waterfall([ + cb => ipfs.add(Buffer.from(hat()), cb), + (res, cb) => { + fileCid = res[0].hash + MockPreloadNode.waitForCids(fileCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => ipfs.files.read(`/ipfs/${fileCid}`, err => cb(err)), + cb => MockPreloadNode.waitForCids(fileCid, cb) + ], done) + }) + + it('should preload content retrieved with files.readReadableStream', done => { + let fileCid + + waterfall([ + cb => ipfs.add(Buffer.from(hat()), cb), + (res, cb) => { + fileCid = res[0].hash + MockPreloadNode.waitForCids(fileCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => { + ipfs.files.readReadableStream(`/ipfs/${fileCid}`) + .on('data', () => {}) + .on('error', cb) + .on('end', cb) + }, + cb => MockPreloadNode.waitForCids(fileCid, cb) + ], done) + }) + + it('should preload content retrieved with files.readPullStream', done => { + let fileCid + + waterfall([ + cb => ipfs.add(Buffer.from(hat()), cb), + (res, cb) => { + fileCid = res[0].hash + MockPreloadNode.waitForCids(fileCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => pull( + ipfs.files.readPullStream(`/ipfs/${fileCid}`), + pull.onEnd(cb) + ), + cb => MockPreloadNode.waitForCids(fileCid, cb) + ], done) + }) + + it('should preload content retrieved with files.stat', done => { + let fileCid + + waterfall([ + cb => ipfs.add(Buffer.from(hat()), cb), + (res, cb) => { + fileCid = res[0].hash + MockPreloadNode.waitForCids(fileCid, cb) + }, + cb => MockPreloadNode.clearPreloadCids(cb), + cb => ipfs.files.stat(`/ipfs/${fileCid}`, err => cb(err)), + cb => MockPreloadNode.waitForCids(fileCid, cb) + ], done) + }) }) describe('preload disabled', function () { From 6f65de457be134e7a834b66bfc75ee7976db11bf Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 21 Aug 2019 18:16:11 +0100 Subject: [PATCH 3/3] fix: allow preload IPFS/IPNS paths License: MIT Signed-off-by: Alan Shaw --- src/core/components/files-mfs.js | 13 +++---------- src/core/preload.js | 12 ++++++------ test/core/preload.spec.js | 14 +++++++------- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/core/components/files-mfs.js b/src/core/components/files-mfs.js index 2f59e1a1aa..94a5ea8594 100644 --- a/src/core/components/files-mfs.js +++ b/src/core/components/files-mfs.js @@ -35,19 +35,12 @@ module.exports = self => { }) const withPreload = fn => (...args) => { - const cids = args.reduce((cids, p) => { - if (isIpfs.ipfsPath(p)) { - cids.push(p.split('/')[2]) - } else if (isIpfs.cid(p)) { - cids.push(p) - } - return cids - }, []) + const paths = args.filter(arg => isIpfs.ipfsPath(arg) || isIpfs.cid(arg)) - if (cids.length) { + if (paths.length) { const options = args[args.length - 1] if (options.preload !== false) { - cids.forEach(cid => self._preload(cid)) + paths.forEach(path => self._preload(path)) } } diff --git a/src/core/preload.js b/src/core/preload.js index 983b0445d1..ac16037701 100644 --- a/src/core/preload.js +++ b/src/core/preload.js @@ -34,12 +34,12 @@ module.exports = self => { let requests = [] const apiUris = options.addresses.map(apiAddrToUri) - const api = (cid, callback) => { + const api = (path, callback) => { callback = callback || noop - if (typeof cid !== 'string') { + if (typeof path !== 'string') { try { - cid = new CID(cid).toBaseEncodedString() + path = new CID(path).toBaseEncodedString() } catch (err) { return setImmediate(() => callback(err)) } @@ -50,14 +50,14 @@ module.exports = self => { const now = Date.now() retry({ times: fallbackApiUris.length }, (cb) => { - if (stopped) return cb(new Error(`preload aborted for ${cid}`)) + if (stopped) return cb(new Error(`preload aborted for ${path}`)) // Remove failed request from a previous attempt requests = requests.filter(r => r !== request) const apiUri = fallbackApiUris.shift() - request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb) + request = preload(`${apiUri}/api/v0/refs?r=true&arg=${encodeURIComponent(path)}`, cb) requests = requests.concat(request) }, (err) => { requests = requests.filter(r => r !== request) @@ -66,7 +66,7 @@ module.exports = self => { return callback(err) } - log(`preloaded ${cid} in ${Date.now() - now}ms`) + log(`preloaded ${path} in ${Date.now() - now}ms`) callback() }) } diff --git a/test/core/preload.spec.js b/test/core/preload.spec.js index edb658b567..e40c0a740f 100644 --- a/test/core/preload.spec.js +++ b/test/core/preload.spec.js @@ -340,7 +340,7 @@ describe('preload', () => { }, cb => MockPreloadNode.clearPreloadCids(cb), cb => ipfs.files.ls(`/ipfs/${dirCid}`, err => cb(err)), - cb => MockPreloadNode.waitForCids(dirCid, cb) + cb => MockPreloadNode.waitForCids(`/ipfs/${dirCid}`, cb) ], done) }) @@ -375,7 +375,7 @@ describe('preload', () => { .on('error', cb) .on('end', cb) }, - cb => MockPreloadNode.waitForCids(dirCid, cb) + cb => MockPreloadNode.waitForCids(`/ipfs/${dirCid}`, cb) ], done) }) @@ -393,7 +393,7 @@ describe('preload', () => { ipfs.files.lsPullStream(`/ipfs/${dirCid}`), pull.onEnd(cb) ), - cb => MockPreloadNode.waitForCids(dirCid, cb) + cb => MockPreloadNode.waitForCids(`/ipfs/${dirCid}`, cb) ], done) }) @@ -408,7 +408,7 @@ describe('preload', () => { }, cb => MockPreloadNode.clearPreloadCids(cb), cb => ipfs.files.read(`/ipfs/${fileCid}`, err => cb(err)), - cb => MockPreloadNode.waitForCids(fileCid, cb) + cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb) ], done) }) @@ -428,7 +428,7 @@ describe('preload', () => { .on('error', cb) .on('end', cb) }, - cb => MockPreloadNode.waitForCids(fileCid, cb) + cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb) ], done) }) @@ -446,7 +446,7 @@ describe('preload', () => { ipfs.files.readPullStream(`/ipfs/${fileCid}`), pull.onEnd(cb) ), - cb => MockPreloadNode.waitForCids(fileCid, cb) + cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb) ], done) }) @@ -461,7 +461,7 @@ describe('preload', () => { }, cb => MockPreloadNode.clearPreloadCids(cb), cb => ipfs.files.stat(`/ipfs/${fileCid}`, err => cb(err)), - cb => MockPreloadNode.waitForCids(fileCid, cb) + cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb) ], done) }) })