Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 0e0d1dd

Browse files
author
Alan Shaw
authored
fix: enable preload on MFS commands that accept IPFS paths (#2355)
fixes #2335 License: MIT Signed-off-by: Alan Shaw <[email protected]>
1 parent 76e7a51 commit 0e0d1dd

File tree

3 files changed

+177
-24
lines changed

3 files changed

+177
-24
lines changed

‎src/core/components/files-mfs.js

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,15 @@ const callbackify = require('callbackify')
1010
const PassThrough = require('stream').PassThrough
1111
const pull = require('pull-stream/pull')
1212
const map = require('pull-stream/throughs/map')
13+
const isIpfs = require('is-ipfs')
14+
const { cidToString } = require('../../utils/cid')
1315

1416
const mapLsFile = (options = {}) => {
1517
const long = options.long || options.l
1618

1719
return (file) => {
1820
return {
19-
hash: long ? file.cid.toBaseEncodedString(options.cidBase) : '',
21+
hash: long ? cidToString(file.cid, { base: options.cidBase }) : '',
2022
name: file.name,
2123
type: long ? file.type : 0,
2224
size: long ? file.size || 0 : 0
@@ -32,15 +34,28 @@ module.exports = self => {
3234
repoOwner: self._options.repoOwner
3335
})
3436

37+
const withPreload = fn => (...args) => {
38+
const paths = args.filter(arg => isIpfs.ipfsPath(arg) || isIpfs.cid(arg))
39+
40+
if (paths.length) {
41+
const options = args[args.length - 1]
42+
if (options.preload !== false) {
43+
paths.forEach(path => self._preload(path))
44+
}
45+
}
46+
47+
return fn(...args)
48+
}
49+
3550
return {
36-
cp: callbackify.variadic(methods.cp),
51+
cp: callbackify.variadic(withPreload(methods.cp)),
3752
flush: callbackify.variadic(methods.flush),
38-
ls: callbackify.variadic(async (path, options = {}) => {
53+
ls: callbackify.variadic(withPreload(async (path, options = {}) => {
3954
const files = await all(methods.ls(path, options))
4055

4156
return files.map(mapLsFile(options))
42-
}),
43-
lsReadableStream: (path, options = {}) => {
57+
})),
58+
lsReadableStream: withPreload((path, options = {}) => {
4459
const stream = toReadableStream.obj(methods.ls(path, options))
4560
const through = new PassThrough({
4661
objectMode: true
@@ -60,33 +75,33 @@ module.exports = self => {
6075
})
6176

6277
return through
63-
},
64-
lsPullStream: (path, options = {}) => {
78+
}),
79+
lsPullStream: withPreload((path, options = {}) => {
6580
return pull(
6681
toPullStream.source(methods.ls(path, options)),
6782
map(mapLsFile(options))
6883
)
69-
},
84+
}),
7085
mkdir: callbackify.variadic(methods.mkdir),
71-
mv: callbackify.variadic(methods.mv),
72-
read: callbackify(async (path, options = {}) => {
86+
mv: callbackify.variadic(withPreload(methods.mv)),
87+
read: callbackify.variadic(withPreload(async (path, options = {}) => {
7388
return Buffer.concat(await all(methods.read(path, options)))
74-
}),
75-
readPullStream: (path, options = {}) => {
89+
})),
90+
readPullStream: withPreload((path, options = {}) => {
7691
return toPullStream.source(methods.read(path, options))
77-
},
78-
readReadableStream: (path, options = {}) => {
92+
}),
93+
readReadableStream: withPreload((path, options = {}) => {
7994
return toReadableStream(methods.read(path, options))
80-
},
95+
}),
8196
rm: callbackify.variadic(methods.rm),
82-
stat: callbackify(async (path, options = {}) => {
97+
stat: callbackify.variadic(withPreload(async (path, options = {}) => {
8398
const stats = await methods.stat(path, options)
8499

85-
stats.hash = stats.cid.toBaseEncodedString(options && options.cidBase)
100+
stats.hash = cidToString(stats.cid, { base: options.cidBase })
86101
delete stats.cid
87102

88103
return stats
89-
}),
104+
})),
90105
write: callbackify.variadic(async (path, content, options = {}) => {
91106
if (isPullStream.isSource(content)) {
92107
content = pullStreamToAsyncIterator(content)

‎src/core/preload.js

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,12 @@ module.exports = self => {
3434
let requests = []
3535
const apiUris = options.addresses.map(toUri)
3636

37-
const api = (cid, callback) => {
37+
const api = (path, callback) => {
3838
callback = callback || noop
3939

40-
if (typeof cid !== 'string') {
40+
if (typeof path !== 'string') {
4141
try {
42-
cid = new CID(cid).toBaseEncodedString()
42+
path = new CID(path).toBaseEncodedString()
4343
} catch (err) {
4444
return setImmediate(() => callback(err))
4545
}
@@ -50,14 +50,14 @@ module.exports = self => {
5050
const now = Date.now()
5151

5252
retry({ times: fallbackApiUris.length }, (cb) => {
53-
if (stopped) return cb(new Error(`preload aborted for ${cid}`))
53+
if (stopped) return cb(new Error(`preload aborted for ${path}`))
5454

5555
// Remove failed request from a previous attempt
5656
requests = requests.filter(r => r !== request)
5757

5858
const apiUri = fallbackApiUris.shift()
5959

60-
request = preload(`${apiUri}/api/v0/refs?r=true&arg=${cid}`, cb)
60+
request = preload(`${apiUri}/api/v0/refs?r=true&arg=${encodeURIComponent(path)}`, cb)
6161
requests = requests.concat(request)
6262
}, (err) => {
6363
requests = requests.filter(r => r !== request)
@@ -66,7 +66,7 @@ module.exports = self => {
6666
return callback(err)
6767
}
6868

69-
log(`preloaded ${cid} in ${Date.now() - now}ms`)
69+
log(`preloaded ${path} in ${Date.now() - now}ms`)
7070
callback()
7171
})
7272
}

‎test/core/preload.spec.js

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ const chai = require('chai')
99
const dirtyChai = require('dirty-chai')
1010
const expect = chai.expect
1111
chai.use(dirtyChai)
12+
const pull = require('pull-stream')
13+
const CID = require('cids')
1214

1315
const MockPreloadNode = require('../utils/mock-preload-node')
1416
const IPFS = require('../../src')
@@ -326,6 +328,142 @@ describe('preload', () => {
326328
})
327329
})
328330
})
331+
332+
it('should preload content retrieved with files.ls', done => {
333+
let dirCid
334+
335+
waterfall([
336+
cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb),
337+
(res, cb) => {
338+
dirCid = res[res.length - 1].hash
339+
MockPreloadNode.waitForCids(dirCid, cb)
340+
},
341+
cb => MockPreloadNode.clearPreloadCids(cb),
342+
cb => ipfs.files.ls(`/ipfs/${dirCid}`, err => cb(err)),
343+
cb => MockPreloadNode.waitForCids(`/ipfs/${dirCid}`, cb)
344+
], done)
345+
})
346+
347+
it('should preload content retrieved with files.ls by CID', done => {
348+
let dirCid
349+
350+
waterfall([
351+
cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb),
352+
(res, cb) => {
353+
dirCid = res[res.length - 1].hash
354+
MockPreloadNode.waitForCids(dirCid, cb)
355+
},
356+
cb => MockPreloadNode.clearPreloadCids(cb),
357+
cb => ipfs.files.ls(new CID(dirCid), err => cb(err)),
358+
cb => MockPreloadNode.waitForCids(dirCid, cb)
359+
], done)
360+
})
361+
362+
it('should preload content retrieved with files.lsReadableStream', done => {
363+
let dirCid
364+
365+
waterfall([
366+
cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb),
367+
(res, cb) => {
368+
dirCid = res[res.length - 1].hash
369+
MockPreloadNode.waitForCids(dirCid, cb)
370+
},
371+
cb => MockPreloadNode.clearPreloadCids(cb),
372+
cb => {
373+
ipfs.files.lsReadableStream(`/ipfs/${dirCid}`)
374+
.on('data', () => {})
375+
.on('error', cb)
376+
.on('end', cb)
377+
},
378+
cb => MockPreloadNode.waitForCids(`/ipfs/${dirCid}`, cb)
379+
], done)
380+
})
381+
382+
it('should preload content retrieved with files.lsPullStream', done => {
383+
let dirCid
384+
385+
waterfall([
386+
cb => ipfs.add({ path: `/t/${hat()}`, content: Buffer.from(hat()) }, cb),
387+
(res, cb) => {
388+
dirCid = res[res.length - 1].hash
389+
MockPreloadNode.waitForCids(dirCid, cb)
390+
},
391+
cb => MockPreloadNode.clearPreloadCids(cb),
392+
cb => pull(
393+
ipfs.files.lsPullStream(`/ipfs/${dirCid}`),
394+
pull.onEnd(cb)
395+
),
396+
cb => MockPreloadNode.waitForCids(`/ipfs/${dirCid}`, cb)
397+
], done)
398+
})
399+
400+
it('should preload content retrieved with files.read', done => {
401+
let fileCid
402+
403+
waterfall([
404+
cb => ipfs.add(Buffer.from(hat()), cb),
405+
(res, cb) => {
406+
fileCid = res[0].hash
407+
MockPreloadNode.waitForCids(fileCid, cb)
408+
},
409+
cb => MockPreloadNode.clearPreloadCids(cb),
410+
cb => ipfs.files.read(`/ipfs/${fileCid}`, err => cb(err)),
411+
cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb)
412+
], done)
413+
})
414+
415+
it('should preload content retrieved with files.readReadableStream', done => {
416+
let fileCid
417+
418+
waterfall([
419+
cb => ipfs.add(Buffer.from(hat()), cb),
420+
(res, cb) => {
421+
fileCid = res[0].hash
422+
MockPreloadNode.waitForCids(fileCid, cb)
423+
},
424+
cb => MockPreloadNode.clearPreloadCids(cb),
425+
cb => {
426+
ipfs.files.readReadableStream(`/ipfs/${fileCid}`)
427+
.on('data', () => {})
428+
.on('error', cb)
429+
.on('end', cb)
430+
},
431+
cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb)
432+
], done)
433+
})
434+
435+
it('should preload content retrieved with files.readPullStream', done => {
436+
let fileCid
437+
438+
waterfall([
439+
cb => ipfs.add(Buffer.from(hat()), cb),
440+
(res, cb) => {
441+
fileCid = res[0].hash
442+
MockPreloadNode.waitForCids(fileCid, cb)
443+
},
444+
cb => MockPreloadNode.clearPreloadCids(cb),
445+
cb => pull(
446+
ipfs.files.readPullStream(`/ipfs/${fileCid}`),
447+
pull.onEnd(cb)
448+
),
449+
cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb)
450+
], done)
451+
})
452+
453+
it('should preload content retrieved with files.stat', done => {
454+
let fileCid
455+
456+
waterfall([
457+
cb => ipfs.add(Buffer.from(hat()), cb),
458+
(res, cb) => {
459+
fileCid = res[0].hash
460+
MockPreloadNode.waitForCids(fileCid, cb)
461+
},
462+
cb => MockPreloadNode.clearPreloadCids(cb),
463+
cb => ipfs.files.stat(`/ipfs/${fileCid}`, err => cb(err)),
464+
cb => MockPreloadNode.waitForCids(`/ipfs/${fileCid}`, cb)
465+
], done)
466+
})
329467
})
330468

331469
describe('preload disabled', function () {

0 commit comments

Comments
 (0)