diff --git a/package.json b/package.json index 3c6dc5f..1fccd17 100644 --- a/package.json +++ b/package.json @@ -42,20 +42,20 @@ "detect-node": "^2.0.4", "detect-webworker": "^1.0.0", "dirty-chai": "^2.0.1", - "ipfs": "~0.33.0", + "ipld": "~0.20.0", "pull-buffer-stream": "^1.0.0", - "tmp": "~0.0.33" + "pull-traverse": "^1.0.3" }, "dependencies": { "async": "^2.6.1", - "blob": "~0.0.5", "cids": "~0.5.5", "debug": "^4.1.0", - "file-api": "~0.10.4", "filereader-stream": "^2.0.0", "interface-datastore": "~0.6.0", + "ipfs-multipart": "~0.1.0", "ipfs-unixfs": "~0.1.16", - "ipfs-unixfs-engine": "~0.33.0", + "ipfs-unixfs-engine": "~0.34.0", + "ipld-dag-pb": "~0.15.0", "is-pull-stream": "~0.0.0", "is-stream": "^1.1.0", "joi": "^14.0.4", @@ -65,11 +65,8 @@ "promisify-es6": "^1.0.3", "pull-cat": "^1.1.11", "pull-defer": "~0.2.3", - "pull-paramap": "^1.2.2", - "pull-pushable": "^2.2.0", "pull-stream": "^3.6.9", "pull-stream-to-stream": "^1.3.4", - "pull-traverse": "^1.0.3", "stream-to-pull-stream": "^1.7.2" }, "contributors": [ diff --git a/src/core/cp.js b/src/core/cp.js index 009eae0..edb35ad 100644 --- a/src/core/cp.js +++ b/src/core/cp.js @@ -21,7 +21,7 @@ const defaultOptions = { hashAlg: 'sha2-256' } -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsCp () { const args = Array.from(arguments) const { @@ -41,21 +41,21 @@ module.exports = (ipfs) => { options.parents = options.p || options.parents - traverseTo(ipfs, destination.path, {}, (error, result) => { + traverseTo(context, destination.path, {}, (error, result) => { if (error) { if (sources.length === 1) { log('Only one source, copying to a file') - return copyToFile(ipfs, sources.pop(), destination, options, callback) + return copyToFile(context, sources.pop(), destination, options, callback) } else { log('Multiple sources, copying to a directory') - return copyToDirectory(ipfs, sources, destination, options, callback) + return copyToDirectory(context, sources, destination, options, callback) } } const meta = UnixFs.unmarshal(result.node.data) if (meta.type === 'directory') { - return copyToDirectory(ipfs, sources, destination, options, callback) + return copyToDirectory(context, sources, destination, options, callback) } callback(new Error('directory already has entry by that name')) @@ -63,52 +63,51 @@ module.exports = (ipfs) => { } } -const copyToFile = (ipfs, source, destination, options, callback) => { +const copyToFile = (context, source, destination, options, callback) => { waterfall([ (cb) => { parallel([ - (next) => stat(ipfs)(source.path, options, next), - (next) => stat(ipfs)(destination.path, options, (error) => { + (next) => stat(context)(source.path, options, next), + (next) => stat(context)(destination.path, options, (error) => { if (!error) { return next(new Error('directory already has entry by that name')) } next() }), - (next) => traverseTo(ipfs, destination.dir, options, next) + (next) => traverseTo(context, destination.dir, options, next) ], cb) }, ([sourceStats, _, dest], cb) => { waterfall([ - (next) => addLink(ipfs, { + (next) => addLink(context, { parent: dest.node, - child: { - size: sourceStats.cumulativeSize, - hash: sourceStats.hash - }, + size: sourceStats.cumulativeSize, + cid: sourceStats.hash, name: destination.name }, next), - (newParent, next) => { - dest.node = newParent - updateTree(ipfs, dest, next) + ({ node, cid }, next) => { + dest.node = node + dest.cid = cid + updateTree(context, dest, next) }, - (newRoot, cb) => updateMfsRoot(ipfs, newRoot.node.multihash, cb) + ({ node, cid }, cb) => updateMfsRoot(context, cid, cb) ], cb) } ], (error) => callback(error)) } -const copyToDirectory = (ipfs, sources, destination, options, callback) => { +const copyToDirectory = (context, sources, destination, options, callback) => { waterfall([ (cb) => { series([ // stat in parallel (done) => parallel( - sources.map(source => (next) => stat(ipfs)(source.path, options, next)), + sources.map(source => (next) => stat(context)(source.path, options, next)), done ), // this could end up changing the root mfs node so do it after parallel - (done) => traverseTo(ipfs, destination.path, Object.assign({}, options, { + (done) => traverseTo(context, destination.path, Object.assign({}, options, { createLastComponent: true }), done) ], cb) @@ -123,7 +122,7 @@ const copyToDirectory = (ipfs, sources, destination, options, callback) => { parallel( sources.map(source => { return (cb) => { - stat(ipfs)(`${destination.path}/${source.name}`, options, (error) => { + stat(context)(`${destination.path}/${source.name}`, options, (error) => { if (!error) { return cb(new Error('directory already has entry by that name')) } @@ -138,16 +137,14 @@ const copyToDirectory = (ipfs, sources, destination, options, callback) => { // add links to target directory (next) => { waterfall([ - (done) => done(null, dest.node) + (done) => done(null, dest) ].concat( sourceStats.map((sourceStat, index) => { return (dest, done) => { - return addLink(ipfs, { - parent: dest, - child: { - size: sourceStat.cumulativeSize, - hash: sourceStat.hash - }, + return addLink(context, { + parent: dest.node, + size: sourceStat.cumulativeSize, + cid: sourceStat.hash, name: sources[index].name }, done) } @@ -155,13 +152,14 @@ const copyToDirectory = (ipfs, sources, destination, options, callback) => { ), next) }, // update mfs tree - (newParent, next) => { - dest.node = newParent + ({ node, cid }, next) => { + dest.node = node + dest.cid = cid - updateTree(ipfs, dest, next) + updateTree(context, dest, next) }, // save new root CID - (newRoot, cb) => updateMfsRoot(ipfs, newRoot.node.multihash, cb) + (newRoot, cb) => updateMfsRoot(context, newRoot.cid, cb) ], cb) } ], (error) => callback(error)) diff --git a/src/core/flush.js b/src/core/flush.js index ce5ad74..abacb9b 100644 --- a/src/core/flush.js +++ b/src/core/flush.js @@ -8,7 +8,7 @@ const { const defaultOptions = {} -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsFlush (path, options, callback) { if (typeof options === 'function') { callback = options @@ -28,7 +28,7 @@ module.exports = (ipfs) => { options = Object.assign({}, defaultOptions, options) waterfall([ - (cb) => traverseTo(ipfs, path, {}, cb), + (cb) => traverseTo(context, path, {}, cb), (root, cb) => { cb() } diff --git a/src/core/index.js b/src/core/index.js index 33d97a4..496fa7c 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -1,5 +1,6 @@ 'use strict' +const assert = require('assert') const promisify = require('promisify-es6') const { createLock @@ -33,22 +34,27 @@ const unwrappedSynchronousOperations = { } const wrap = ({ - ipfs, mfs, operations, lock + options, mfs, operations, lock }) => { Object.keys(operations).forEach(key => { - mfs[key] = promisify(lock(operations[key](ipfs))) + mfs[key] = promisify(lock(operations[key](options))) }) } const defaultOptions = { - repoOwner: true + repoOwner: true, + ipld: null, + repo: null } -module.exports = (ipfs, options) => { +module.exports = (options) => { const { repoOwner } = Object.assign({}, defaultOptions || {}, options) + assert(options.ipld, 'MFS requires an IPLD instance') + assert(options.repo, 'MFS requires an ipfs-repo instance') + const lock = createLock(repoOwner) const readLock = (operation) => { @@ -62,18 +68,18 @@ module.exports = (ipfs, options) => { const mfs = {} wrap({ - ipfs, mfs, operations: readOperations, lock: readLock + options, mfs, operations: readOperations, lock: readLock }) wrap({ - ipfs, mfs, operations: writeOperations, lock: writeLock + options, mfs, operations: writeOperations, lock: writeLock }) Object.keys(unwrappedOperations).forEach(key => { - mfs[key] = promisify(unwrappedOperations[key](ipfs)) + mfs[key] = promisify(unwrappedOperations[key](options)) }) Object.keys(unwrappedSynchronousOperations).forEach(key => { - mfs[key] = unwrappedSynchronousOperations[key](ipfs) + mfs[key] = unwrappedSynchronousOperations[key](options) }) return mfs diff --git a/src/core/ls.js b/src/core/ls.js index 0f89a75..b7dcb92 100644 --- a/src/core/ls.js +++ b/src/core/ls.js @@ -17,7 +17,7 @@ const defaultOptions = { unsorted: false } -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsLs (path, options, callback) { if (typeof path === 'function') { callback = path @@ -35,21 +35,21 @@ module.exports = (ipfs) => { options.long = options.l || options.long waterfall([ - (cb) => traverseTo(ipfs, path, {}, cb), + (cb) => traverseTo(context, path, {}, cb), (result, cb) => { const meta = UnixFs.unmarshal(result.node.data) if (meta.type === 'directory') { map(result.node.links, (link, next) => { waterfall([ - (done) => loadNode(ipfs, link, done), - (node, done) => { + (done) => loadNode(context, link, done), + ({ node, cid }, done) => { const meta = UnixFs.unmarshal(node.data) done(null, { name: link.name, type: meta.type, - hash: formatCid(node.multihash, options.cidBase), + hash: formatCid(cid, options.cidBase), size: meta.fileSize() || 0 }) } @@ -59,7 +59,7 @@ module.exports = (ipfs) => { cb(null, [{ name: result.name, type: meta.type, - hash: formatCid(result.node.multihash, options.cidBase), + hash: formatCid(result.cid, options.cidBase), size: meta.fileSize() || 0 }]) } diff --git a/src/core/mkdir.js b/src/core/mkdir.js index 2ac62a9..0f991c8 100644 --- a/src/core/mkdir.js +++ b/src/core/mkdir.js @@ -15,7 +15,7 @@ const defaultOptions = { cidVersion: 0 } -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsMkdir (path, options, callback) { if (typeof options === 'function') { callback = options @@ -41,7 +41,7 @@ module.exports = (ipfs) => { waterfall([ (cb) => { - traverseTo(ipfs, path, { + traverseTo(context, path, { parents: false, createLastComponent: false }, (error) => { @@ -58,13 +58,13 @@ module.exports = (ipfs) => { return cb(error) }) }, - (cb) => traverseTo(ipfs, path, { + (cb) => traverseTo(context, path, { parents: options.parents, flush: options.flush, createLastComponent: true }, cb), - (result, cb) => updateTree(ipfs, result, cb), - (newRoot, next) => updateMfsRoot(ipfs, newRoot.node.multihash, next) + (result, cb) => updateTree(context, result, cb), + (newRoot, next) => updateMfsRoot(context, newRoot.cid, next) ], (error) => { if (error && error.message.includes('file already exists') && options.parents) { // when the directory already exists and we are creating intermediate diff --git a/src/core/mv.js b/src/core/mv.js index 22db86b..fee5998 100644 --- a/src/core/mv.js +++ b/src/core/mv.js @@ -15,7 +15,7 @@ const defaultOptions = { hashAlg: 'sha2-256' } -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsMv () { let args = Array.from(arguments) @@ -42,8 +42,8 @@ module.exports = (ipfs) => { })) series([ - (cb) => cp(ipfs).apply(null, cpArgs.concat(cb)), - (cb) => rm(ipfs).apply(null, rmArgs.concat(cb)) + (cb) => cp(context).apply(null, cpArgs.concat(cb)), + (cb) => rm(context).apply(null, rmArgs.concat(cb)) ], callback) } } diff --git a/src/core/read-pull-stream.js b/src/core/read-pull-stream.js index ed7d7e2..f908eca 100644 --- a/src/core/read-pull-stream.js +++ b/src/core/read-pull-stream.js @@ -18,7 +18,7 @@ const defaultOptions = { length: undefined } -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsReadPullStream (path, options = {}) { options = Object.assign({}, defaultOptions, options) @@ -30,13 +30,12 @@ module.exports = (ipfs) => { once(path), asyncMap((path, cb) => { createLock().readLock((next) => { - traverseTo(ipfs, path, { + traverseTo(context, path, { parents: false }, next) })(cb) }), - asyncMap((result, cb) => { - const node = result.node + asyncMap(({ node, cid }, cb) => { const meta = UnixFs.unmarshal(node.data) if (meta.type !== 'file') { @@ -46,7 +45,7 @@ module.exports = (ipfs) => { log(`Getting ${path} content`) pull( - exporter(node.multihash, ipfs.dag, { + exporter(cid, context.ipld, { offset: options.offset, length: options.length }), diff --git a/src/core/read-readable-stream.js b/src/core/read-readable-stream.js index 50eceb2..a851722 100644 --- a/src/core/read-readable-stream.js +++ b/src/core/read-readable-stream.js @@ -3,8 +3,8 @@ const readPullStream = require('./read-pull-stream') const toStream = require('pull-stream-to-stream') -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsReadReadableStream (path, options = {}) { - return toStream.source(readPullStream(ipfs)(path, options)) + return toStream.source(readPullStream(context)(path, options)) } } diff --git a/src/core/read.js b/src/core/read.js index df924da..9160427 100644 --- a/src/core/read.js +++ b/src/core/read.js @@ -4,7 +4,7 @@ const pull = require('pull-stream/pull') const collect = require('pull-stream/sinks/collect') const readPullStream = require('./read-pull-stream') -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsRead (path, options, callback) { if (typeof options === 'function') { callback = options @@ -12,7 +12,7 @@ module.exports = (ipfs) => { } pull( - readPullStream(ipfs)(path, options), + readPullStream(context)(path, options), collect((error, buffers) => { if (error) { return callback(error) diff --git a/src/core/rm.js b/src/core/rm.js index e826efe..b6752d5 100644 --- a/src/core/rm.js +++ b/src/core/rm.js @@ -6,7 +6,6 @@ const series = require('async/series') const { DAGNode } = require('ipld-dag-pb') -const CID = require('cids') const { traverseTo, updateTree, @@ -16,10 +15,13 @@ const { } = require('./utils') const defaultOptions = { - recursive: false + recursive: false, + cidVersion: 0, + hashAlg: 'sha2-256', + codec: 'dag-pb' } -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsRm () { const args = Array.from(arguments) const { @@ -34,20 +36,20 @@ module.exports = (ipfs) => { series( sources.map(source => { - return (done) => removePath(ipfs, source.path, options, done) + return (done) => removePath(context, source.path, options, done) }), (error) => callback(error) ) } } -const removePath = (ipfs, path, options, callback) => { +const removePath = (context, path, options, callback) => { if (path === FILE_SEPARATOR) { return callback(new Error('Cannot delete root')) } waterfall([ - (cb) => traverseTo(ipfs, path, { + (cb) => traverseTo(context, path, { withCreateHint: false }, cb), (result, cb) => { @@ -60,16 +62,22 @@ const removePath = (ipfs, path, options, callback) => { waterfall([ (next) => DAGNode.rmLink(result.parent.node, result.name, next), (newParentNode, next) => { - ipfs.dag.put(newParentNode, { - cid: new CID(newParentNode.hash || newParentNode.multihash) - }, (error) => next(error, newParentNode)) + context.ipld.put(newParentNode, { + version: options.cidVersion, + format: options.codec, + hashAlg: options.hashAlg + }, (error, cid) => next(error, { + node: newParentNode, + cid + })) }, - (newParentNode, next) => { - result.parent.node = newParentNode + ({ node, cid }, next) => { + result.parent.node = node + result.parent.cid = cid - updateTree(ipfs, result.parent, next) + updateTree(context, result.parent, next) }, - (newRoot, next) => updateMfsRoot(ipfs, newRoot.node.multihash, next) + (newRoot, next) => updateMfsRoot(context, newRoot.cid, next) ], cb) } ], callback) diff --git a/src/core/stat.js b/src/core/stat.js index fa5ead9..79d1978 100644 --- a/src/core/stat.js +++ b/src/core/stat.js @@ -15,7 +15,7 @@ const defaultOptions = { cidBase: 'base58btc' } -module.exports = (ipfs) => { +module.exports = (context) => { return function mfsStat (path, options, callback) { if (typeof options === 'function') { callback = options @@ -27,13 +27,13 @@ module.exports = (ipfs) => { log(`Fetching stats for ${path}`) waterfall([ - (done) => traverseTo(ipfs, path, { + (done) => traverseTo(context, path, { withCreateHint: false }, done), - ({ node }, done) => { + ({ node, cid }, done) => { if (options.hash) { return done(null, { - hash: formatCid(node.multihash, options.cidBase) + hash: formatCid(cid, options.cidBase) }) } else if (options.size) { return done(null, { @@ -50,7 +50,7 @@ module.exports = (ipfs) => { } done(null, { - hash: formatCid(node.multihash, options.cidBase), + hash: formatCid(cid, options.cidBase), size: meta.fileSize() || 0, cumulativeSize: node.size, blocks: blocks, diff --git a/src/core/utils/add-link.js b/src/core/utils/add-link.js index 00ae3b7..8ee1bd4 100644 --- a/src/core/utils/add-link.js +++ b/src/core/utils/add-link.js @@ -1,27 +1,38 @@ 'use strict' -const CID = require('cids') -const dagPb = require('ipld-dag-pb') const { DAGNode, - DAGLink -} = dagPb + DAGLink, + util: { + cid + } +} = require('ipld-dag-pb') const waterfall = require('async/waterfall') -const addLink = (ipfs, options, callback) => { - options = Object.assign({}, { - parent: undefined, - child: undefined, - name: '', - flush: true - }, options) +const defaultOptions = { + parent: undefined, + cid: undefined, + name: '', + size: undefined, + flush: true, + cidVersion: 0, + hashAlg: 'sha2-256', + codec: 'dag-pb' +} + +const addLink = (context, options, callback) => { + options = Object.assign({}, defaultOptions, options) if (!options.parent) { - return callback(new Error('No parent passed to addLink')) + return callback(new Error('No parent DAGNode passed to addLink')) + } + + if (!options.cid) { + return callback(new Error('No child cid passed to addLink')) } - if (!options.child) { - return callback(new Error('No child passed to addLink')) + if (!options.size) { + return callback(new Error('No child size passed to addLink')) } waterfall([ @@ -35,17 +46,30 @@ const addLink = (ipfs, options, callback) => { }, (parent, done) => { // Add the new link to the parent - DAGNode.addLink(parent, new DAGLink(options.name, options.child.size, options.child.hash || options.child.multihash), done) + DAGNode.addLink(parent, new DAGLink(options.name, options.size, options.cid), done) }, (parent, done) => { if (!options.flush) { - return done(null, parent) + return cid(parent, { + version: options.cidVersion, + hashAlg: options.hashAlg + }, (err, cid) => { + done(err, { + node: parent, + cid + }) + }) } // Persist the new parent DAGNode - ipfs.dag.put(parent, { - cid: new CID(parent.hash || parent.multihash) - }, (error) => done(error, parent)) + context.ipld.put(parent, { + version: options.cidVersion, + format: options.codec, + hashAlg: options.hashAlg + }, (error, cid) => done(error, { + node: parent, + cid + })) } ], callback) } diff --git a/src/core/utils/create-node.js b/src/core/utils/create-node.js deleted file mode 100644 index 8e426ba..0000000 --- a/src/core/utils/create-node.js +++ /dev/null @@ -1,29 +0,0 @@ -'use strict' - -const waterfall = require('async/waterfall') -const { - DAGNode -} = require('ipld-dag-pb') - -const defaultOptions = { - format: 'dag-pb', - hashAlg: 'sha2-256' -} - -const createNode = (ipfs, data, links, options, callback) => { - options = Object.assign({}, defaultOptions, options) - - waterfall([ - // Create a DAGNode with the new data - (cb) => DAGNode.create(data, links, cb), - (newNode, cb) => { - // Persist it - ipfs.dag.put(newNode, { - format: options.format, - hashAlg: options.hashAlg - }, (error) => cb(error, newNode)) - } - ], callback) -} - -module.exports = createNode diff --git a/src/core/utils/index.js b/src/core/utils/index.js index dee0ecb..ceecb8b 100644 --- a/src/core/utils/index.js +++ b/src/core/utils/index.js @@ -7,7 +7,6 @@ module.exports = { bufferPullStreamSource: require('./buffer-pull-stream-source'), countStreamBytes: require('./count-stream-bytes'), createLock: require('./create-lock'), - createNode: require('./create-node'), errors: require('./errors'), formatCid: require('./format-cid'), limitStreamBytes: require('./limit-stream-bytes'), diff --git a/src/core/utils/load-node.js b/src/core/utils/load-node.js index 4d64f3b..2572a33 100644 --- a/src/core/utils/load-node.js +++ b/src/core/utils/load-node.js @@ -4,22 +4,17 @@ const waterfall = require('async/waterfall') const CID = require('cids') const log = require('debug')('ipfs:mfs:utils:load-node') -const loadNode = (ipfs, object, callback) => { - const multihash = object && (object.multihash || object.hash) - - if (!multihash) { - log(`No multihash passed so cannot load DAGNode`) - - return callback() - } - - const cid = new CID(multihash) +const loadNode = (context, dagLink, callback) => { + const cid = new CID(dagLink.cid) log(`Loading DAGNode for child ${cid.toBaseEncodedString()}`) waterfall([ - (cb) => ipfs.dag.get(cid, cb), - (result, cb) => cb(null, result.value) + (cb) => context.ipld.get(cid, cb), + (result, cb) => cb(null, { + node: result.value, + cid + }) ], callback) } diff --git a/src/core/utils/traverse-to.js b/src/core/utils/traverse-to.js index 8adaf04..048554c 100644 --- a/src/core/utils/traverse-to.js +++ b/src/core/utils/traverse-to.js @@ -5,13 +5,15 @@ const log = require('debug')('ipfs:mfs:utils:traverse-to') const UnixFS = require('ipfs-unixfs') const waterfall = require('async/waterfall') const reduce = require('async/reduce') +const { + DAGNode +} = require('ipld-dag-pb') const withMfsRoot = require('./with-mfs-root') const validatePath = require('./validate-path') const addLink = require('./add-link') const { FILE_SEPARATOR } = require('./constants') -const createNode = require('./create-node') const { NonFatalError } = require('./errors') @@ -20,10 +22,13 @@ const defaultOptions = { parents: false, flush: true, createLastComponent: false, - withCreateHint: true + withCreateHint: true, + cidVersion: 0, + format: 'dag-pb', + hashAlg: 'sha2-256' } -const traverseTo = (ipfs, path, options, callback) => { +const traverseTo = (context, path, options, callback) => { options = Object.assign({}, defaultOptions, options) log('Traversing to', path) @@ -32,30 +37,33 @@ const traverseTo = (ipfs, path, options, callback) => { (cb) => validatePath(path, cb), (path, cb) => { if (path.type === 'mfs') { - return traverseToMfsObject(ipfs, path, options, cb) + return traverseToMfsObject(context, path, options, cb) } - return traverseToIpfsObject(ipfs, path, options, cb) + return traverseToIpfsObject(context, path, options, cb) } ], callback) } -const traverseToIpfsObject = (ipfs, path, options, callback) => { +const traverseToIpfsObject = (context, path, options, callback) => { log('IPFS', path) + const cid = new CID(path.path) + waterfall([ - (cb) => ipfs.dag.get(path.path, cb), + (cb) => context.ipld.get(cid, cb), (result, cb) => cb(null, { name: path.name, node: result && result.value, - parent: null + parent: null, + cid }) ], callback) } -const traverseToMfsObject = (ipfs, path, options, callback) => { +const traverseToMfsObject = (context, path, options, callback) => { waterfall([ - (done) => withMfsRoot(ipfs, done), + (done) => withMfsRoot(context, done), (root, done) => { const pathSegments = path.path .split(FILE_SEPARATOR) @@ -64,20 +72,22 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { const trail = [] waterfall([ - (cb) => ipfs.dag.get(root, cb), + (cb) => context.ipld.get(root, cb), (result, cb) => { const rootNode = result.value trail.push({ name: FILE_SEPARATOR, node: rootNode, - parent: null + parent: null, + cid: root }) reduce(pathSegments.map((pathSegment, index) => ({ pathSegment, index })), { name: FILE_SEPARATOR, node: rootNode, - parent: null + parent: null, + cid: root }, (parent, { pathSegment, index }, done) => { const existingLink = parent.node.links.find(link => link.name === pathSegment) @@ -97,20 +107,26 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { log(`Adding empty directory '${pathSegment}' to parent ${parent.name}`) return waterfall([ - (next) => createNode(ipfs, new UnixFS('directory').marshal(), [], options, next), + (next) => createNode(context, new UnixFS('directory').marshal(), [], options, next), (emptyDirectory, next) => { - addLink(ipfs, { + addLink(context, { parent: parent.node, - child: emptyDirectory, + size: emptyDirectory.node.size, + cid: emptyDirectory.cid, name: pathSegment, flush: options.flush - }, (error, updatedParent) => { - parent.node = updatedParent + }, (error, result) => { + if (error) { + return next(error) + } + + parent.node = result.node + parent.cid = result.cid - next(error, { + next(null, { name: pathSegment, - node: emptyDirectory, - cid: new CID(emptyDirectory.multihash), + node: emptyDirectory.node, + cid: emptyDirectory.cid, parent: parent }) }) @@ -122,11 +138,8 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { }) } - let hash = existingLink.hash || existingLink.multihash - const cid = new CID(hash) - // child existed, fetch it - ipfs.dag.get(cid, (error, result) => { + context.ipld.get(existingLink.cid, (error, result) => { if (error) { return done(error) } @@ -136,7 +149,8 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { const child = { name: pathSegment, node, - parent: parent + parent: parent, + cid: existingLink.cid } trail.push(child) @@ -150,4 +164,24 @@ const traverseToMfsObject = (ipfs, path, options, callback) => { ], callback) } +const createNode = (context, data, links, options, callback) => { + options = Object.assign({}, defaultOptions, options) + + waterfall([ + // Create a DAGNode with the new data + (cb) => DAGNode.create(data, links, cb), + (newNode, cb) => { + // Persist it + context.ipld.put(newNode, { + version: options.cidVersion, + format: options.format, + hashAlg: options.hashAlg + }, (error, cid) => cb(error, { + node: newNode, + cid + })) + } + ], callback) +} + module.exports = traverseTo diff --git a/src/core/utils/update-mfs-root.js b/src/core/utils/update-mfs-root.js index 3dffdf5..3dac620 100644 --- a/src/core/utils/update-mfs-root.js +++ b/src/core/utils/update-mfs-root.js @@ -7,29 +7,13 @@ const { MFS_ROOT_KEY } = require('./constants') -const updateMfsRoot = (ipfs, buffer, callback) => { - const repo = ipfs._repo - const datastore = repo && repo.datastore - - if (!repo || !datastore) { - return callback(new Error('Please run jsipfs init first')) - } - +const updateMfsRoot = (context, buffer, callback) => { const cid = new CID(buffer) log(`New MFS root will be ${cid.toBaseEncodedString()}`) waterfall([ - (cb) => { - if (repo.closed) { - log('Opening datastore') - return datastore.open((error) => cb(error)) - } - - log('Datastore was already open') - cb() - }, - (cb) => datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => cb(error)) + (cb) => context.repo.datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => cb(error)) ], (error) => callback(error, cid)) } diff --git a/src/core/utils/update-tree.js b/src/core/utils/update-tree.js index c9d5a29..2a07dc6 100644 --- a/src/core/utils/update-tree.js +++ b/src/core/utils/update-tree.js @@ -3,27 +3,40 @@ const doWhilst = require('async/doWhilst') const addLink = require('./add-link') -const updateTree = (ipfs, child, callback) => { +const updateTree = (context, child, callback) => { doWhilst( (next) => { if (!child.parent) { - const lastChild = child + const previousChild = child child = null - return next(null, lastChild) + + return next(null, { + node: previousChild.node, + cid: previousChild.cid + }) } - addLink(ipfs, { + addLink(context, { parent: child.parent.node, - child: child.node, + size: child.node.size, + cid: child.cid, name: child.name, flush: true - }, (error, updatedParent) => { - child.parent.node = updatedParent + }, (error, result) => { + if (error) { + return next(error) + } + + child.parent.node = result.node + child.parent.cid = result.cid - const lastChild = child + const previousChild = child child = child.parent - next(error, lastChild) + next(null, { + node: previousChild.node, + cid: previousChild.cid + }) }) }, () => Boolean(child), diff --git a/src/core/utils/with-mfs-root.js b/src/core/utils/with-mfs-root.js index 820a22b..9dcedd2 100644 --- a/src/core/utils/with-mfs-root.js +++ b/src/core/utils/with-mfs-root.js @@ -1,6 +1,10 @@ 'use strict' const CID = require('cids') +const UnixFs = require('ipfs-unixfs') +const { + DAGNode +} = require('ipld-dag-pb') const log = require('debug')('ipfs:mfs:utils:with-mfs-root') const waterfall = require('async/waterfall') @@ -8,39 +12,31 @@ const { MFS_ROOT_KEY } = require('./constants') -const withMfsRoot = (ipfs, callback) => { - const repo = ipfs._repo - const datastore = repo && repo.datastore - - if (!repo || !datastore) { - return callback(new Error('Please run jsipfs init first')) - } - +const withMfsRoot = (context, callback) => { waterfall([ // Open the repo if it's been closed - (cb) => datastore.open((error) => cb(error)), + (cb) => context.repo.datastore.open((error) => cb(error)), (cb) => { // Load the MFS root CID - datastore.get(MFS_ROOT_KEY, (error, result) => { + context.repo.datastore.get(MFS_ROOT_KEY, (error, result) => { // Once datastore-level releases its error.code addition, we can remove error.notFound logic if (error && (error.notFound || error.code === 'ERR_NOT_FOUND')) { log('Creating new MFS root') return waterfall([ // Store an empty node as the root - (next) => ipfs.add({ - path: '/' + (next) => DAGNode.create(new UnixFs('directory').marshal(), next), + (node, next) => context.ipld.put(node, { + version: 0, + hashAlg: 'sha2-256', + format: 'dag-pb' }, next), - // Turn the hash into a Buffer - ([{ hash }], next) => next(null, new CID(hash)), - (cid, next) => repo.closed ? datastore.open((error) => next(error, cid)) : next(null, cid), // Store the Buffer in the datastore - (cid, next) => datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => next(error, cid)) + (cid, next) => context.repo.datastore.put(MFS_ROOT_KEY, cid.buffer, (error) => next(error, cid)) ], cb) } - const cid = result ? new CID(result) : null - cb(error, cid) + cb(error, result ? new CID(result) : null) }) }, // Turn the Buffer into a CID diff --git a/src/core/write.js b/src/core/write.js index a520cd8..089fec0 100644 --- a/src/core/write.js +++ b/src/core/write.js @@ -50,7 +50,7 @@ const defaultOptions = { leafType: 'raw' } -module.exports = function mfsWrite (ipfs) { +module.exports = function mfsWrite (context) { return promisify((path, content, options, callback) => { if (typeof options === 'function') { callback = options @@ -90,16 +90,16 @@ module.exports = function mfsWrite (ipfs) { if (opts.createLastComponent) { createLock().writeLock((callback) => { - traverseTo(ipfs, path.directory, opts, (error, result) => callback(error, { source, containingFolder: result })) + traverseTo(context, path.directory, opts, (error, result) => callback(error, { source, containingFolder: result })) })(next) } else { createLock().readLock((callback) => { - traverseTo(ipfs, path.directory, opts, (error, result) => callback(error, { source, containingFolder: result })) + traverseTo(context, path.directory, opts, (error, result) => callback(error, { source, containingFolder: result })) })(next) } }, ({ source, containingFolder }, next) => { - updateOrImport(ipfs, options, path, source, containingFolder, next) + updateOrImport(context, options, path, source, containingFolder, next) } ], done) } @@ -107,7 +107,7 @@ module.exports = function mfsWrite (ipfs) { }) } -const updateOrImport = (ipfs, options, path, source, containingFolder, callback) => { +const updateOrImport = (context, options, path, source, containingFolder, callback) => { waterfall([ (next) => { const existingChild = containingFolder.node.links.reduce((last, child) => { @@ -119,7 +119,7 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback) }, null) if (existingChild) { - return loadNode(ipfs, existingChild, next) + return loadNode(context, existingChild, next) } if (!options.create) { @@ -129,8 +129,12 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback) next(null, null) }, - (existingChild, next) => { - write(ipfs, existingChild, source, options, next) + (result, next) => { + const { + cid, node + } = result || {} + + write(context, cid, node, source, options, next) }, // The slow bit is done, now add or replace the DAGLink in the containing directory @@ -141,33 +145,32 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback) createLastComponent: options.parents }) - traverseTo(ipfs, path.directory, opts, (error, containingFolder) => { + traverseTo(context, path.directory, opts, (error, containingFolder) => { if (error) { return callback(error) } waterfall([ (next) => { - addLink(ipfs, { + addLink(context, { parent: containingFolder.node, name: path.name, - child: { - multihash: child.multihash || child.hash, - size: child.size - }, + cid: child.multihash || child.hash, + size: child.size, flush: options.flush - }, (error, newContaingFolder) => { + }, (error, { node, cid }) => { // Store new containing folder CID - containingFolder.node = newContaingFolder + containingFolder.node = node + containingFolder.cid = cid next(error) }) }, // Update the MFS tree from the containingFolder upwards - (next) => updateTree(ipfs, containingFolder, next), + (next) => updateTree(context, containingFolder, next), // Update the MFS record with the new CID for the root of the tree - (newRoot, next) => updateMfsRoot(ipfs, newRoot.node.multihash, next) + (newRoot, next) => updateMfsRoot(context, newRoot.cid, next) ], (error, result) => { callback(error, result) }) @@ -176,12 +179,10 @@ const updateOrImport = (ipfs, options, path, source, containingFolder, callback) }], callback) } -const write = (ipfs, existingNode, source, options, callback) => { - let existingNodeCid +const write = (context, existingNodeCid, existingNode, source, options, callback) => { let existingNodeMeta if (existingNode) { - existingNodeCid = new CID(existingNode.multihash) existingNodeMeta = unmarshal(existingNode.data) log(`Overwriting file ${existingNodeCid.toBaseEncodedString()} offset ${options.offset} length ${options.length}`) } else { @@ -200,7 +201,7 @@ const write = (ipfs, existingNode, source, options, callback) => { sources.push(startFile) pull( - exporter(existingNodeCid, ipfs.dag, { + exporter(existingNodeCid, context.ipld, { offset: 0, length: options.offset }), @@ -237,7 +238,7 @@ const write = (ipfs, existingNode, source, options, callback) => { if (fileSize > offset) { log(`Writing last ${fileSize - offset} of ${fileSize} bytes from original file`) pull( - exporter(existingNodeCid, ipfs.dag, { + exporter(existingNodeCid, context.ipld, { offset }), collect((error, files) => { @@ -269,7 +270,7 @@ const write = (ipfs, existingNode, source, options, callback) => { path: '', content: cat(sources) }]), - importer(ipfs.dag, { + importer(context.ipld, { progress: options.progress, hashAlg: options.hash, cidVersion: options.cidVersion, diff --git a/test/cp.spec.js b/test/cp.spec.js index a2d054c..2e4deb0 100644 --- a/test/cp.spec.js +++ b/test/cp.spec.js @@ -22,10 +22,6 @@ describe('cp', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('refuses to copy files without arguments', () => { return mfs.cp() .then(() => { diff --git a/test/flush.spec.js b/test/flush.spec.js index 25f375a..3220f84 100644 --- a/test/flush.spec.js +++ b/test/flush.spec.js @@ -17,10 +17,6 @@ describe('flush', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('flushes the root node', () => { return mfs.flush() }) diff --git a/test/helpers/collect-leaf-cids.js b/test/helpers/collect-leaf-cids.js index c4c876b..d71a9c7 100644 --- a/test/helpers/collect-leaf-cids.js +++ b/test/helpers/collect-leaf-cids.js @@ -4,14 +4,14 @@ const pull = require('pull-stream') const traverse = require('pull-traverse') const CID = require('cids') -module.exports = (ipfs, multihash) => { +module.exports = (mfs, multihash) => { return new Promise((resolve, reject) => { pull( traverse.depthFirst(new CID(multihash), (cid) => { return pull( pull.values([cid]), pull.asyncMap((cid, callback) => { - ipfs.dag.get(cid, (error, result) => { + mfs.ipld.get(cid, (error, result) => { callback(error, !error && result.value) }) }), @@ -21,7 +21,7 @@ module.exports = (ipfs, multihash) => { } return callback( - null, node.links.map(link => new CID(link.multihash)) + null, node.links.map(link => link.cid) ) }), pull.filter(Boolean), diff --git a/test/helpers/index.js b/test/helpers/index.js index f350c06..94a1d68 100644 --- a/test/helpers/index.js +++ b/test/helpers/index.js @@ -1,40 +1,33 @@ 'use strict' -const ipfs = require('ipfs') -const path = require('path') -const os = require('os') -const promisify = require('promisify-es6') -const { - race, - waterfall -} = require('async') const core = require('../../src/core') const isWebWorker = require('detect-webworker') +const promisify = require('promisify-es6') +const InMemoryDataStore = require('interface-datastore').MemoryDatastore +const inMemoryIpld = promisify(require('ipld').inMemory) + +const createMfs = async () => { + let ipld = await inMemoryIpld() + let datastore = new InMemoryDataStore() -const createMfs = promisify((cb) => { - let node = ipfs.createNode({ - repo: path.join(os.tmpdir(), `ipfs-mfs-tests-${Math.random()}`), - mfs: { - // https://github.com/Joris-van-der-Wel/karma-mocha-webworker/issues/4 - // There is no IPFS node running on the main thread so run it on the - // worker along with the tests - repoOwner: isWebWorker - } + const mfs = core({ + ipld, + repo: { + datastore + }, + + // https://github.com/Joris-van-der-Wel/karma-mocha-webworker/issues/4 + // There is no IPFS node running on the main thread so run it on the + // worker along with the tests + repoOwner: isWebWorker }) - waterfall([ - (done) => race([ - (next) => node.once('error', next), - (next) => node.once('ready', next) - ], (error) => done(error, node)), - (node, done) => { - const mfs = core(node) - mfs.node = node + // to allow tests to verify information + mfs.ipld = ipld + mfs.datastore = datastore - done(null, mfs) - } - ], cb) -}) + return mfs +} module.exports = { createMfs, diff --git a/test/ls.spec.js b/test/ls.spec.js index c9f575c..0db2e11 100644 --- a/test/ls.spec.js +++ b/test/ls.spec.js @@ -24,10 +24,6 @@ describe('ls', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('lists the root directory by default', () => { const fileName = `small-file-${Math.random()}.txt` const content = Buffer.from('Hello world') diff --git a/test/mkdir.spec.js b/test/mkdir.spec.js index a1f7f87..bbd6eae 100644 --- a/test/mkdir.spec.js +++ b/test/mkdir.spec.js @@ -20,10 +20,6 @@ describe('mkdir', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('requires a directory', (done) => { mfs.mkdir('', (error) => { expect(error.message).to.contain('no path given') diff --git a/test/mv.spec.js b/test/mv.spec.js index 2c616c6..746a340 100644 --- a/test/mv.spec.js +++ b/test/mv.spec.js @@ -21,10 +21,6 @@ describe('mv', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('refuses to move files without arguments', () => { return mfs.mv() .then(() => { diff --git a/test/read.spec.js b/test/read.spec.js index 9451d5f..4e830b9 100644 --- a/test/read.spec.js +++ b/test/read.spec.js @@ -26,10 +26,6 @@ describe('read', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - const methods = [{ name: 'read', read: function () { diff --git a/test/rm.spec.js b/test/rm.spec.js index 1410336..811d638 100644 --- a/test/rm.spec.js +++ b/test/rm.spec.js @@ -24,10 +24,6 @@ describe('rm', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('refuses to remove files without arguments', () => { return mfs.rm() .then(() => { diff --git a/test/stat.spec.js b/test/stat.spec.js index b5e8e02..c0a8b95 100644 --- a/test/stat.spec.js +++ b/test/stat.spec.js @@ -27,10 +27,6 @@ describe('stat', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('refuses to stat files with an empty path', () => { return mfs.stat('') .then(() => { diff --git a/test/write.spec.js b/test/write.spec.js index 0058fbd..127939a 100644 --- a/test/write.spec.js +++ b/test/write.spec.js @@ -72,10 +72,6 @@ describe('write', function () { }) }) - after((done) => { - mfs.node.stop(done) - }) - it('explodes if it cannot convert content to a pull stream', () => { return mfs.write('/foo', -1, { create: true @@ -418,7 +414,7 @@ describe('write', function () { rawLeaves: true }) .then(() => mfs.stat(path)) - .then((stats) => collectLeafCids(mfs.node, stats.hash)) + .then((stats) => collectLeafCids(mfs, stats.hash)) .then((cids) => { const rawNodes = cids .filter(cid => cid.codec === 'raw')