From c819392cebe7c42fface31211a5202aa5625ad9e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 13 Mar 2019 19:53:47 +0000 Subject: [PATCH 1/5] feat: adds http DAG api --- src/http/api/resources/dag.js | 237 ++++++++++++++++++++++++++++++++ src/http/api/resources/index.js | 1 + src/http/api/routes/dag.js | 43 ++++++ src/http/api/routes/index.js | 1 + test/http-api/dag.js | 111 +++++++++++++++ test/http-api/index.js | 1 + test/http-api/interface.js | 4 +- 7 files changed, 395 insertions(+), 3 deletions(-) create mode 100644 src/http/api/resources/dag.js create mode 100644 src/http/api/routes/dag.js create mode 100644 test/http-api/dag.js diff --git a/src/http/api/resources/dag.js b/src/http/api/resources/dag.js new file mode 100644 index 0000000000..9a9c71b399 --- /dev/null +++ b/src/http/api/resources/dag.js @@ -0,0 +1,237 @@ +'use strict' + +const promisify = require('promisify-es6') +const CID = require('cids') +const multipart = require('ipfs-multipart') +const Joi = require('joi') +const multibase = require('multibase') +const Boom = require('boom') +const debug = require('debug') +const log = debug('ipfs:http-api:dag') +log.error = debug('ipfs:http-api:dag:error') + +// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` +exports.parseKey = (request, h) => { + if (!request.query.arg) { + throw Boom.badRequest("Argument 'key' is required") + } + + let key = request.query.arg.trim() + let path + + if (key.startsWith('/ipfs')) { + key = key.substring(5) + } + + const parts = key.split('/') + + if (parts.length > 1) { + key = parts.shift() + path = `${parts.join('/')}` + } + + if (path.endsWith('/')) { + path = path.substring(0, path.length - 1) + } + + try { + return { + key: new CID(key), + path + } + } catch (err) { + log.error(err) + throw Boom.badRequest("invalid 'ipfs ref' path") + } +} + +exports.get = { + validate: { + query: Joi.object().keys({ + 'data-encoding': Joi.string().valid(['text', 'base64']).default('base64'), + 'cid-base': Joi.string().valid(multibase.names) + }).unknown() + }, + + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + async handler (request, h) { + const { + key, + path + } = request.pre.args + const { ipfs } = request.server.app + + let result + + try { + result = await ipfs.dag.get(key, path) + } catch (err) { + throw Boom.boomify(err, { message: 'Failed to get dag node' }) + } + + if (key.codec === 'dag-pb' && result.value) { + if (typeof result.value.toJSON === 'function') { + result.value = result.value.toJSON() + } + + if (Buffer.isBuffer(result.value.data)) { + result.value.data = result.value.data.toString(request.query.dataencoding) + } + } + + return h.response(result.value) + } +} + +exports.put = { + validate: { + query: Joi.object().keys({ + // TODO validate format, & hash + format: Joi.string(), + 'input-enc': Joi.string().valid('dag-cbor', 'dag-pb', 'raw'), + pin: Joi.boolean(), + hash: Joi.string(), + 'cid-base': Joi.string().valid(multibase.names).default('base58btc') + }).unknown() + }, + + // pre request handler that parses the args and returns `node` + // which is assigned to `request.pre.args` + async parseArgs (request, h) { + if (!request.payload) { + throw Boom.badRequest("File argument 'data' is required") + } + + const enc = request.query.inputenc + + const fileStream = await new Promise((resolve, reject) => { + multipart.reqParser(request.payload) + .on('file', (name, stream) => resolve(stream)) + .on('end', () => reject(Boom.badRequest("File argument 'data' is required"))) + }) + + let data = await new Promise((resolve, reject) => { + fileStream + .on('data', data => resolve(data)) + .on('end', () => reject(Boom.badRequest("File argument 'data' is required"))) + }) + + if (enc === 'json') { + try { + data = JSON.parse(data.toString()) + } catch (err) { + throw Boom.badRequest('Failed to parse the JSON: ' + err) + } + } + + try { + return { + buffer: data + } + } catch (err) { + throw Boom.badRequest('Failed to create DAG node: ' + err) + } + }, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + async handler (request, h) { + const { ipfs } = request.server.app + const { buffer } = request.pre.args + + let cid + + return new Promise((resolve, reject) => { + const format = ipfs._ipld.resolvers[request.query.format] + + if (!format) { + return reject(Boom.badRequest(`Missing IPLD format "${request.query.format}"`)) + } + + format.util.deserialize(buffer, async (err, node) => { + if (err) { + return reject(err) + } + + try { + cid = await ipfs.dag.put(node, { + format: request.query.format, + hashAlg: request.query.hash + }) + } catch (err) { + throw Boom.boomify(err, { message: 'Failed to put node' }) + } + + if (request.query.pin) { + await ipfs.pin.add(cid) + } + + resolve(h.response({ + Cid: { + '/': cid.toBaseEncodedString(request.query.cidbase) + } + })) + }) + }) + } +} + +exports.resolve = { + validate: { + query: Joi.object().keys({ + 'cid-base': Joi.string().valid(multibase.names) + }).unknown() + }, + + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + async handler (request, h) { + let { key, path } = request.pre.args + const cidBase = request.query['cid-base'] + const { ipfs } = request.server.app + + // to be consistent with go we need to return the CID to the last node we've traversed + // along with the path inside that node as the remainder path + try { + let lastCid = key + let lastRemainderPath = path + + while (true) { + const block = await ipfs.block.get(lastCid) + const codec = ipfs._ipld.resolvers[lastCid.codec] + + if (!codec) { + throw Boom.badRequest(`Missing IPLD format "${lastCid.codec}"`) + } + + const resolve = promisify(codec.resolver.resolve) + const res = await resolve(block.data, lastRemainderPath) + + if (!res.remainderPath) { + break + } + + lastRemainderPath = res.remainderPath + + if (!CID.isCID(res.value)) { + break + } + + lastCid = res.value + } + + return h.response({ + Cid: { + '/': lastCid.toBaseEncodedString(cidBase) + }, + RemPath: lastRemainderPath + }) + } catch (err) { + throw Boom.boomify(err) + } + } +} diff --git a/src/http/api/resources/index.js b/src/http/api/resources/index.js index a54aa0d4f3..3f32cccdf5 100644 --- a/src/http/api/resources/index.js +++ b/src/http/api/resources/index.js @@ -15,6 +15,7 @@ exports.bitswap = require('./bitswap') exports.file = require('./file') exports.filesRegular = require('./files-regular') exports.pubsub = require('./pubsub') +exports.dag = require('./dag') exports.dns = require('./dns') exports.key = require('./key') exports.stats = require('./stats') diff --git a/src/http/api/routes/dag.js b/src/http/api/routes/dag.js new file mode 100644 index 0000000000..dd3829b60f --- /dev/null +++ b/src/http/api/routes/dag.js @@ -0,0 +1,43 @@ +'use strict' + +const resources = require('../resources') + +module.exports = [ + { + method: 'POST', + path: '/api/v0/dag/get', + options: { + pre: [ + { method: resources.dag.get.parseArgs, assign: 'args' } + ], + validate: resources.dag.get.validate + }, + handler: resources.dag.get.handler + }, + { + method: 'POST', + path: '/api/v0/dag/put', + options: { + payload: { + parse: false, + output: 'stream' + }, + pre: [ + { method: resources.dag.put.parseArgs, assign: 'args' } + ], + validate: resources.dag.put.validate + }, + handler: resources.dag.put.handler + }, + { + method: 'POST', + path: '/api/v0/dag/resolve', + options: { + pre: [ + { method: resources.dag.resolve.parseArgs, assign: 'args' } + ], + validate: resources.dag.resolve.validate + }, + handler: resources.dag.resolve.handler + } +] diff --git a/src/http/api/routes/index.js b/src/http/api/routes/index.js index 0f459aaf8b..48fcfff3f8 100644 --- a/src/http/api/routes/index.js +++ b/src/http/api/routes/index.js @@ -19,6 +19,7 @@ module.exports = [ ...require('./pubsub'), require('./debug'), ...require('./webui'), + ...require('./dag'), require('./dns'), ...require('./key'), ...require('./stats'), diff --git a/test/http-api/dag.js b/test/http-api/dag.js new file mode 100644 index 0000000000..9217c3185a --- /dev/null +++ b/test/http-api/dag.js @@ -0,0 +1,111 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const path = require('path') +const expect = chai.expect +chai.use(dirtyChai) +const DaemonFactory = require('ipfsd-ctl') +const df = DaemonFactory.create({ + exec: path.resolve(__dirname, '..', '..', 'src', 'cli', 'bin.js') +}) + +describe('dag endpoint', () => { + let ipfs = null + let ipfsd = null + + before(function (done) { + this.timeout(20 * 1000) + + df.spawn({ + initOptions: { + bits: 1024 + }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } + }, (err, _ipfsd) => { + if (err) { + console.error(err) + } + + expect(err).to.not.exist() + ipfsd = _ipfsd + ipfs = ipfsd.api + done() + }) + }) + + after((done) => ipfsd.stop(done)) + + it('returns error for request without argument', (done) => { + ipfs.dag.get(null, (err, result) => { + expect(err.message).to.include("invalid 'ipfs ref' path") + done() + }) + }) + + it('returns error for request with invalid argument', (done) => { + ipfs.dag.get('invalid', { enc: 'base58' }, (err, result) => { + expect(err.message).to.include("invalid 'ipfs ref' path") + done() + }) + }) + + it('returns value', (done) => { + ipfs.dag.get('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n', (err, result) => { + expect(err).to.not.exist() + expect(result.value).to.be.ok() + expect(result.value.links).to.be.empty() + expect(result.value.data).to.be.empty() + + done() + }) + }) + + it('returns value with a path as part of the cid', (done) => { + ipfs.dag.put({ + foo: 'bar' + }, { + format: 'dag-cbor', + hash: 'sha2-256' + }, (err, cid) => { + expect(err).to.not.exist() + + ipfs.dag.get(`${cid.toBaseEncodedString()}/foo`, (err, result) => { + expect(err).to.not.exist() + expect(result.value).to.equal('bar') + + done() + }) + }) + }) + + it('returns value with a path as a separate argument', (done) => { + ipfs.dag.put({ + foo: 'bar' + }, { + format: 'dag-cbor', + hash: 'sha2-256' + }, (err, cid) => { + expect(err).to.not.exist() + + ipfs.dag.get(cid, 'foo', (err, result) => { + expect(err).to.not.exist() + expect(result.value).to.equal('bar') + + done() + }) + }) + }) +}) diff --git a/test/http-api/index.js b/test/http-api/index.js index a4cceb57a9..43079247f3 100644 --- a/test/http-api/index.js +++ b/test/http-api/index.js @@ -3,6 +3,7 @@ require('./block') require('./bootstrap') require('./config') +require('./dag') require('./dns') require('./id') require('./routes') diff --git a/test/http-api/interface.js b/test/http-api/interface.js index 9529fdece2..7169a6acc0 100644 --- a/test/http-api/interface.js +++ b/test/http-api/interface.js @@ -17,9 +17,7 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { tests.config(defaultCommonFactory) - tests.dag(defaultCommonFactory, { - skip: { reason: 'TODO: DAG HTTP endpoints not implemented in js-ipfs yet!' } - }) + tests.dag(defaultCommonFactory) tests.dht(CommonFactory.create({ spawnOptions: { From 48b79dd932a0fadf8765243f326741dc16a14ba1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 15 Mar 2019 12:30:35 +0000 Subject: [PATCH 2/5] chore: update pr after review --- src/http/api/resources/dag.js | 216 +++++++++++-------- test/http-api/dag.js | 111 ---------- test/http-api/index.js | 1 - test/http-api/inject/dag.js | 395 ++++++++++++++++++++++++++++++++++ 4 files changed, 525 insertions(+), 198 deletions(-) delete mode 100644 test/http-api/dag.js create mode 100644 test/http-api/inject/dag.js diff --git a/src/http/api/resources/dag.js b/src/http/api/resources/dag.js index 9a9c71b399..3271a195c3 100644 --- a/src/http/api/resources/dag.js +++ b/src/http/api/resources/dag.js @@ -3,58 +3,89 @@ const promisify = require('promisify-es6') const CID = require('cids') const multipart = require('ipfs-multipart') +const mh = require('multihashes') const Joi = require('joi') const multibase = require('multibase') const Boom = require('boom') const debug = require('debug') +const { + cidToString +} = require('../../../utils/cid') const log = debug('ipfs:http-api:dag') log.error = debug('ipfs:http-api:dag:error') // common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` -exports.parseKey = (request, h) => { - if (!request.query.arg) { - throw Boom.badRequest("Argument 'key' is required") - } +exports.parseKey = (argument = 'Argument', name = 'key', quote = "'") => { + return (request) => { + if (!request.query.arg) { + // for compatibility with go error messages + throw Boom.badRequest(`${argument} ${quote}${name}${quote} is required`) + } - let key = request.query.arg.trim() - let path + let key = request.query.arg.trim() + let path - if (key.startsWith('/ipfs')) { - key = key.substring(5) - } + if (key.startsWith('/ipfs')) { + key = key.substring(5) + } - const parts = key.split('/') + const parts = key.split('/') + + if (parts.length > 1) { + key = parts.shift() + path = `${parts.join('/')}` + } + + if (path && path.endsWith('/')) { + path = path.substring(0, path.length - 1) + } - if (parts.length > 1) { - key = parts.shift() - path = `${parts.join('/')}` + try { + return { + [name]: new CID(key), + path + } + } catch (err) { + log.error(err) + throw Boom.badRequest("invalid 'ipfs ref' path") + } } +} - if (path.endsWith('/')) { - path = path.substring(0, path.length - 1) +const encodeBufferKeys = (obj, encoding) => { + if (!obj) { + return obj } - try { - return { - key: new CID(key), - path - } - } catch (err) { - log.error(err) - throw Boom.badRequest("invalid 'ipfs ref' path") + if (Buffer.isBuffer(obj)) { + return obj.toString(encoding) } + + Object.keys(obj).forEach(key => { + if (Buffer.isBuffer(obj)) { + obj[key] = obj[key].toString(encoding) + + return + } + + if (typeof obj[key] === 'object') { + obj[key] = encodeBufferKeys(obj[key], encoding) + } + }) + + return obj } exports.get = { validate: { query: Joi.object().keys({ - 'data-encoding': Joi.string().valid(['text', 'base64']).default('base64'), + 'data-encoding': Joi.string().valid(['text', 'base64', 'hex']).default('text'), 'cid-base': Joi.string().valid(multibase.names) }).unknown() }, // uses common parseKey method that returns a `key` - parseArgs: exports.parseKey, + parseArgs: exports.parseKey(), // main route handler which is called after the above `parseArgs`, but only if the args were valid async handler (request, h) { @@ -64,22 +95,24 @@ exports.get = { } = request.pre.args const { ipfs } = request.server.app + let dataEncoding = request.query['data-encoding'] + + if (dataEncoding === 'text') { + dataEncoding = 'utf8' + } + let result try { result = await ipfs.dag.get(key, path) } catch (err) { - throw Boom.boomify(err, { message: 'Failed to get dag node' }) + throw Boom.badRequest(err) } - if (key.codec === 'dag-pb' && result.value) { - if (typeof result.value.toJSON === 'function') { - result.value = result.value.toJSON() - } - - if (Buffer.isBuffer(result.value.data)) { - result.value.data = result.value.data.toString(request.query.dataencoding) - } + try { + result.value = encodeBufferKeys(result.value, dataEncoding) + } catch (err) { + throw Boom.boomify(err) } return h.response(result.value) @@ -89,11 +122,10 @@ exports.get = { exports.put = { validate: { query: Joi.object().keys({ - // TODO validate format, & hash - format: Joi.string(), - 'input-enc': Joi.string().valid('dag-cbor', 'dag-pb', 'raw'), + format: Joi.string().default('cbor'), + 'input-enc': Joi.string().default('json'), pin: Joi.boolean(), - hash: Joi.string(), + hash: Joi.string().valid(mh.names).default('sha2-256'), 'cid-base': Joi.string().valid(multibase.names).default('base58btc') }).unknown() }, @@ -102,78 +134,89 @@ exports.put = { // which is assigned to `request.pre.args` async parseArgs (request, h) { if (!request.payload) { - throw Boom.badRequest("File argument 'data' is required") + throw Boom.badRequest("File argument 'object data' is required") } - const enc = request.query.inputenc + const enc = request.query['input-enc'] + + if (!request.headers['content-type']) { + throw Boom.badRequest("File argument 'object data' is required") + } const fileStream = await new Promise((resolve, reject) => { multipart.reqParser(request.payload) .on('file', (name, stream) => resolve(stream)) - .on('end', () => reject(Boom.badRequest("File argument 'data' is required"))) + .on('end', () => reject(Boom.badRequest("File argument 'object data' is required"))) }) let data = await new Promise((resolve, reject) => { fileStream .on('data', data => resolve(data)) - .on('end', () => reject(Boom.badRequest("File argument 'data' is required"))) + .on('end', () => reject(Boom.badRequest("File argument 'object data' is required"))) }) - if (enc === 'json') { + let format = request.query.format + + if (format === 'cbor') { + format = 'dag-cbor' + } + + let node + + if (format === 'raw') { + node = data + } else if (enc === 'json') { try { - data = JSON.parse(data.toString()) + node = JSON.parse(data.toString()) } catch (err) { throw Boom.badRequest('Failed to parse the JSON: ' + err) } - } + } else { + const { ipfs } = request.server.app + const codec = ipfs._ipld.resolvers[format] - try { - return { - buffer: data + if (!codec) { + throw Boom.badRequest(`Missing IPLD format "${request.query.format}"`) } - } catch (err) { - throw Boom.badRequest('Failed to create DAG node: ' + err) + + const deserialize = promisify(codec.util.deserialize) + + node = await deserialize(data) + } + + return { + node, + format, + hashAlg: request.query.hash } }, // main route handler which is called after the above `parseArgs`, but only if the args were valid async handler (request, h) { const { ipfs } = request.server.app - const { buffer } = request.pre.args + const { node, format, hashAlg } = request.pre.args let cid - return new Promise((resolve, reject) => { - const format = ipfs._ipld.resolvers[request.query.format] - - if (!format) { - return reject(Boom.badRequest(`Missing IPLD format "${request.query.format}"`)) - } - - format.util.deserialize(buffer, async (err, node) => { - if (err) { - return reject(err) - } - - try { - cid = await ipfs.dag.put(node, { - format: request.query.format, - hashAlg: request.query.hash - }) - } catch (err) { - throw Boom.boomify(err, { message: 'Failed to put node' }) - } + try { + cid = await ipfs.dag.put(node, { + format: format, + hashAlg: hashAlg + }) + } catch (err) { + throw Boom.boomify(err, { message: 'Failed to put node' }) + } - if (request.query.pin) { - await ipfs.pin.add(cid) - } + if (request.query.pin) { + await ipfs.pin.add(cid) + } - resolve(h.response({ - Cid: { - '/': cid.toBaseEncodedString(request.query.cidbase) - } - })) - }) + return h.response({ + Cid: { + '/': cidToString(cid, { + base: request.query['cid-base'] + }) + } }) } } @@ -186,18 +229,17 @@ exports.resolve = { }, // uses common parseKey method that returns a `key` - parseArgs: exports.parseKey, + parseArgs: exports.parseKey('argument', 'ref', '"'), // main route handler which is called after the above `parseArgs`, but only if the args were valid async handler (request, h) { - let { key, path } = request.pre.args - const cidBase = request.query['cid-base'] + let { ref, path } = request.pre.args const { ipfs } = request.server.app // to be consistent with go we need to return the CID to the last node we've traversed // along with the path inside that node as the remainder path try { - let lastCid = key + let lastCid = ref let lastRemainderPath = path while (true) { @@ -226,9 +268,11 @@ exports.resolve = { return h.response({ Cid: { - '/': lastCid.toBaseEncodedString(cidBase) + '/': cidToString(lastCid, { + base: request.query['cid-base'] + }) }, - RemPath: lastRemainderPath + RemPath: lastRemainderPath || '' }) } catch (err) { throw Boom.boomify(err) diff --git a/test/http-api/dag.js b/test/http-api/dag.js deleted file mode 100644 index 9217c3185a..0000000000 --- a/test/http-api/dag.js +++ /dev/null @@ -1,111 +0,0 @@ -/* eslint max-nested-callbacks: ["error", 8] */ -/* eslint-env mocha */ -'use strict' - -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const path = require('path') -const expect = chai.expect -chai.use(dirtyChai) -const DaemonFactory = require('ipfsd-ctl') -const df = DaemonFactory.create({ - exec: path.resolve(__dirname, '..', '..', 'src', 'cli', 'bin.js') -}) - -describe('dag endpoint', () => { - let ipfs = null - let ipfsd = null - - before(function (done) { - this.timeout(20 * 1000) - - df.spawn({ - initOptions: { - bits: 1024 - }, - config: { - Bootstrap: [], - Discovery: { - MDNS: { - Enabled: false - }, - webRTCStar: { - Enabled: false - } - } - } - }, (err, _ipfsd) => { - if (err) { - console.error(err) - } - - expect(err).to.not.exist() - ipfsd = _ipfsd - ipfs = ipfsd.api - done() - }) - }) - - after((done) => ipfsd.stop(done)) - - it('returns error for request without argument', (done) => { - ipfs.dag.get(null, (err, result) => { - expect(err.message).to.include("invalid 'ipfs ref' path") - done() - }) - }) - - it('returns error for request with invalid argument', (done) => { - ipfs.dag.get('invalid', { enc: 'base58' }, (err, result) => { - expect(err.message).to.include("invalid 'ipfs ref' path") - done() - }) - }) - - it('returns value', (done) => { - ipfs.dag.get('QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.be.ok() - expect(result.value.links).to.be.empty() - expect(result.value.data).to.be.empty() - - done() - }) - }) - - it('returns value with a path as part of the cid', (done) => { - ipfs.dag.put({ - foo: 'bar' - }, { - format: 'dag-cbor', - hash: 'sha2-256' - }, (err, cid) => { - expect(err).to.not.exist() - - ipfs.dag.get(`${cid.toBaseEncodedString()}/foo`, (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.equal('bar') - - done() - }) - }) - }) - - it('returns value with a path as a separate argument', (done) => { - ipfs.dag.put({ - foo: 'bar' - }, { - format: 'dag-cbor', - hash: 'sha2-256' - }, (err, cid) => { - expect(err).to.not.exist() - - ipfs.dag.get(cid, 'foo', (err, result) => { - expect(err).to.not.exist() - expect(result.value).to.equal('bar') - - done() - }) - }) - }) -}) diff --git a/test/http-api/index.js b/test/http-api/index.js index 43079247f3..a4cceb57a9 100644 --- a/test/http-api/index.js +++ b/test/http-api/index.js @@ -3,7 +3,6 @@ require('./block') require('./bootstrap') require('./config') -require('./dag') require('./dns') require('./id') require('./routes') diff --git a/test/http-api/inject/dag.js b/test/http-api/inject/dag.js new file mode 100644 index 0000000000..a95aa0b2f8 --- /dev/null +++ b/test/http-api/inject/dag.js @@ -0,0 +1,395 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const promisify = require('promisify-es6') +const DAGNode = require('ipld-dag-pb').DAGNode +const createDAGPBNode = promisify(DAGNode.create) +const Readable = require('stream').Readable +const FormData = require('form-data') +const streamToPromise = require('stream-to-promise') +const CID = require('cids') + +const toHeadersAndPayload = async (thing) => { + const stream = new Readable() + stream.push(thing) + stream.push(null) + + const form = new FormData() + form.append('file', stream) + + return { + headers: form.getHeaders(), + payload: await streamToPromise(form) + } +} + +module.exports = (http) => { + describe('dag endpoint', () => { + let api + + before(() => { + api = http.api._apiServers[0] + }) + + describe('/dag/get', () => { + it('returns error for request without argument', async () => { + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/get' + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Message).to.include("Argument 'key' is required") + }) + + it('returns error for request with invalid argument', async () => { + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/get?arg=5' + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Message).to.include("invalid 'ipfs ref' path") + }) + + it('returns value', async () => { + const node = await createDAGPBNode(Buffer.from([]), []) + const cid = await http.api._ipfs.dag.put(node, { + format: 'dag-pb', + hashAlg: 'sha2-256' + }) + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.be.ok() + expect(res.result.links).to.be.empty() + expect(res.result.data).to.be.empty() + }) + + it('uses text encoding for data by default', async () => { + const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), []) + const cid = await http.api._ipfs.dag.put(node, { + format: 'dag-pb', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.be.ok() + expect(res.result.links).to.be.empty() + expect(res.result.data).to.equal('\u0000\u0001\u0002\u0003') + }) + + it('overrides data encoding', async () => { + const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), []) + const cid = await http.api._ipfs.dag.put(node, { + format: 'dag-pb', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}&data-encoding=base64` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.be.ok() + expect(res.result.links).to.be.empty() + expect(res.result.data).to.equal('AAECAw==') + }) + + it('returns value with a path as part of the cid', async () => { + const cid = await http.api._ipfs.dag.put({ + foo: 'bar' + }, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}/foo` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.equal('bar') + }) + + it('returns value with a path as part of the cid for dag-pb nodes', async () => { + const node = await createDAGPBNode(Buffer.from([0, 1, 2, 3]), []) + const cid = await http.api._ipfs.dag.put(node, { + format: 'dag-pb', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}/Data&data-encoding=base64` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.equal('AAECAw==') + }) + + it('encodes buffers in arbitrary positions', async () => { + const cid = await http.api._ipfs.dag.put({ + foo: 'bar', + baz: { + qux: Buffer.from([0, 1, 2, 3]) + } + }, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}&data-encoding=base64` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result.baz.qux).to.equal('AAECAw==') + }) + + it('supports specifying buffer encoding', async () => { + const cid = await http.api._ipfs.dag.put({ + foo: 'bar', + baz: Buffer.from([0, 1, 2, 3]) + }, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/get?arg=${cid.toBaseEncodedString()}&data-encoding=hex` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result.baz).to.equal('00010203') + }) + }) + + describe('/dag/put', () => { + it('returns error for request without file argument', async () => { + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/put' + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Message).to.include("File argument 'object data' is required") + }) + + it('adds a dag-cbor node by default', async () => { + const node = { + foo: 'bar' + } + + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/put', + ...await toHeadersAndPayload(JSON.stringify(node)) + }) + + expect(res.statusCode).to.equal(200) + + const cid = new CID(res.result.Cid['/']) + + expect(cid.codec).to.equal('dag-cbor') + + const added = await http.api._ipfs.dag.get(cid) + + expect(added.value).to.deep.equal(node) + }) + + it('adds a dag-pb node', async () => { + const node = { + data: [], + links: [] + } + + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/put?format=dag-pb', + ...await toHeadersAndPayload(JSON.stringify(node)) + }) + + expect(res.statusCode).to.equal(200) + + const cid = new CID(res.result.Cid['/']) + + expect(cid.codec).to.equal('dag-pb') + + const added = await http.api._ipfs.dag.get(cid) + + expect(added.value.data).to.be.empty() + expect(added.value.links).to.be.empty() + }) + + it('adds a raw node', async () => { + const node = Buffer.from([0, 1, 2, 3]) + + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/put?format=raw', + ...await toHeadersAndPayload(node) + }) + + expect(res.statusCode).to.equal(200) + + const cid = new CID(res.result.Cid['/']) + + expect(cid.codec).to.equal('raw') + + const added = await http.api._ipfs.dag.get(cid) + + expect(added.value).to.deep.equal(node) + }) + + it('pins a node after adding', async () => { + const node = { + foo: 'bar' + } + + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/put?pin=true', + ...await toHeadersAndPayload(JSON.stringify(node)) + }) + + expect(res.statusCode).to.equal(200) + + const cid = new CID(res.result.Cid['/']) + const pinset = await http.api._ipfs.pin.ls() + + expect(pinset.map(pin => pin.hash)).to.contain(cid.toBaseEncodedString('base58btc')) + }) + }) + + describe('/dag/resolve', () => { + it('returns error for request without argument', async () => { + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/resolve' + }) + + expect(res.statusCode).to.equal(400) + expect(res.result.Message).to.include('argument "ref" is required') + }) + + it('resolves a node', async () => { + const node = { + foo: 'bar' + } + const cid = await http.api._ipfs.dag.put(node, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/resolve?arg=${cid.toBaseEncodedString()}` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.be.ok() + + const returnedCid = new CID(res.result.Cid['/']) + const returnedRemainerPath = res.result.RemPath + + expect(returnedCid).to.deep.equal(cid) + expect(returnedRemainerPath).to.be.empty() + }) + + it('returns the remainder path from within the resolved node', async () => { + const node = { + foo: 'bar' + } + const cid = await http.api._ipfs.dag.put(node, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/resolve?arg=${cid.toBaseEncodedString()}/foo` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.be.ok() + + const returnedCid = new CID(res.result.Cid['/']) + const returnedRemainerPath = res.result.RemPath + + expect(returnedCid).to.deep.equal(cid) + expect(returnedRemainerPath).to.equal('foo') + }) + + it('returns an error when the path is not available', async () => { + const node = { + foo: 'bar' + } + const cid = await http.api._ipfs.dag.put(node, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/resolve?arg=${cid.toBaseEncodedString()}/bar` + }) + + expect(res.statusCode).to.equal(500) + expect(res.result).to.be.ok() + }) + + it('resolves across multiple nodes, returning the CID of the last node traversed', async () => { + const node2 = { + bar: 'baz' + } + const cid2 = await http.api._ipfs.dag.put(node2, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const node1 = { + foo: { + '/': cid2 + } + } + + const cid1 = await http.api._ipfs.dag.put(node1, { + format: 'dag-cbor', + hashAlg: 'sha2-256' + }) + + const res = await api.inject({ + method: 'POST', + url: `/api/v0/dag/resolve?arg=${cid1.toBaseEncodedString()}/foo/bar` + }) + + expect(res.statusCode).to.equal(200) + expect(res.result).to.be.ok() + + const returnedCid = new CID(res.result.Cid['/']) + const returnedRemainerPath = res.result.RemPath + + expect(returnedCid).to.deep.equal(cid2) + expect(returnedRemainerPath).to.equal('bar') + }) + }) + }) +} From 934a5b2cd135345aaed674d7c1b689eabe1c47b3 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 15 Mar 2019 13:45:32 +0000 Subject: [PATCH 3/5] test: add test to make sure we do not pin when sending pin=false --- test/http-api/inject/dag.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/http-api/inject/dag.js b/test/http-api/inject/dag.js index a95aa0b2f8..2a70b669ba 100644 --- a/test/http-api/inject/dag.js +++ b/test/http-api/inject/dag.js @@ -277,6 +277,25 @@ module.exports = (http) => { expect(pinset.map(pin => pin.hash)).to.contain(cid.toBaseEncodedString('base58btc')) }) + + it('does not pin a node after adding', async () => { + const node = { + foo: 'bar' + } + + const res = await api.inject({ + method: 'POST', + url: '/api/v0/dag/put?pin=false', + ...await toHeadersAndPayload(JSON.stringify(node)) + }) + + expect(res.statusCode).to.equal(200) + + const cid = new CID(res.result.Cid['/']) + const pinset = await http.api._ipfs.pin.ls() + + expect(pinset.map(pin => pin.hash)).to.not.contain(cid.toBaseEncodedString('base58btc')) + }) }) describe('/dag/resolve', () => { From dadc065dd05327f28ab8b3fb1d299e7cfae6577c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Fri, 15 Mar 2019 16:45:26 +0000 Subject: [PATCH 4/5] chore: disambiguate pin nodes so they do not fight --- test/http-api/inject/dag.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/http-api/inject/dag.js b/test/http-api/inject/dag.js index 2a70b669ba..62eca02574 100644 --- a/test/http-api/inject/dag.js +++ b/test/http-api/inject/dag.js @@ -261,7 +261,8 @@ module.exports = (http) => { it('pins a node after adding', async () => { const node = { - foo: 'bar' + foo: 'bar', + disambiguator: Math.random() } const res = await api.inject({ @@ -280,7 +281,8 @@ module.exports = (http) => { it('does not pin a node after adding', async () => { const node = { - foo: 'bar' + foo: 'bar', + disambiguator: Math.random() } const res = await api.inject({ From ec3b44ff06f70b4af81afa30e58c206a6e28d59f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Mon, 18 Mar 2019 21:36:15 +0000 Subject: [PATCH 5/5] fix: valid multihash names in querystring License: MIT Signed-off-by: Alan Shaw --- src/http/api/resources/dag.js | 2 +- test/http-api/interface.js | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/http/api/resources/dag.js b/src/http/api/resources/dag.js index 3271a195c3..325588f646 100644 --- a/src/http/api/resources/dag.js +++ b/src/http/api/resources/dag.js @@ -125,7 +125,7 @@ exports.put = { format: Joi.string().default('cbor'), 'input-enc': Joi.string().default('json'), pin: Joi.boolean(), - hash: Joi.string().valid(mh.names).default('sha2-256'), + hash: Joi.string().valid(Object.keys(mh.names)).default('sha2-256'), 'cid-base': Joi.string().valid(multibase.names).default('base58btc') }).unknown() }, diff --git a/test/http-api/interface.js b/test/http-api/interface.js index 7169a6acc0..3ff76c65c8 100644 --- a/test/http-api/interface.js +++ b/test/http-api/interface.js @@ -17,7 +17,15 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { tests.config(defaultCommonFactory) - tests.dag(defaultCommonFactory) + tests.dag(defaultCommonFactory, { + skip: [{ + name: 'should get only a CID, due to resolving locally only', + reason: 'Local resolve option is not implemented yet' + }, { + name: 'tree', + reason: 'dag.tree is not implemented yet' + }] + }) tests.dht(CommonFactory.create({ spawnOptions: {