diff --git a/package.json b/package.json index 627c77334e..ca4502bd2f 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "lodash.get": "^4.3.0", "lodash.set": "^4.2.0", "multiaddr": "^2.0.0", + "multihashes": "^0.2.2", "path-exists": "^3.0.0", "peer-book": "^0.1.1", "peer-id": "^0.6.6", @@ -122,4 +123,4 @@ "kumavis ", "nginnever " ] -} \ No newline at end of file +} diff --git a/src/http-api/index.js b/src/http-api/index.js index d49582bb97..b9b5c98b9a 100644 --- a/src/http-api/index.js +++ b/src/http-api/index.js @@ -19,7 +19,6 @@ exports = module.exports = function HttpApi (repo) { this.start = (callback) => { if (typeof repo === 'string') { - console.log('here') repo = new IPFSRepo(repo, {stores: fsbs}) } diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 8647a8e314..610b0e7b99 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -1,6 +1,9 @@ 'use strict' const bs58 = require('bs58') +const multihash = require('multihashes') +const ndjson = require('ndjson') +const multipart = require('ipfs-multipart') const debug = require('debug') const log = debug('http-api:files') log.error = debug('http-api:files:error') @@ -48,3 +51,60 @@ exports.cat = { }) } } + +exports.add = { + handler: (request, reply) => { + if (!request.payload) { + return reply('Array, Buffer, or String is required.').code(400).takeover() + } + + const parser = multipart.reqParser(request.payload) + + var filesParsed = false + var filesAdded = 0 + + var serialize = ndjson.serialize() + // hapi doesn't permit object streams: http://hapijs.com/api#replyerr-result + serialize._readableState.objectMode = false + + var fileAdder = request.server.app.ipfs.files.add() + + fileAdder.on('data', (file) => { + serialize.write({ + Name: file.path, + Hash: multihash.toB58String(file.multihash) + }) + filesAdded++ + }) + + fileAdder.on('end', () => { + if (filesAdded === 0 && filesParsed) { + return reply({ + Message: 'Failed to add files.', + Code: 0 + }).code(500) + } else { + serialize.end() + return reply(serialize) + .header('x-chunked-output', '1') + .header('content-type', 'application/json') + } + }) + + parser.on('file', (fileName, fileStream) => { + var filePair = { + path: fileName, + stream: fileStream + } + filesParsed = true + fileAdder.write(filePair) + }) + + parser.on('end', () => { + if (!filesParsed) { + return reply("File argument 'data' is required.").code(400).takeover() + } + fileAdder.end() + }) + } +} diff --git a/src/http-api/routes/files.js b/src/http-api/routes/files.js index 99c47741e3..ddf722f850 100644 --- a/src/http-api/routes/files.js +++ b/src/http-api/routes/files.js @@ -15,4 +15,16 @@ module.exports = (server) => { handler: resources.files.cat.handler } }) + + api.route({ + method: '*', + path: '/api/v0/add', + config: { + payload: { + parse: false, + output: 'stream' + }, + handler: resources.files.add.handler + } + }) } diff --git a/src/http-api/routes/index.js b/src/http-api/routes/index.js index 587f25de77..6f73019f1e 100644 --- a/src/http-api/routes/index.js +++ b/src/http-api/routes/index.js @@ -8,7 +8,7 @@ module.exports = (server) => { require('./object')(server) // require('./repo')(server) require('./config')(server) + require('./files')(server) require('./swarm')(server) require('./bitswap')(server) - require('./files')(server) } diff --git a/test/cli-tests/test-bitswap.js b/test/cli-tests/test-bitswap.js index baec2e6356..bde99fe902 100644 --- a/test/cli-tests/test-bitswap.js +++ b/test/cli-tests/test-bitswap.js @@ -12,7 +12,7 @@ const createTempNode = require('../utils/temp-node') const repoPath = require('./index').repoPath describe('bitswap', function () { - this.timeout(20000) + this.timeout(80000) const env = _.clone(process.env) env.IPFS_PATH = repoPath diff --git a/test/cli-tests/test-swarm.js b/test/cli-tests/test-swarm.js index 629cc381af..055bc6b83a 100644 --- a/test/cli-tests/test-swarm.js +++ b/test/cli-tests/test-swarm.js @@ -9,7 +9,7 @@ const repoPath = require('./index').repoPath const _ = require('lodash') describe('swarm', function () { - this.timeout(20000) + this.timeout(80000) const env = _.clone(process.env) env.IPFS_PATH = repoPath diff --git a/test/core-tests/test-swarm-node.js b/test/core-tests/test-swarm-node.js index 56ee09c800..384e41a318 100644 --- a/test/core-tests/test-swarm-node.js +++ b/test/core-tests/test-swarm-node.js @@ -6,7 +6,7 @@ const expect = require('chai').expect const createTempNode = require('../utils/temp-node') describe('swarm', function () { - this.timeout(20000) + this.timeout(80000) var ipfsA var ipfsB diff --git a/test/http-api-tests/test-bitswap.js b/test/http-api-tests/test-bitswap.js index 60785afbb8..765d0ac333 100644 --- a/test/http-api-tests/test-bitswap.js +++ b/test/http-api-tests/test-bitswap.js @@ -5,7 +5,7 @@ const expect = require('chai').expect module.exports = (httpAPI) => { describe('bitswap', function () { - this.timeout(20000) + this.timeout(80000) describe('commands', () => { let api before(() => { diff --git a/test/http-api-tests/test-files.js b/test/http-api-tests/test-files.js index c41dda2862..26efaba61b 100644 --- a/test/http-api-tests/test-files.js +++ b/test/http-api-tests/test-files.js @@ -4,6 +4,19 @@ const expect = require('chai').expect const APIctl = require('ipfs-api') +const fs = require('fs') +const FormData = require('form-data') +const streamToPromise = require('stream-to-promise') +const Readable = require('stream').Readable +const http = require('http') + +function singleFileServer (filename) { + return http.createServer(function (req, res) { + fs.createReadStream(filename).pipe(res) + }) +} + + module.exports = (httpAPI) => { describe('files', () => { describe('api', () => { @@ -83,4 +96,155 @@ module.exports = (httpAPI) => { }) }) }) + + describe('files', () => { + describe('api', () => { + let api + + it('api', () => { + api = httpAPI.server.select('API') + }) + + describe('/files/add', () => { + it('returns 400 if no tuple is provided', (done) => { + const form = new FormData() + const headers = form.getHeaders() + + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/add', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(400) + done() + }) + }) + }) + + it('adds a file', (done) => { + const form = new FormData() + const filePath = 'test/test-data/node.json' + form.append('file', fs.createReadStream(filePath)) + const headers = form.getHeaders() + + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/add', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(200) + var result = JSON.parse(res.result) + expect(result.Name).to.equal('node.json') + expect(result.Hash).to.equal('QmRRdjTN2PjyEPrW73GBxJNAZrstH5tCZzwHYFJpSTKkhe') + done() + }) + }) + }) + + it('adds multiple files', (done) => { + const form = new FormData() + const filePath = 'test/test-data/hello' + const filePath2 = 'test/test-data/otherconfig' + form.append('file', fs.createReadStream(filePath)) + form.append('file', fs.createReadStream(filePath2)) + const headers = form.getHeaders() + + streamToPromise(form).then((payload) => { + api.inject({ + method: 'POST', + url: '/api/v0/add', + headers: headers, + payload: payload + }, (res) => { + expect(res.statusCode).to.equal(200) + var results = res.result.split('\n').slice(0, -1).map(JSON.parse) + expect(results[0].Name).to.equal('hello') + expect(results[0].Hash).to.equal('QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o') + expect(results[1].Name).to.equal('otherconfig') + expect(results[1].Hash).to.equal('QmayedZNznnEbHtyfjeQvvt29opSLjYjLtLqwfwSWq28ds') + done() + }) + }) + }) + }) + }) + + describe('using js-ipfs-api', () => { + var ctl + + it('start IPFS API ctl', (done) => { + ctl = APIctl('/ip4/127.0.0.1/tcp/6001') + done() + }) + + describe('ipfs.add', () => { + it('adds two files under a chunk Size', (done) => { + const rs = new Readable() + const rs2 = new Readable() + var files = [] + const buffered = fs.readFileSync('test/test-data/hello') + const buffered2 = fs.readFileSync('test/test-data/otherconfig') + rs.push(buffered) + rs.push(null) + rs2.push(buffered2) + rs2.push(null) + const filePair = {path: 'hello', content: rs} + const filePair2 = {path: 'otherconfig', content: rs2} + files.push(filePair) + files.push(filePair2) + + ctl.add(files, (err, res) => { + expect(err).to.not.exist + expect(res[0].Name).to.equal('hello') + expect(res[0].Hash).to.equal('QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o') + expect(res[1].Name).to.equal('otherconfig') + expect(res[1].Hash).to.equal('QmayedZNznnEbHtyfjeQvvt29opSLjYjLtLqwfwSWq28ds') + done() + }) + }) + + it('adds a large file > a chunk', (done) => { + const rs = new Readable() + var files = [] + const buffered = fs.readFileSync('test/test-data/1.2MiB.txt') + rs.push(buffered) + rs.push(null) + const filePair = {path: '1.2MiB.txt', content: rs} + files.push(filePair) + + ctl.add(filePair, (err, res) => { + expect(err).to.not.exist + expect(res[0].Name).to.equal('1.2MiB.txt') + expect(res[0].Hash).to.equal('QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') + done() + }) + }) + + it('adds a buffer', (done) => { + const buffer = new Buffer('hello world') + ctl.add(buffer, (err, res) => { + expect(err).to.not.exist + expect(res[0].Hash).to.equal('Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD') + done() + }) + }) + + it('adds a url', (done) => { + var server = singleFileServer('test/test-data/1.2MiB.txt') + server.listen(2913, function () { + ctl.add('http://localhost:2913/', (err, res) => { + expect(err).to.not.exist + const added = res[0] != null ? res[0] : res + expect(added).to.have.a.property('Hash', 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q') + done() + }) + }) + }) + }) + }) + }) } diff --git a/test/http-api-tests/test-swarm.js b/test/http-api-tests/test-swarm.js index d3d3cd281f..26f11a5171 100644 --- a/test/http-api-tests/test-swarm.js +++ b/test/http-api-tests/test-swarm.js @@ -7,7 +7,7 @@ const createTempNode = require('../utils/temp-node') module.exports = (httpAPI) => { describe('swarm', function () { - this.timeout(20000) + this.timeout(80000) describe('api', () => { var api diff --git a/test/test-data/1.2MiB.txt b/test/test-data/1.2MiB.txt new file mode 100644 index 0000000000..6e306c55ab Binary files /dev/null and b/test/test-data/1.2MiB.txt differ