From 51aa43f7bf0ea2439ae60c8711949f4e5d79b392 Mon Sep 17 00:00:00 2001 From: Yahya Date: Fri, 25 Aug 2017 01:40:26 +0200 Subject: [PATCH 1/4] Gateway /ipfs/* route, based on harshjv implementation License: MIT Signed-off-by: Yahya --- package.json | 3 + src/http-api/gateway/resolver.js | 155 ++++++++++++++++++++----------- src/http-api/resources/files.js | 60 +++++++++--- 3 files changed, 155 insertions(+), 63 deletions(-) diff --git a/package.json b/package.json index f714d4f6ce..cf6825f85c 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,8 @@ "boom": "^5.2.0", "cids": "^0.5.1", "debug": "^3.0.0", + "file-type": "^6.1.0", + "filesize": "^3.5.10", "fsm-event": "^2.1.0", "glob": "^7.1.2", "hapi": "^16.5.2", @@ -128,6 +130,7 @@ "lodash.values": "^4.3.0", "mime-types": "^2.1.13", "mafmt": "^2.1.8", + "mime-types": "^2.1.16", "mkdirp": "^0.5.1", "multiaddr": "^2.3.0", "multihashes": "~0.4.5", diff --git a/src/http-api/gateway/resolver.js b/src/http-api/gateway/resolver.js index 9c5ab7e208..c164e8c9f9 100644 --- a/src/http-api/gateway/resolver.js +++ b/src/http-api/gateway/resolver.js @@ -1,7 +1,9 @@ 'use strict' const mh = require('multihashes') -const pf = require('promised-for') +// const pf = require('promised-for') +const promisify = require('promisify-es6') +const eachOf = require('async/eachOf') const html = require('./utils/html') const PathUtil = require('./utils/path') @@ -25,60 +27,109 @@ const resolveDirectory = (ipfs, path, multihash) => { }) } -const resolveMultihash = (ipfs, path) => { +const noop = function () {} + +const resolveMultihash = promisify((ipfs, path, callback) => { + if (!callback) { + callback = noop + } + const parts = PathUtil.splitPath(path) const partsLength = parts.length - return pf( - { - multihash: parts[0], - index: 0 - }, - (i) => i.index < partsLength, - (i) => { - const currentIndex = i.index - const currentMultihash = i.multihash - - // throws error when invalid multihash is passed - mh.validate(mh.fromB58String(currentMultihash)) - - return ipfs - .object - .get(currentMultihash, { enc: 'base58' }) - .then((DAGNode) => { - if (currentIndex === partsLength - 1) { - // leaf node - return { - multihash: currentMultihash, - index: currentIndex + 1 - } - } else { - // find multihash of requested named-file - // in current DAGNode's links - let multihashOfNextFile - const nextFileName = parts[currentIndex + 1] - const links = DAGNode.links - - for (let link of links) { - if (link.name === nextFileName) { - // found multihash of requested named-file - multihashOfNextFile = mh.toB58String(link.multihash) - break - } - } - - if (!multihashOfNextFile) { - throw new Error(`no link named "${nextFileName}" under ${currentMultihash}`) - } - - return { - multihash: multihashOfNextFile, - index: currentIndex + 1 - } - } - }) - }) -} + let currentMultihash = parts[0] + + eachOf(parts, (multihash, currentIndex, next) => { + // throws error when invalid multihash is passed + mh.validate(mh.fromB58String(currentMultihash)) + + ipfs + .object + .get(currentMultihash, { enc: 'base58' }) + .then((DAGNode) => { + if (currentIndex === partsLength - 1) { + // leaf node + console.log('leaf node: ', currentMultihash) + next() + } else { + // find multihash of requested named-file + // in current DAGNode's links + let multihashOfNextFile + const nextFileName = parts[currentIndex + 1] + const links = DAGNode.links + + for (let link of links) { + if (link.name === nextFileName) { + // found multihash of requested named-file + multihashOfNextFile = mh.toB58String(link.multihash) + console.log('found multihash: ', multihashOfNextFile) + break + } + } + + if (!multihashOfNextFile) { + throw new Error(`no link named "${nextFileName}" under ${currentMultihash}`) + } + + currentMultihash = multihashOfNextFile + next() + } + }) + }, (err) => { + if (err) throw err + callback(null, {multihash: currentMultihash}) + }) + // Original implementation + // return pf( + // { + // multihash: parts[0], + // index: 0 + // }, + // (i) => i.index < partsLength, + // (i) => { + // const currentIndex = i.index + // const currentMultihash = i.multihash + // + // // throws error when invalid multihash is passed + // mh.validate(mh.fromB58String(currentMultihash)) + // + // return ipfs + // .object + // .get(currentMultihash, { enc: 'base58' }) + // .then((DAGNode) => { + // if (currentIndex === partsLength - 1) { + // // leaf node + // return { + // multihash: currentMultihash, + // index: currentIndex + 1 + // } + // } else { + // // find multihash of requested named-file + // // in current DAGNode's links + // let multihashOfNextFile + // const nextFileName = parts[currentIndex + 1] + // const links = DAGNode.links + // + // for (let link of links) { + // if (link.name === nextFileName) { + // // found multihash of requested named-file + // multihashOfNextFile = mh.toB58String(link.multihash) + // break + // } + // } + // + // if (!multihashOfNextFile) { + // throw new Error(`no link named "${nextFileName}" under ${currentMultihash}`) + // } + // + // return { + // multihash: multihashOfNextFile, + // index: currentIndex + 1 + // } + // } + // }) + // }) +}) module.exports = { resolveDirectory, diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 6644cad315..0ee9a4b31c 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -11,10 +11,11 @@ const toPull = require('stream-to-pull-stream') const pushable = require('pull-pushable') const EOL = require('os').EOL const toStream = require('pull-stream-to-stream') +const fileType = require('file-type') const mime = require('mime-types') - const GatewayResolver = require('../gateway/resolver') const PathUtils = require('../gateway/utils/path') +const Stream = require('stream') exports = module.exports @@ -245,21 +246,58 @@ exports.gateway = { .redirect(PathUtils.removeTrailingSlash(ref)) .permanent(true) } else { - const mimeType = mime.lookup(ref) - if (!stream._read) { stream._read = () => {} stream._readableState = {} } - if (mimeType) { - return reply(stream) - .header('Content-Type', mime.contentType(mimeType)) - .header('X-Stream-Output', '1') - } else { - return reply(stream) - .header('X-Stream-Output', '1') - } + let filetypeChecked = false + let stream2 = new Stream.PassThrough({highWaterMark: 1}) + let response = reply(stream2).hold() + + pull( + toPull.source(stream), + pull.drain((chunk) => { + if (chunk.length > 0 && !filetypeChecked) { + console.log('got first chunk') + let fileSignature = fileType(chunk) + console.log('file type: ', fileSignature) + + filetypeChecked = true + const mimeType = mime.lookup((fileSignature) ? fileSignature.ext : null) + console.log('ref ', ref) + console.log('mime-type ', mimeType) + + if (mimeType) { + console.log('writing mimeType') + + response + .header('Content-Type', mime.contentType(mimeType)) + .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .header('Access-Control-Allow-Methods', 'GET') + .header('Access-Control-Allow-Origin', '*') + .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .send() + } else { + response + .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .header('Access-Control-Allow-Methods', 'GET') + .header('Access-Control-Allow-Origin', '*') + .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .send() + } + + stream2.write(chunk) + } else { + // console.log('chunk length: ', chunk.length) + stream2.write(chunk) + } + }, (err) => { + if (err) throw err + console.log('stream ended.') + stream2.end() + }) + ) } }) .catch((err) => { From aed58846d2fc8beb5a6353de9534ee52d169b318 Mon Sep 17 00:00:00 2001 From: Yahya Date: Fri, 25 Aug 2017 22:02:55 +0200 Subject: [PATCH 2/4] gateway detecting directories and rendering file browser ui License: MIT Signed-off-by: Yahya --- src/http-api/gateway/resolver.js | 48 +++++++++++++++++++---------- src/http-api/resources/files.js | 53 ++++++++++++++++---------------- 2 files changed, 59 insertions(+), 42 deletions(-) diff --git a/src/http-api/gateway/resolver.js b/src/http-api/gateway/resolver.js index c164e8c9f9..f191419b43 100644 --- a/src/http-api/gateway/resolver.js +++ b/src/http-api/gateway/resolver.js @@ -10,22 +10,29 @@ const PathUtil = require('./utils/path') const INDEX_HTML_FILES = [ 'index.html', 'index.htm', 'index.shtml' ] -const resolveDirectory = (ipfs, path, multihash) => { - return ipfs - .object - .get(multihash, { enc: 'base58' }) - .then((DAGNode) => { - const links = DAGNode.links - const indexFiles = links.filter((link) => INDEX_HTML_FILES.indexOf(link.name) !== -1) - - // found index file in links - if (indexFiles.length > 0) { - return indexFiles - } +const resolveDirectory = promisify((ipfs, path, callback) => { + if (!callback) { + callback = noop + } - return html.build(path, links) - }) -} + const parts = PathUtil.splitPath(path) + const multihash = parts[0] + + ipfs + .object + .get(multihash, { enc: 'base58' }) + .then((DAGNode) => { + const links = DAGNode.links + const indexFiles = links.filter((link) => INDEX_HTML_FILES.indexOf(link.name) !== -1) + + // found index file in links + if (indexFiles.length > 0) { + return callback(null, indexFiles) + } + + return callback(null, html.build(path, links)) + }) +}) const noop = function () {} @@ -47,6 +54,13 @@ const resolveMultihash = promisify((ipfs, path, callback) => { .object .get(currentMultihash, { enc: 'base58' }) .then((DAGNode) => { + // console.log('DAGNode: ', DAGNode) + if (DAGNode.links && DAGNode.links.length > 0 && DAGNode.links[0].name.length > 0) { + // this is a directory. + // fire directory error here. + return next(new Error('This dag node is a directory')) + } + if (currentIndex === partsLength - 1) { // leaf node console.log('leaf node: ', currentMultihash) @@ -76,7 +90,9 @@ const resolveMultihash = promisify((ipfs, path, callback) => { } }) }, (err) => { - if (err) throw err + if (err) { + return callback(err) + } callback(null, {multihash: currentMultihash}) }) // Original implementation diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 0ee9a4b31c..51638015ca 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -301,38 +301,39 @@ exports.gateway = { } }) .catch((err) => { - if (err.toString() === 'Error: This dag node is a directory') { - return GatewayResolver - .resolveDirectory(ipfs, ref, data.multihash) - .then((data) => { - if (typeof data === 'string') { - // no index file found - if (!ref.endsWith('/')) { - // for a directory, if URL doesn't end with a / - // append / and redirect permanent to that URL - return reply.redirect(`${ref}/`).permanent(true) - } else { - // send directory listing - return reply(data) - } - } else { - // found index file - // redirect to URL/ - return reply.redirect(PathUtils.joinURLParts(ref, data[0].name)) - } - }).catch((err) => { - log.error(err) - return reply(err.toString()).code(500) - }) - } else { + if (err) { log.error(err) return reply(err.toString()).code(500) } }) }).catch((err) => { - const errorToString = err.toString() + console.log('err: ', err.toString()) - if (errorToString.startsWith('Error: no link named')) { + const errorToString = err.toString() + if (errorToString === 'Error: This dag node is a directory') { + return GatewayResolver + .resolveDirectory(ipfs, ref) + .then((data) => { + if (typeof data === 'string') { + // no index file found + if (!ref.endsWith('/')) { + // for a directory, if URL doesn't end with a / + // append / and redirect permanent to that URL + return reply.redirect(`${ref}/`).permanent(true) + } else { + // send directory listing + return reply(data) + } + } else { + // found index file + // redirect to URL/ + return reply.redirect(PathUtils.joinURLParts(ref, data[0].name)) + } + }).catch((err) => { + log.error(err) + return reply(err.toString()).code(500) + }) + } else if (errorToString.startsWith('Error: no link named')) { return reply(errorToString).code(404) } else if (errorToString.startsWith('Error: multihash length inconsistent') || errorToString.startsWith('Error: Non-base58 character')) { From dfe22353ce95c5831f123abb365a1a16ccb999d0 Mon Sep 17 00:00:00 2001 From: Yahya Date: Sat, 26 Aug 2017 00:17:18 +0200 Subject: [PATCH 3/4] gateway detecting directory fix and use eachOfSeries License: MIT Signed-off-by: Yahya --- src/http-api/gateway/resolver.js | 37 +++++++++++++++++++++----------- src/http-api/resources/files.js | 4 ++-- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/http-api/gateway/resolver.js b/src/http-api/gateway/resolver.js index f191419b43..672ac8f158 100644 --- a/src/http-api/gateway/resolver.js +++ b/src/http-api/gateway/resolver.js @@ -3,20 +3,19 @@ const mh = require('multihashes') // const pf = require('promised-for') const promisify = require('promisify-es6') -const eachOf = require('async/eachOf') +const eachOfSeries = require('async/eachOfSeries') const html = require('./utils/html') const PathUtil = require('./utils/path') const INDEX_HTML_FILES = [ 'index.html', 'index.htm', 'index.shtml' ] -const resolveDirectory = promisify((ipfs, path, callback) => { +const resolveDirectory = promisify((ipfs, path, multihash, callback) => { if (!callback) { callback = noop } - const parts = PathUtil.splitPath(path) - const multihash = parts[0] + mh.validate(mh.fromB58String(multihash)) ipfs .object @@ -46,24 +45,38 @@ const resolveMultihash = promisify((ipfs, path, callback) => { let currentMultihash = parts[0] - eachOf(parts, (multihash, currentIndex, next) => { + eachOfSeries(parts, (multihash, currentIndex, next) => { // throws error when invalid multihash is passed mh.validate(mh.fromB58String(currentMultihash)) + console.log('currentMultihash: ', currentMultihash) + console.log('currentIndex: ', currentIndex, '/', partsLength) ipfs .object .get(currentMultihash, { enc: 'base58' }) .then((DAGNode) => { - // console.log('DAGNode: ', DAGNode) - if (DAGNode.links && DAGNode.links.length > 0 && DAGNode.links[0].name.length > 0) { - // this is a directory. - // fire directory error here. - return next(new Error('This dag node is a directory')) - } - + // console.log('DAGNode: ', DAGNode) if (currentIndex === partsLength - 1) { // leaf node console.log('leaf node: ', currentMultihash) + console.log('DAGNode: ', DAGNode.links) + + if (DAGNode.links && + DAGNode.links.length > 0 && + DAGNode.links[0].name.length > 0) { + for (let link of DAGNode.links) { + if (mh.toB58String(link.multihash) === currentMultihash) { + return next() + } + } + + // this is a directory. + let isDirErr = new Error('This dag node is a directory') + // add currentMultihash as a fileName so it can be used by resolveDirectory + isDirErr.fileName = currentMultihash + return next(isDirErr) + } + next() } else { // find multihash of requested named-file diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js index 51638015ca..12576b60f4 100644 --- a/src/http-api/resources/files.js +++ b/src/http-api/resources/files.js @@ -307,12 +307,12 @@ exports.gateway = { } }) }).catch((err) => { - console.log('err: ', err.toString()) + console.log('err: ', err.toString(), ' fileName: ', err.fileName) const errorToString = err.toString() if (errorToString === 'Error: This dag node is a directory') { return GatewayResolver - .resolveDirectory(ipfs, ref) + .resolveDirectory(ipfs, ref, err.fileName) .then((data) => { if (typeof data === 'string') { // no index file found From bd4aa04fd3abb25d10ad4c70cdf9780399e6dcf7 Mon Sep 17 00:00:00 2001 From: Yahya Date: Mon, 28 Aug 2017 15:41:26 +0200 Subject: [PATCH 4/4] renaming http-api -> http, gateway spec tests License: MIT Signed-off-by: Yahya --- gulpfile.js | 2 +- package.json | 1 + src/cli/commands/daemon.js | 2 +- src/http-api/resources/files.js | 347 ------------------ .../api}/resources/bitswap.js | 0 src/{http-api => http/api}/resources/block.js | 0 .../api}/resources/bootstrap.js | 0 .../api}/resources/config.js | 0 src/http/api/resources/files.js | 345 +++++++++++++++++ src/{http-api => http/api}/resources/id.js | 0 src/{http-api => http/api}/resources/index.js | 0 .../api}/resources/object.js | 0 .../api}/resources/pubsub.js | 0 src/{http-api => http/api}/resources/repo.js | 0 src/{http-api => http/api}/resources/swarm.js | 0 .../api}/resources/version.js | 0 src/{http-api => http/api}/routes/bitswap.js | 0 src/{http-api => http/api}/routes/block.js | 0 .../api}/routes/bootstrap.js | 0 src/{http-api => http/api}/routes/config.js | 0 src/{http-api => http/api}/routes/files.js | 12 - src/{http-api => http/api}/routes/id.js | 0 src/{http-api => http/api}/routes/index.js | 0 src/{http-api => http/api}/routes/object.js | 0 src/{http-api => http/api}/routes/pubsub.js | 0 src/{http-api => http/api}/routes/repo.js | 0 src/{http-api => http/api}/routes/swarm.js | 0 src/{http-api => http/api}/routes/version.js | 0 src/{http-api => http}/error-handler.js | 0 src/{http-api => http}/gateway/resolver.js | 78 +--- src/http/gateway/resources/gateway.js | 148 ++++++++ src/http/gateway/resources/index.js | 3 + src/http/gateway/routes/gateway.js | 18 + src/http/gateway/routes/index.js | 5 + src/{http-api => http}/gateway/utils/html.js | 0 src/{http-api => http}/gateway/utils/path.js | 0 src/{http-api => http}/gateway/utils/style.js | 0 src/{http-api => http}/index.js | 4 +- test/cli/pubsub.js | 2 +- test/gateway/index.js | 55 +++ test/gateway/spec/gateway.js | 50 +++ test/http-api/index.js | 2 +- test/interop/daemons/js.js | 2 +- test/node.js | 10 + test/utils/ipfs-factory-daemon/index.js | 2 +- 45 files changed, 657 insertions(+), 431 deletions(-) delete mode 100644 src/http-api/resources/files.js rename src/{http-api => http/api}/resources/bitswap.js (100%) rename src/{http-api => http/api}/resources/block.js (100%) rename src/{http-api => http/api}/resources/bootstrap.js (100%) rename src/{http-api => http/api}/resources/config.js (100%) create mode 100644 src/http/api/resources/files.js rename src/{http-api => http/api}/resources/id.js (100%) rename src/{http-api => http/api}/resources/index.js (100%) rename src/{http-api => http/api}/resources/object.js (100%) rename src/{http-api => http/api}/resources/pubsub.js (100%) rename src/{http-api => http/api}/resources/repo.js (100%) rename src/{http-api => http/api}/resources/swarm.js (100%) rename src/{http-api => http/api}/resources/version.js (100%) rename src/{http-api => http/api}/routes/bitswap.js (100%) rename src/{http-api => http/api}/routes/block.js (100%) rename src/{http-api => http/api}/routes/bootstrap.js (100%) rename src/{http-api => http/api}/routes/config.js (100%) rename src/{http-api => http/api}/routes/files.js (75%) rename src/{http-api => http/api}/routes/id.js (100%) rename src/{http-api => http/api}/routes/index.js (100%) rename src/{http-api => http/api}/routes/object.js (100%) rename src/{http-api => http/api}/routes/pubsub.js (100%) rename src/{http-api => http/api}/routes/repo.js (100%) rename src/{http-api => http/api}/routes/swarm.js (100%) rename src/{http-api => http/api}/routes/version.js (100%) rename src/{http-api => http}/error-handler.js (100%) rename src/{http-api => http}/gateway/resolver.js (52%) create mode 100644 src/http/gateway/resources/gateway.js create mode 100644 src/http/gateway/resources/index.js create mode 100644 src/http/gateway/routes/gateway.js create mode 100644 src/http/gateway/routes/index.js rename src/{http-api => http}/gateway/utils/html.js (100%) rename src/{http-api => http}/gateway/utils/path.js (100%) rename src/{http-api => http}/gateway/utils/style.js (100%) rename src/{http-api => http}/index.js (97%) create mode 100644 test/gateway/index.js create mode 100644 test/gateway/spec/gateway.js diff --git a/gulpfile.js b/gulpfile.js index 2be23a3e2b..26304b51a1 100644 --- a/gulpfile.js +++ b/gulpfile.js @@ -4,7 +4,7 @@ const gulp = require('gulp') const parallel = require('async/parallel') const series = require('async/series') const createTempRepo = require('./test/utils/create-repo-nodejs.js') -const HTTPAPI = require('./src/http-api') +const HTTPAPI = require('./src/http') const leftPad = require('left-pad') let nodes = [] diff --git a/package.json b/package.json index cf6825f85c..34004a2e80 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "test:unit:node": "gulp test:node", "test:unit:node:core": "TEST=core npm run test:unit:node", "test:unit:node:http": "TEST=http npm run test:unit:node", + "test:unit:node:gateway": "TEST=gateway npm run test:unit:node", "test:unit:node:cli": "TEST=cli npm run test:unit:node", "test:unit:browser": "gulp test:browser", "test:interop": "npm run test:interop:node", diff --git a/src/cli/commands/daemon.js b/src/cli/commands/daemon.js index 0414ca9654..4529e2af54 100644 --- a/src/cli/commands/daemon.js +++ b/src/cli/commands/daemon.js @@ -1,6 +1,6 @@ 'use strict' -const HttpAPI = require('../../http-api') +const HttpAPI = require('../../http') const utils = require('../utils') const print = utils.print diff --git a/src/http-api/resources/files.js b/src/http-api/resources/files.js deleted file mode 100644 index 12576b60f4..0000000000 --- a/src/http-api/resources/files.js +++ /dev/null @@ -1,347 +0,0 @@ -'use strict' - -const mh = require('multihashes') -const multipart = require('ipfs-multipart') -const debug = require('debug') -const tar = require('tar-stream') -const log = debug('jsipfs:http-api:files') -log.error = debug('jsipfs:http-api:files:error') -const pull = require('pull-stream') -const toPull = require('stream-to-pull-stream') -const pushable = require('pull-pushable') -const EOL = require('os').EOL -const toStream = require('pull-stream-to-stream') -const fileType = require('file-type') -const mime = require('mime-types') -const GatewayResolver = require('../gateway/resolver') -const PathUtils = require('../gateway/utils/path') -const Stream = require('stream') - -exports = module.exports - -// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` -exports.parseKey = (request, reply) => { - if (!request.query.arg) { - return reply({ - Message: "Argument 'key' is required", - Code: 0 - }).code(400).takeover() - } - - let key = request.query.arg - if (key.indexOf('/ipfs/') === 0) { - key = key.substring(6) - } - - const slashIndex = key.indexOf('/') - if (slashIndex > 0) { - key = key.substring(0, slashIndex) - } - - try { - mh.fromB58String(key) - } catch (err) { - log.error(err) - return reply({ - Message: 'invalid ipfs ref path', - Code: 0 - }).code(500).takeover() - } - - reply({ - key: request.query.arg - }) -} - -exports.cat = { - // uses common parseKey method that returns a `key` - parseArgs: exports.parseKey, - - // main route handler which is called after the above `parseArgs`, but only if the args were valid - handler: (request, reply) => { - const key = request.pre.args.key - const ipfs = request.server.app.ipfs - - ipfs.files.cat(key, (err, stream) => { - if (err) { - log.error(err) - return reply({ - Message: 'Failed to cat file: ' + err, - Code: 0 - }).code(500) - } - - // hapi is not very clever and throws if no - // - _read method - // - _readableState object - // are there :( - if (!stream._read) { - stream._read = () => {} - stream._readableState = {} - } - return reply(stream).header('X-Stream-Output', '1') - }) - } -} - -exports.get = { - // uses common parseKey method that returns a `key` - parseArgs: exports.parseKey, - - // main route handler which is called after the above `parseArgs`, but only if the args were valid - handler: (request, reply) => { - const key = request.pre.args.key - const ipfs = request.server.app.ipfs - const pack = tar.pack() - - ipfs.files.getPull(key, (err, stream) => { - if (err) { - log.error(err) - - reply({ - Message: 'Failed to get file: ' + err, - Code: 0 - }).code(500) - return - } - - pull( - stream, - pull.asyncMap((file, cb) => { - const header = {name: file.path} - if (!file.content) { - header.type = 'directory' - pack.entry(header) - cb() - } else { - header.size = file.size - const packStream = pack.entry(header, cb) - if (!packStream) { - // this happens if the request is aborted - // we just skip things then - log('other side hung up') - return cb() - } - toStream.source(file.content).pipe(packStream) - } - }), - pull.onEnd((err) => { - if (err) { - log.error(err) - pack.emit('error', err) - pack.destroy() - return - } - - pack.finalize() - }) - ) - - // the reply must read the tar stream, - // to pull values through - reply(pack).header('X-Stream-Output', '1') - }) - } -} - -exports.add = { - handler: (request, reply) => { - if (!request.payload) { - return reply({ - Message: 'Array, Buffer, or String is required.', - code: 0 - }).code(400).takeover() - } - - const ipfs = request.server.app.ipfs - // TODO: make pull-multipart - const parser = multipart.reqParser(request.payload) - let filesParsed = false - - const fileAdder = pushable() - - parser.on('file', (fileName, fileStream) => { - const filePair = { - path: fileName, - content: toPull(fileStream) - } - filesParsed = true - fileAdder.push(filePair) - }) - - parser.on('directory', (directory) => { - fileAdder.push({ - path: directory, - content: '' - }) - }) - - parser.on('end', () => { - if (!filesParsed) { - return reply({ - Message: "File argument 'data' is required.", - code: 0 - }).code(400).takeover() - } - fileAdder.end() - }) - - pull( - fileAdder, - ipfs.files.createAddPullStream(), - pull.map((file) => { - return { - Name: file.path ? file.path : file.hash, - Hash: file.hash - } - }), - pull.map((file) => JSON.stringify(file) + EOL), - pull.collect((err, files) => { - if (err) { - return reply({ - Message: err, - Code: 0 - }).code(500) - } - - if (files.length === 0 && filesParsed) { - return reply({ - Message: 'Failed to add files.', - Code: 0 - }).code(500) - } - - reply(files.join('\n')) - .header('x-chunked-output', '1') - .header('content-type', 'application/json') - }) - ) - } -} - -exports.gateway = { - checkHash: (request, reply) => { - if (!request.params.hash) { - return reply('Path Resolve error: path must contain at least one component').code(400).takeover() - } - - return reply({ - ref: `/ipfs/${request.params.hash}` - }) - }, - handler: (request, reply) => { - const ref = request.pre.args.ref - const ipfs = request.server.app.ipfs - - return GatewayResolver - .resolveMultihash(ipfs, ref) - .then((data) => { - ipfs - .files - .cat(data.multihash) - .then((stream) => { - if (ref.endsWith('/')) { - // remove trailing slash for files - return reply - .redirect(PathUtils.removeTrailingSlash(ref)) - .permanent(true) - } else { - if (!stream._read) { - stream._read = () => {} - stream._readableState = {} - } - - let filetypeChecked = false - let stream2 = new Stream.PassThrough({highWaterMark: 1}) - let response = reply(stream2).hold() - - pull( - toPull.source(stream), - pull.drain((chunk) => { - if (chunk.length > 0 && !filetypeChecked) { - console.log('got first chunk') - let fileSignature = fileType(chunk) - console.log('file type: ', fileSignature) - - filetypeChecked = true - const mimeType = mime.lookup((fileSignature) ? fileSignature.ext : null) - console.log('ref ', ref) - console.log('mime-type ', mimeType) - - if (mimeType) { - console.log('writing mimeType') - - response - .header('Content-Type', mime.contentType(mimeType)) - .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') - .header('Access-Control-Allow-Methods', 'GET') - .header('Access-Control-Allow-Origin', '*') - .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') - .send() - } else { - response - .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') - .header('Access-Control-Allow-Methods', 'GET') - .header('Access-Control-Allow-Origin', '*') - .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') - .send() - } - - stream2.write(chunk) - } else { - // console.log('chunk length: ', chunk.length) - stream2.write(chunk) - } - }, (err) => { - if (err) throw err - console.log('stream ended.') - stream2.end() - }) - ) - } - }) - .catch((err) => { - if (err) { - log.error(err) - return reply(err.toString()).code(500) - } - }) - }).catch((err) => { - console.log('err: ', err.toString(), ' fileName: ', err.fileName) - - const errorToString = err.toString() - if (errorToString === 'Error: This dag node is a directory') { - return GatewayResolver - .resolveDirectory(ipfs, ref, err.fileName) - .then((data) => { - if (typeof data === 'string') { - // no index file found - if (!ref.endsWith('/')) { - // for a directory, if URL doesn't end with a / - // append / and redirect permanent to that URL - return reply.redirect(`${ref}/`).permanent(true) - } else { - // send directory listing - return reply(data) - } - } else { - // found index file - // redirect to URL/ - return reply.redirect(PathUtils.joinURLParts(ref, data[0].name)) - } - }).catch((err) => { - log.error(err) - return reply(err.toString()).code(500) - }) - } else if (errorToString.startsWith('Error: no link named')) { - return reply(errorToString).code(404) - } else if (errorToString.startsWith('Error: multihash length inconsistent') || - errorToString.startsWith('Error: Non-base58 character')) { - return reply(errorToString).code(400) - } else { - log.error(err) - return reply(errorToString).code(500) - } - }) - } -} diff --git a/src/http-api/resources/bitswap.js b/src/http/api/resources/bitswap.js similarity index 100% rename from src/http-api/resources/bitswap.js rename to src/http/api/resources/bitswap.js diff --git a/src/http-api/resources/block.js b/src/http/api/resources/block.js similarity index 100% rename from src/http-api/resources/block.js rename to src/http/api/resources/block.js diff --git a/src/http-api/resources/bootstrap.js b/src/http/api/resources/bootstrap.js similarity index 100% rename from src/http-api/resources/bootstrap.js rename to src/http/api/resources/bootstrap.js diff --git a/src/http-api/resources/config.js b/src/http/api/resources/config.js similarity index 100% rename from src/http-api/resources/config.js rename to src/http/api/resources/config.js diff --git a/src/http/api/resources/files.js b/src/http/api/resources/files.js new file mode 100644 index 0000000000..1ea7cb0099 --- /dev/null +++ b/src/http/api/resources/files.js @@ -0,0 +1,345 @@ +'use strict' + +const mh = require('multihashes') +const multipart = require('ipfs-multipart') +const debug = require('debug') +const tar = require('tar-stream') +const log = debug('jsipfs:http-api:files') +log.error = debug('jsipfs:http-api:files:error') +const pull = require('pull-stream') +const toPull = require('stream-to-pull-stream') +const pushable = require('pull-pushable') +const EOL = require('os').EOL +const toStream = require('pull-stream-to-stream') +// const fileType = require('file-type') +// const mime = require('mime-types') +// const GatewayResolver = require('../gateway/resolver') +// const PathUtils = require('../gateway/utils/path') +// const Stream = require('stream') + +exports = module.exports + +// common pre request handler that parses the args and returns `key` which is assigned to `request.pre.args` +exports.parseKey = (request, reply) => { + if (!request.query.arg) { + return reply({ + Message: "Argument 'key' is required", + Code: 0 + }).code(400).takeover() + } + + let key = request.query.arg + if (key.indexOf('/ipfs/') === 0) { + key = key.substring(6) + } + + const slashIndex = key.indexOf('/') + if (slashIndex > 0) { + key = key.substring(0, slashIndex) + } + + try { + mh.fromB58String(key) + } catch (err) { + log.error(err) + return reply({ + Message: 'invalid ipfs ref path', + Code: 0 + }).code(500).takeover() + } + + reply({ + key: request.query.arg + }) +} + +exports.cat = { + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + handler: (request, reply) => { + const key = request.pre.args.key + const ipfs = request.server.app.ipfs + + ipfs.files.cat(key, (err, stream) => { + if (err) { + log.error(err) + return reply({ + Message: 'Failed to cat file: ' + err, + Code: 0 + }).code(500) + } + + // hapi is not very clever and throws if no + // - _read method + // - _readableState object + // are there :( + if (!stream._read) { + stream._read = () => {} + stream._readableState = {} + } + return reply(stream).header('X-Stream-Output', '1') + }) + } +} + +exports.get = { + // uses common parseKey method that returns a `key` + parseArgs: exports.parseKey, + + // main route handler which is called after the above `parseArgs`, but only if the args were valid + handler: (request, reply) => { + const key = request.pre.args.key + const ipfs = request.server.app.ipfs + const pack = tar.pack() + + ipfs.files.getPull(key, (err, stream) => { + if (err) { + log.error(err) + + reply({ + Message: 'Failed to get file: ' + err, + Code: 0 + }).code(500) + return + } + + pull( + stream, + pull.asyncMap((file, cb) => { + const header = {name: file.path} + if (!file.content) { + header.type = 'directory' + pack.entry(header) + cb() + } else { + header.size = file.size + const packStream = pack.entry(header, cb) + if (!packStream) { + // this happens if the request is aborted + // we just skip things then + log('other side hung up') + return cb() + } + toStream.source(file.content).pipe(packStream) + } + }), + pull.onEnd((err) => { + if (err) { + log.error(err) + pack.emit('error', err) + pack.destroy() + return + } + + pack.finalize() + }) + ) + + // the reply must read the tar stream, + // to pull values through + reply(pack).header('X-Stream-Output', '1') + }) + } +} + +exports.add = { + handler: (request, reply) => { + if (!request.payload) { + return reply({ + Message: 'Array, Buffer, or String is required.', + code: 0 + }).code(400).takeover() + } + + const ipfs = request.server.app.ipfs + // TODO: make pull-multipart + const parser = multipart.reqParser(request.payload) + let filesParsed = false + + const fileAdder = pushable() + + parser.on('file', (fileName, fileStream) => { + const filePair = { + path: fileName, + content: toPull(fileStream) + } + filesParsed = true + fileAdder.push(filePair) + }) + + parser.on('directory', (directory) => { + fileAdder.push({ + path: directory, + content: '' + }) + }) + + parser.on('end', () => { + if (!filesParsed) { + return reply({ + Message: "File argument 'data' is required.", + code: 0 + }).code(400).takeover() + } + fileAdder.end() + }) + + pull( + fileAdder, + ipfs.files.createAddPullStream(), + pull.map((file) => { + return { + Name: file.path ? file.path : file.hash, + Hash: file.hash + } + }), + pull.map((file) => JSON.stringify(file) + EOL), + pull.collect((err, files) => { + if (err) { + return reply({ + Message: err, + Code: 0 + }).code(500) + } + + if (files.length === 0 && filesParsed) { + return reply({ + Message: 'Failed to add files.', + Code: 0 + }).code(500) + } + + reply(files.join('\n')) + .header('x-chunked-output', '1') + .header('content-type', 'application/json') + }) + ) + } +} + +// exports.gateway = { +// checkHash: (request, reply) => { +// if (!request.params.hash) { +// return reply('Path Resolve error: path must contain at least one component').code(400).takeover() +// } +// +// return reply({ +// ref: `/ipfs/${request.params.hash}` +// }) +// }, +// handler: (request, reply) => { +// const ref = request.pre.args.ref +// const ipfs = request.server.app.ipfs +// +// return GatewayResolver +// .resolveMultihash(ipfs, ref) +// .then((data) => { +// ipfs +// .files +// .cat(data.multihash) +// .then((stream) => { +// if (ref.endsWith('/')) { +// // remove trailing slash for files +// return reply +// .redirect(PathUtils.removeTrailingSlash(ref)) +// .permanent(true) +// } else { +// if (!stream._read) { +// stream._read = () => {} +// stream._readableState = {} +// } +// // response.continue() +// let filetypeChecked = false +// let stream2 = new Stream.PassThrough({highWaterMark: 1}) +// let response = reply(stream2).hold() +// +// pull( +// toPull.source(stream), +// pull.drain((chunk) => { +// // Check file type. do this once. +// if (chunk.length > 0 && !filetypeChecked) { +// console.log('got first chunk') +// let fileSignature = fileType(chunk) +// console.log('file type: ', fileSignature) +// +// filetypeChecked = true +// const mimeType = mime.lookup((fileSignature) ? fileSignature.ext : null) +// console.log('ref ', ref) +// console.log('mime-type ', mimeType) +// +// if (mimeType) { +// console.log('writing mimeType') +// +// response +// .header('Content-Type', mime.contentType(mimeType)) +// .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') +// .header('Access-Control-Allow-Methods', 'GET') +// .header('Access-Control-Allow-Origin', '*') +// .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') +// .send() +// } else { +// response +// .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') +// .header('Access-Control-Allow-Methods', 'GET') +// .header('Access-Control-Allow-Origin', '*') +// .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') +// .send() +// } +// } +// +// stream2.write(chunk) +// }, (err) => { +// if (err) throw err +// console.log('stream ended.') +// stream2.end() +// }) +// ) +// } +// }) +// .catch((err) => { +// if (err) { +// log.error(err) +// return reply(err.toString()).code(500) +// } +// }) +// }).catch((err) => { +// console.log('err: ', err.toString(), ' fileName: ', err.fileName) +// +// const errorToString = err.toString() +// if (errorToString === 'Error: This dag node is a directory') { +// return GatewayResolver +// .resolveDirectory(ipfs, ref, err.fileName) +// .then((data) => { +// if (typeof data === 'string') { +// // no index file found +// if (!ref.endsWith('/')) { +// // for a directory, if URL doesn't end with a / +// // append / and redirect permanent to that URL +// return reply.redirect(`${ref}/`).permanent(true) +// } else { +// // send directory listing +// return reply(data) +// } +// } else { +// // found index file +// // redirect to URL/ +// return reply.redirect(PathUtils.joinURLParts(ref, data[0].name)) +// } +// }).catch((err) => { +// log.error(err) +// return reply(err.toString()).code(500) +// }) +// } else if (errorToString.startsWith('Error: no link named')) { +// return reply(errorToString).code(404) +// } else if (errorToString.startsWith('Error: multihash length inconsistent') || +// errorToString.startsWith('Error: Non-base58 character')) { +// return reply(errorToString).code(400) +// } else { +// log.error(err) +// return reply(errorToString).code(500) +// } +// }) +// } +// } diff --git a/src/http-api/resources/id.js b/src/http/api/resources/id.js similarity index 100% rename from src/http-api/resources/id.js rename to src/http/api/resources/id.js diff --git a/src/http-api/resources/index.js b/src/http/api/resources/index.js similarity index 100% rename from src/http-api/resources/index.js rename to src/http/api/resources/index.js diff --git a/src/http-api/resources/object.js b/src/http/api/resources/object.js similarity index 100% rename from src/http-api/resources/object.js rename to src/http/api/resources/object.js diff --git a/src/http-api/resources/pubsub.js b/src/http/api/resources/pubsub.js similarity index 100% rename from src/http-api/resources/pubsub.js rename to src/http/api/resources/pubsub.js diff --git a/src/http-api/resources/repo.js b/src/http/api/resources/repo.js similarity index 100% rename from src/http-api/resources/repo.js rename to src/http/api/resources/repo.js diff --git a/src/http-api/resources/swarm.js b/src/http/api/resources/swarm.js similarity index 100% rename from src/http-api/resources/swarm.js rename to src/http/api/resources/swarm.js diff --git a/src/http-api/resources/version.js b/src/http/api/resources/version.js similarity index 100% rename from src/http-api/resources/version.js rename to src/http/api/resources/version.js diff --git a/src/http-api/routes/bitswap.js b/src/http/api/routes/bitswap.js similarity index 100% rename from src/http-api/routes/bitswap.js rename to src/http/api/routes/bitswap.js diff --git a/src/http-api/routes/block.js b/src/http/api/routes/block.js similarity index 100% rename from src/http-api/routes/block.js rename to src/http/api/routes/block.js diff --git a/src/http-api/routes/bootstrap.js b/src/http/api/routes/bootstrap.js similarity index 100% rename from src/http-api/routes/bootstrap.js rename to src/http/api/routes/bootstrap.js diff --git a/src/http-api/routes/config.js b/src/http/api/routes/config.js similarity index 100% rename from src/http-api/routes/config.js rename to src/http/api/routes/config.js diff --git a/src/http-api/routes/files.js b/src/http/api/routes/files.js similarity index 75% rename from src/http-api/routes/files.js rename to src/http/api/routes/files.js index 6328ef2faa..da57b3f2f1 100644 --- a/src/http-api/routes/files.js +++ b/src/http/api/routes/files.js @@ -4,7 +4,6 @@ const resources = require('./../resources') module.exports = (server) => { const api = server.select('API') - const gateway = server.select('Gateway') api.route({ // TODO fix method @@ -42,15 +41,4 @@ module.exports = (server) => { handler: resources.files.add.handler } }) - - gateway.route({ - method: '*', - path: '/ipfs/{hash*}', - config: { - pre: [ - { method: resources.files.gateway.checkHash, assign: 'args' } - ], - handler: resources.files.gateway.handler - } - }) } diff --git a/src/http-api/routes/id.js b/src/http/api/routes/id.js similarity index 100% rename from src/http-api/routes/id.js rename to src/http/api/routes/id.js diff --git a/src/http-api/routes/index.js b/src/http/api/routes/index.js similarity index 100% rename from src/http-api/routes/index.js rename to src/http/api/routes/index.js diff --git a/src/http-api/routes/object.js b/src/http/api/routes/object.js similarity index 100% rename from src/http-api/routes/object.js rename to src/http/api/routes/object.js diff --git a/src/http-api/routes/pubsub.js b/src/http/api/routes/pubsub.js similarity index 100% rename from src/http-api/routes/pubsub.js rename to src/http/api/routes/pubsub.js diff --git a/src/http-api/routes/repo.js b/src/http/api/routes/repo.js similarity index 100% rename from src/http-api/routes/repo.js rename to src/http/api/routes/repo.js diff --git a/src/http-api/routes/swarm.js b/src/http/api/routes/swarm.js similarity index 100% rename from src/http-api/routes/swarm.js rename to src/http/api/routes/swarm.js diff --git a/src/http-api/routes/version.js b/src/http/api/routes/version.js similarity index 100% rename from src/http-api/routes/version.js rename to src/http/api/routes/version.js diff --git a/src/http-api/error-handler.js b/src/http/error-handler.js similarity index 100% rename from src/http-api/error-handler.js rename to src/http/error-handler.js diff --git a/src/http-api/gateway/resolver.js b/src/http/gateway/resolver.js similarity index 52% rename from src/http-api/gateway/resolver.js rename to src/http/gateway/resolver.js index 672ac8f158..ae89956f06 100644 --- a/src/http-api/gateway/resolver.js +++ b/src/http/gateway/resolver.js @@ -1,9 +1,11 @@ 'use strict' const mh = require('multihashes') -// const pf = require('promised-for') const promisify = require('promisify-es6') const eachOfSeries = require('async/eachOfSeries') +const debug = require('debug') +const log = debug('jsipfs:http-gateway:resolver') +log.error = debug('jsipfs:http-gateway:resolver:error') const html = require('./utils/html') const PathUtil = require('./utils/path') @@ -48,28 +50,22 @@ const resolveMultihash = promisify((ipfs, path, callback) => { eachOfSeries(parts, (multihash, currentIndex, next) => { // throws error when invalid multihash is passed mh.validate(mh.fromB58String(currentMultihash)) - console.log('currentMultihash: ', currentMultihash) - console.log('currentIndex: ', currentIndex, '/', partsLength) + log('currentMultihash: ', currentMultihash) + log('currentIndex: ', currentIndex, '/', partsLength) ipfs .object .get(currentMultihash, { enc: 'base58' }) .then((DAGNode) => { - // console.log('DAGNode: ', DAGNode) + // log('DAGNode: ', DAGNode) if (currentIndex === partsLength - 1) { // leaf node - console.log('leaf node: ', currentMultihash) - console.log('DAGNode: ', DAGNode.links) + log('leaf node: ', currentMultihash) + // log('DAGNode: ', DAGNode.links) if (DAGNode.links && - DAGNode.links.length > 0 && - DAGNode.links[0].name.length > 0) { - for (let link of DAGNode.links) { - if (mh.toB58String(link.multihash) === currentMultihash) { - return next() - } - } - + DAGNode.links.length > 0 && + DAGNode.links[0].name.length > 0) { // this is a directory. let isDirErr = new Error('This dag node is a directory') // add currentMultihash as a fileName so it can be used by resolveDirectory @@ -89,12 +85,13 @@ const resolveMultihash = promisify((ipfs, path, callback) => { if (link.name === nextFileName) { // found multihash of requested named-file multihashOfNextFile = mh.toB58String(link.multihash) - console.log('found multihash: ', multihashOfNextFile) + log('found multihash: ', multihashOfNextFile) break } } if (!multihashOfNextFile) { + log.error(`no link named "${nextFileName}" under ${currentMultihash}`) throw new Error(`no link named "${nextFileName}" under ${currentMultihash}`) } @@ -104,60 +101,11 @@ const resolveMultihash = promisify((ipfs, path, callback) => { }) }, (err) => { if (err) { + log.error(err) return callback(err) } callback(null, {multihash: currentMultihash}) }) - // Original implementation - // return pf( - // { - // multihash: parts[0], - // index: 0 - // }, - // (i) => i.index < partsLength, - // (i) => { - // const currentIndex = i.index - // const currentMultihash = i.multihash - // - // // throws error when invalid multihash is passed - // mh.validate(mh.fromB58String(currentMultihash)) - // - // return ipfs - // .object - // .get(currentMultihash, { enc: 'base58' }) - // .then((DAGNode) => { - // if (currentIndex === partsLength - 1) { - // // leaf node - // return { - // multihash: currentMultihash, - // index: currentIndex + 1 - // } - // } else { - // // find multihash of requested named-file - // // in current DAGNode's links - // let multihashOfNextFile - // const nextFileName = parts[currentIndex + 1] - // const links = DAGNode.links - // - // for (let link of links) { - // if (link.name === nextFileName) { - // // found multihash of requested named-file - // multihashOfNextFile = mh.toB58String(link.multihash) - // break - // } - // } - // - // if (!multihashOfNextFile) { - // throw new Error(`no link named "${nextFileName}" under ${currentMultihash}`) - // } - // - // return { - // multihash: multihashOfNextFile, - // index: currentIndex + 1 - // } - // } - // }) - // }) }) module.exports = { diff --git a/src/http/gateway/resources/gateway.js b/src/http/gateway/resources/gateway.js new file mode 100644 index 0000000000..a3733b9e94 --- /dev/null +++ b/src/http/gateway/resources/gateway.js @@ -0,0 +1,148 @@ +'use strict' + +// const mh = require('multihashes') +// const multipart = require('ipfs-multipart') +const debug = require('debug') +// const tar = require('tar-stream') +const log = debug('jsipfs:http-gateway') +log.error = debug('jsipfs:http-gateway:error') +const pull = require('pull-stream') +const toPull = require('stream-to-pull-stream') +// const pushable = require('pull-pushable') +// const EOL = require('os').EOL +// const toStream = require('pull-stream-to-stream') +const fileType = require('file-type') +const mime = require('mime-types') +const GatewayResolver = require('../resolver') +const PathUtils = require('../utils/path') +const Stream = require('stream') + +exports = module.exports + +module.exports = { + checkHash: (request, reply) => { + if (!request.params.hash) { + return reply({ + Message: 'Path Resolve error: path must contain at least one component', + Code: 0 + }).code(400).takeover() + } + + return reply({ + ref: `/ipfs/${request.params.hash}` + }) + }, + handler: (request, reply) => { + const ref = request.pre.args.ref + const ipfs = request.server.app.ipfs + + return GatewayResolver + .resolveMultihash(ipfs, ref) + .then((data) => { + ipfs + .files + .cat(data.multihash) + .then((stream) => { + if (ref.endsWith('/')) { + // remove trailing slash for files + return reply + .redirect(PathUtils.removeTrailingSlash(ref)) + .permanent(true) + } else { + if (!stream._read) { + stream._read = () => {} + stream._readableState = {} + } + // response.continue() + let filetypeChecked = false + let stream2 = new Stream.PassThrough({highWaterMark: 1}) + let response = reply(stream2).hold() + + pull( + toPull.source(stream), + pull.drain((chunk) => { + // Check file type. do this once. + if (chunk.length > 0 && !filetypeChecked) { + log('got first chunk') + let fileSignature = fileType(chunk) + log('file type: ', fileSignature) + + filetypeChecked = true + const mimeType = mime.lookup((fileSignature) ? fileSignature.ext : null) + log('ref ', ref) + log('mime-type ', mimeType) + + if (mimeType) { + log('writing mimeType') + + response + .header('Content-Type', mime.contentType(mimeType)) + .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .header('Access-Control-Allow-Methods', 'GET') + .header('Access-Control-Allow-Origin', '*') + .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .send() + } else { + response + .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .header('Access-Control-Allow-Methods', 'GET') + .header('Access-Control-Allow-Origin', '*') + .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput') + .send() + } + } + + stream2.write(chunk) + }, (err) => { + if (err) throw err + log('stream ended.') + stream2.end() + }) + ) + } + }) + .catch((err) => { + if (err) { + log.error(err) + return reply(err.toString()).code(500) + } + }) + }).catch((err) => { + log('err: ', err.toString(), ' fileName: ', err.fileName) + + const errorToString = err.toString() + if (errorToString === 'Error: This dag node is a directory') { + return GatewayResolver + .resolveDirectory(ipfs, ref, err.fileName) + .then((data) => { + if (typeof data === 'string') { + // no index file found + if (!ref.endsWith('/')) { + // for a directory, if URL doesn't end with a / + // append / and redirect permanent to that URL + return reply.redirect(`${ref}/`).permanent(true) + } else { + // send directory listing + return reply(data) + } + } else { + // found index file + // redirect to URL/ + return reply.redirect(PathUtils.joinURLParts(ref, data[0].name)) + } + }).catch((err) => { + log.error(err) + return reply(err.toString()).code(500) + }) + } else if (errorToString.startsWith('Error: no link named')) { + return reply(errorToString).code(404) + } else if (errorToString.startsWith('Error: multihash length inconsistent') || + errorToString.startsWith('Error: Non-base58 character')) { + return reply({Message: errorToString, code: 0}).code(400) + } else { + log.error(err) + return reply({Message: errorToString, code: 0}).code(500) + } + }) + } +} diff --git a/src/http/gateway/resources/index.js b/src/http/gateway/resources/index.js new file mode 100644 index 0000000000..03f9d0901d --- /dev/null +++ b/src/http/gateway/resources/index.js @@ -0,0 +1,3 @@ +'use strict' + +exports.gateway = require('./gateway') diff --git a/src/http/gateway/routes/gateway.js b/src/http/gateway/routes/gateway.js new file mode 100644 index 0000000000..e1c0f3222f --- /dev/null +++ b/src/http/gateway/routes/gateway.js @@ -0,0 +1,18 @@ +'use strict' + +const resources = require('../resources') + +module.exports = (server) => { + const gateway = server.select('Gateway') + + gateway.route({ + method: '*', + path: '/ipfs/{hash*}', + config: { + pre: [ + { method: resources.gateway.checkHash, assign: 'args' } + ], + handler: resources.gateway.handler + } + }) +} diff --git a/src/http/gateway/routes/index.js b/src/http/gateway/routes/index.js new file mode 100644 index 0000000000..0e0656c258 --- /dev/null +++ b/src/http/gateway/routes/index.js @@ -0,0 +1,5 @@ +'use strict' + +module.exports = (server) => { + require('./gateway')(server) +} diff --git a/src/http-api/gateway/utils/html.js b/src/http/gateway/utils/html.js similarity index 100% rename from src/http-api/gateway/utils/html.js rename to src/http/gateway/utils/html.js diff --git a/src/http-api/gateway/utils/path.js b/src/http/gateway/utils/path.js similarity index 100% rename from src/http-api/gateway/utils/path.js rename to src/http/gateway/utils/path.js diff --git a/src/http-api/gateway/utils/style.js b/src/http/gateway/utils/style.js similarity index 100% rename from src/http-api/gateway/utils/style.js rename to src/http/gateway/utils/style.js diff --git a/src/http-api/index.js b/src/http/index.js similarity index 97% rename from src/http-api/index.js rename to src/http/index.js index a0d849e9d2..8ad07ef558 100644 --- a/src/http-api/index.js +++ b/src/http/index.js @@ -106,7 +106,9 @@ function HttpApi (repo, config, cliArgs) { errorHandler(this, this.server) // load routes - require('./routes')(this.server) + require('./api/routes')(this.server) + // load gateway routes + require('./gateway/routes')(this.server) // Set default headers setHeader(this.server, diff --git a/test/cli/pubsub.js b/test/cli/pubsub.js index 951c6a85ba..703b83a5b0 100644 --- a/test/cli/pubsub.js +++ b/test/cli/pubsub.js @@ -8,7 +8,7 @@ const expect = chai.expect chai.use(dirtyChai) const delay = require('delay') const waterfall = require('async/waterfall') -const HttpAPI = require('../../src/http-api') +const HttpAPI = require('../../src/http') // TODO needs to use ipfs-factory-daemon const createTempNode = '' const repoPath = require('./index').repoPath diff --git a/test/gateway/index.js b/test/gateway/index.js new file mode 100644 index 0000000000..356dc45a2f --- /dev/null +++ b/test/gateway/index.js @@ -0,0 +1,55 @@ +/* eslint-env mocha */ +'use strict' + +const fs = require('fs') +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const API = require('../../src/http') +// const APIctl = require('ipfs-api') +const ncp = require('ncp').ncp +const path = require('path') +const clean = require('../utils/clean') + +describe('HTTP GATEWAY', () => { + const repoExample = path.join(__dirname, '../go-ipfs-repo') + const repoTests = path.join(__dirname, '../repo-tests-run') + + let http = {} + + before((done) => { + http.api = new API(repoTests) + + ncp(repoExample, repoTests, (err) => { + expect(err).to.not.exist() + + http.api.start(false, done) + }) + }) + + after((done) => { + http.api.stop((err) => { + expect(err).to.not.exist() + clean(repoTests) + done() + }) + }) + + describe('## http-gateway spec tests', () => { + fs.readdirSync(path.join(__dirname, '/spec')) + .forEach((file) => require('./spec/' + file)(http)) + }) + + // describe('## interface tests', () => { + // fs.readdirSync(path.join(__dirname, '/interface')) + // .forEach((file) => require('./interface/' + file)) + // }) + // + // describe('## custom ipfs-api tests', () => { + // const ctl = APIctl('/ip4/127.0.0.1/tcp/6001') + // + // fs.readdirSync(path.join(__dirname, '/over-ipfs-api')) + // .forEach((file) => require('./over-ipfs-api/' + file)(ctl)) + // }) +}) diff --git a/test/gateway/spec/gateway.js b/test/gateway/spec/gateway.js new file mode 100644 index 0000000000..822c67a498 --- /dev/null +++ b/test/gateway/spec/gateway.js @@ -0,0 +1,50 @@ +/* eslint-env mocha */ +'use strict' + +const expect = require('chai').expect + +module.exports = (http) => { + describe('/files', () => { + let gateway + + before(() => { + gateway = http.api.server.select('Gateway') + }) + + describe('/ipfs', () => { + it('returns 400 for request without argument', (done) => { + gateway.inject({ + method: 'GET', + url: '/ipfs' + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result.Message).to.be.a('string') + done() + }) + }) + + it('400 for request with invalid argument', (done) => { + gateway.inject({ + method: 'GET', + url: '/ipfs/invalid' + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result.Message).to.be.a('string') + done() + }) + }) + + it('valid hash', (done) => { + gateway.inject({ + method: 'GET', + url: '/ipfs/QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.rawPayload).to.deep.equal(new Buffer('hello world' + '\n')) + expect(res.payload).to.equal('hello world' + '\n') + done() + }) + }) + }) + }) +} diff --git a/test/http-api/index.js b/test/http-api/index.js index ad647e92df..16e535322e 100644 --- a/test/http-api/index.js +++ b/test/http-api/index.js @@ -6,7 +6,7 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) -const API = require('../../src/http-api') +const API = require('../../src/http') const APIctl = require('ipfs-api') const ncp = require('ncp').ncp const path = require('path') diff --git a/test/interop/daemons/js.js b/test/interop/daemons/js.js index 66b81975ff..4b4089d494 100644 --- a/test/interop/daemons/js.js +++ b/test/interop/daemons/js.js @@ -6,7 +6,7 @@ const series = require('async/series') const rimraf = require('rimraf') const tmpDir = require('../util').tmpDir -const HttpApi = require('../../../src/http-api') +const HttpApi = require('../../../src/http') function portConfig (port) { port = port + 5 diff --git a/test/node.js b/test/node.js index 22872e6d02..fd97536cf0 100644 --- a/test/node.js +++ b/test/node.js @@ -3,6 +3,7 @@ let testCore = true let testHTTP = true let testCLI = true +let testGatway = true if (process.env.TEST) { switch (process.env.TEST) { @@ -14,6 +15,11 @@ if (process.env.TEST) { testCore = false testCLI = false break + case 'gateway': + testCore = false + testCLI = false + testHTTP = false + break case 'cli': testCore = false testHTTP = false @@ -34,3 +40,7 @@ if (testHTTP) { if (testCLI) { require('./cli') } + +if (testGatway) { + require('./gateway') +} diff --git a/test/utils/ipfs-factory-daemon/index.js b/test/utils/ipfs-factory-daemon/index.js index f9606a7414..37f208cb29 100644 --- a/test/utils/ipfs-factory-daemon/index.js +++ b/test/utils/ipfs-factory-daemon/index.js @@ -3,7 +3,7 @@ const PeerId = require('peer-id') const IPFSAPI = require('ipfs-api') const clean = require('../clean') -const HttpApi = require('../../../src/http-api') +const HttpApi = require('../../../src/http') const series = require('async/series') const eachSeries = require('async/eachSeries') const defaultConfig = require('./default-config.json')