diff --git a/.travis.yml b/.travis.yml index 9561a8e1a1..e18b74ad1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -164,6 +164,41 @@ jobs: script: - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000 + - stage: test + name: js-ipfs interface tests - ipfs-client - node + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t node + + - stage: test + name: js-ipfs interface tests - ipfs-client - chrome + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser + + - stage: test + name: js-ipfs interface tests - ipfs-client - chrome webworker + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000 + + - stage: test + name: js-ipfs interface tests - ipfs-client - firefox + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless + + - stage: test + name: js-ipfs interface tests - ipfs-client - firefox webworker + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000 + + - stage: test + name: js-ipfs interface tests - ipfs-client - electron main + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-main --timeout 60000 + + - stage: test + name: js-ipfs interface tests - ipfs-client - electron renderer + script: + - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000 + - stage: test name: http-api-client interface tests vs go-ipfs - node script: diff --git a/examples/browser-ipns-publish/package.json b/examples/browser-ipns-publish/package.json index 417b7edb8a..020e6bfac5 100644 --- a/examples/browser-ipns-publish/package.json +++ b/examples/browser-ipns-publish/package.json @@ -27,7 +27,7 @@ "devDependencies": { "delay": "^4.4.0", "execa": "^4.0.3", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "go-ipfs": "^0.7.0", "parcel-bundler": "^1.12.4", "path": "^0.12.7", diff --git a/examples/explore-ethereum-blockchain/package.json b/examples/explore-ethereum-blockchain/package.json index 0d4ec63e96..6cd4a169b2 100644 --- a/examples/explore-ethereum-blockchain/package.json +++ b/examples/explore-ethereum-blockchain/package.json @@ -12,7 +12,7 @@ "devDependencies": { "ipfs": "^0.52.2", "ipfs-http-client": "^48.1.2", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "ipld-ethereum": "^5.0.1", "test-ipfs-example": "^2.0.3" } diff --git a/examples/http-client-browser-pubsub/package.json b/examples/http-client-browser-pubsub/package.json index a2620020c3..3df503397d 100644 --- a/examples/http-client-browser-pubsub/package.json +++ b/examples/http-client-browser-pubsub/package.json @@ -21,7 +21,7 @@ "execa": "^4.0.3", "go-ipfs": "^0.7.0", "ipfs": "^0.52.2", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "parcel-bundler": "^1.12.4", "test-ipfs-example": "^2.0.3" } diff --git a/examples/http-client-bundle-webpack/package.json b/examples/http-client-bundle-webpack/package.json index 92fd387d19..f0363aef13 100644 --- a/examples/http-client-bundle-webpack/package.json +++ b/examples/http-client-bundle-webpack/package.json @@ -25,7 +25,7 @@ "copy-webpack-plugin": "^5.0.4", "execa": "^4.0.3", "ipfs": "^0.52.2", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "react-hot-loader": "^4.12.21", "rimraf": "^3.0.2", "test-ipfs-example": "^2.0.3", diff --git a/examples/http-client-name-api/package.json b/examples/http-client-name-api/package.json index f7e7bf9df2..04080259b7 100644 --- a/examples/http-client-name-api/package.json +++ b/examples/http-client-name-api/package.json @@ -18,7 +18,7 @@ "devDependencies": { "execa": "^4.0.3", "go-ipfs": "^0.7.0", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "parcel-bundler": "^1.12.4", "rimraf": "^3.0.2", "test-ipfs-example": "^2.0.3" diff --git a/examples/ipfs-client-add-files/README.md b/examples/ipfs-client-add-files/README.md new file mode 100644 index 0000000000..1fbd3bb0fd --- /dev/null +++ b/examples/ipfs-client-add-files/README.md @@ -0,0 +1,21 @@ +# JS IPFS API - Example Browser - Name + +## Setup + +```sh +npm install -g ipfs +jsipfs init +# Configure CORS to allow ipfs-http-client to access this IPFS node +jsipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["http://127.0.0.1:8888"]' +# Start the IPFS node +jsipfs daemon +``` + +Then in this folder run + +```bash +> npm install +> npm start +``` + +and open your browser at `http://127.0.0.1:8888`. diff --git a/examples/ipfs-client-add-files/index.html b/examples/ipfs-client-add-files/index.html new file mode 100644 index 0000000000..619eb6b919 --- /dev/null +++ b/examples/ipfs-client-add-files/index.html @@ -0,0 +1,36 @@ + + + + + JS IPFS Client example + + + + +

ipfs-client

+
+

Enter IPFS API details

+ + + +
+
+
+ + + + diff --git a/examples/ipfs-client-add-files/index.js b/examples/ipfs-client-add-files/index.js new file mode 100644 index 0000000000..d66a40ad4a --- /dev/null +++ b/examples/ipfs-client-add-files/index.js @@ -0,0 +1,81 @@ +/* eslint-disable no-console */ +'use strict' + +const ipfsClient = require('ipfs-client') +let ipfs + +const COLORS = { + active: 'blue', + success: 'green', + error: 'red' +} + +const showStatus = (text, bg) => { + console.info(text) + + const log = document.getElementById('output') + + if (!log) { + return + } + + const line = document.createElement('p') + line.innerText = text + line.style.color = bg + + log.appendChild(line) +} + +async function * streamFiles () { + for (let i = 0; i < 100; i++) { + await new Promise((resolve) => { + setTimeout(() => resolve(), 100) + }) + + showStatus(`Sending /file-${i}.txt`, COLORS.active) + + yield { + path: `/file-${i}.txt`, + content: `file ${i}` + } + } +} + +async function main (grpcApi, httpApi) { + showStatus(`Connecting to ${grpcApi} using ${httpApi} as fallback`, COLORS.active) + + ipfs = ipfsClient({ + grpc: grpcApi, + http: httpApi + }) + + const id = await ipfs.id() + showStatus(`Daemon active\nID: ${id.id}`, COLORS.success) + + for await (const file of ipfs.addAll(streamFiles(), { + wrapWithDirectory: true, + // this is just to show the interleaving of uploads and progress events + // otherwise we'd have to upload 50 files before we see any response from + // the server. do not specify this so low in production as you'll have + // greatly degraded import performance + fileImportConcurrency: 1, + progress: (bytes, file) => { + showStatus(`File progress ${file} ${bytes}`, COLORS.active) + } + })) { + showStatus(`Added file: ${file.path} ${file.cid}`, COLORS.success) + } + + showStatus('Finished!', COLORS.success) +} + +// Event listeners +document.getElementById('connect-submit').onclick = (e) => { + e.preventDefault() + + main(document.getElementById('grpc-input').value, document.getElementById('http-input').value) + .catch(err => { + showStatus(err.message, COLORS.error) + console.error(err) + }) +} diff --git a/examples/ipfs-client-add-files/package.json b/examples/ipfs-client-add-files/package.json new file mode 100644 index 0000000000..39b48b2e1c --- /dev/null +++ b/examples/ipfs-client-add-files/package.json @@ -0,0 +1,27 @@ +{ + "name": "example-ipfs-client-add-files", + "version": "1.0.0", + "description": "", + "main": "index.js", + "private": true, + "scripts": { + "clean": "rimraf ./dist", + "build": "parcel build index.html --public-url '.'", + "start": "parcel index.html -p 8888", + "test": "test-ipfs-example" + }, + "dependencies": { + "ipfs-client": "^0.1.0" + }, + "devDependencies": { + "execa": "^4.0.3", + "ipfs": "^0.52.0", + "ipfsd-ctl": "^7.2.0", + "parcel-bundler": "^1.12.4", + "rimraf": "^3.0.2", + "test-ipfs-example": "^2.0.3" + }, + "browserslist": [ + "last 2 versions and not dead and > 2%" + ] +} diff --git a/examples/ipfs-client-add-files/test.js b/examples/ipfs-client-add-files/test.js new file mode 100644 index 0000000000..84f6892d9d --- /dev/null +++ b/examples/ipfs-client-add-files/test.js @@ -0,0 +1,82 @@ +'use strict' + +const path = require('path') +const execa = require('execa') +const { createFactory } = require('ipfsd-ctl') +const df = createFactory({ + ipfsClientModule: require('ipfs-client'), + ipfsBin: require.resolve('ipfs/src/cli.js') +}) +const { + startServer +} = require('test-ipfs-example/utils') +const pkg = require('./package.json') + +async function testUI (url, http, grpc, id) { + const proc = execa(require.resolve('test-ipfs-example/node_modules/.bin/nightwatch'), ['--config', require.resolve('test-ipfs-example/nightwatch.conf.js'), path.join(__dirname, 'test.js')], { + cwd: path.resolve(__dirname, '../'), + env: { + ...process.env, + CI: true, + IPFS_EXAMPLE_TEST_URL: url, + IPFS_GRPC_API_MULTIADDR: grpc, + IPFS_HTTP_API_MULTIADDR: http + }, + all: true + }) + proc.all.on('data', (data) => { + process.stdout.write(data) + }) + + await proc +} + +async function runTest () { + const app = await startServer(__dirname) + const daemon = await df.spawn({ + type: 'js', + test: true, + ipfsOptions: { + config: { + Addresses: { + API: '/ip4/127.0.0.1/tcp/0', + RPC: '/ip4/127.0.0.1/tcp/0' + }, + API: { + HTTPHeaders: { + 'Access-Control-Allow-Origin': [ + app.url + ] + } + } + } + } + }) + + try { + await testUI(app.url, daemon.apiAddr, daemon.grpcAddr, daemon.api.peerId.id) + } finally { + await daemon.stop() + await app.stop() + } +} + +module.exports = runTest + +module.exports[pkg.name] = function (browser) { + browser + .url(process.env.IPFS_EXAMPLE_TEST_URL) + .waitForElementVisible('#grpc-input') + .clearValue('#grpc-input') + .setValue('#grpc-input', process.env.IPFS_GRPC_API_MULTIADDR) + .pause(1000) + .waitForElementVisible('#http-input') + .clearValue('#http-input') + .setValue('#http-input', process.env.IPFS_HTTP_API_MULTIADDR) + .pause(1000) + .click('#connect-submit') + + browser.expect.element('#output').text.to.contain('Added file: file-0.txt QmUDLiEJwL3vUhhXNXDF2RrCnVkSB2LemWYffpCCPcQCeU') + + browser.end() +} diff --git a/package.json b/package.json index 9cc642064d..767f2b1e1f 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "description": "JavaScript implementation of the IPFS specification", "scripts": { - "postinstall": "lerna bootstrap", + "postinstall": "lerna bootstrap && npm run build -- --scope=ipfs-grpc-protocol", "link": "lerna link", "reset": "lerna run clean && rimraf packages/*/node_modules node_modules", "test": "lerna run test", @@ -16,6 +16,7 @@ "test:external": "lerna run test:external", "test:cli": "lerna run test:cli", "test:interop": "lerna run test:interop", + "test:interface:client": "lerna run test:interface:client", "test:interface:core": "lerna run test:interface:core", "test:interface:http-go": "lerna run test:interface:http-go", "test:interface:http-js": "lerna run test:interface:http-js", diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js index 8dd06bc229..5b10ca77ac 100644 --- a/packages/interface-ipfs-core/src/add-all.js +++ b/packages/interface-ipfs-core/src/add-all.js @@ -422,6 +422,22 @@ module.exports = (common, options) => { expect(files[0].size).to.equal(18) }) + it('should add directories with metadata', async () => { + const files = await all(ipfs.addAll([{ + path: '/foo', + mode: 0o123, + mtime: { + secs: 1000, + nsecs: 0 + } + }])) + + expect(files.length).to.equal(1) + expect(files[0].cid.toString()).to.equal('QmaZTosBmPwo9LQ48ESPCEcNuX2kFxkpXYy8i3rxqBdzRG') + expect(files[0].cid.codec).to.equal('dag-pb') + expect(files[0].size).to.equal(11) + }) + it('should support bidirectional streaming', async function () { let progressInvoked diff --git a/packages/ipfs-cli/src/commands/daemon.js b/packages/ipfs-cli/src/commands/daemon.js index 83905dd12c..81340208b9 100644 --- a/packages/ipfs-cli/src/commands/daemon.js +++ b/packages/ipfs-cli/src/commands/daemon.js @@ -83,15 +83,17 @@ module.exports = { try { await daemon.start() - // @ts-ignore - _httpApi is possibly undefined + // @ts-ignore - _apiServers is possibly undefined daemon._httpApi._apiServers.forEach(apiServer => { - print(`API listening on ${apiServer.info.ma}`) + print(`HTTP API listening on ${apiServer.info.ma}`) }) + // @ts-ignore - _grpcServer is possibly undefined + print(`gRPC listening on ${daemon._grpcServer.multiaddr}`) // @ts-ignore - _httpGateway is possibly undefined daemon._httpGateway._gatewayServers.forEach(gatewayServer => { print(`Gateway (read only) listening on ${gatewayServer.info.ma}`) }) - // @ts-ignore - _httpApi is possibly undefined + // @ts-ignore - _apiServers is possibly undefined daemon._httpApi._apiServers.forEach(apiServer => { print(`Web UI available at ${toUri(apiServer.info.ma)}/webui`) }) diff --git a/packages/ipfs-client/.aegir.js b/packages/ipfs-client/.aegir.js new file mode 100644 index 0000000000..b55dc70587 --- /dev/null +++ b/packages/ipfs-client/.aegir.js @@ -0,0 +1,7 @@ +'use strict' + +module.exports = { + bundlesize: { + maxSize: '112kB' + } +} diff --git a/packages/ipfs-client/README.md b/packages/ipfs-client/README.md new file mode 100644 index 0000000000..23d7f8500e --- /dev/null +++ b/packages/ipfs-client/README.md @@ -0,0 +1,55 @@ +# ipfs-client + +> A client for [ipfs][] daemons + +This module combines the [ipfs-grpc-client][] and [ipfs-http-client][] modules to give you a client that is capable of bidirectional streaming in the browser as well as node. + +## Install + +```console +$ npm install ipfs-client +``` + +## API + +The client object created by the `createClient` function supports the [IPFS Core API](https://github.com/ipfs/js-ipfs/tree/master/docs/core-api), see the docs for more. + +### `createClient([options])` + +### Parameters + +None + +### Options + +An optional object which may have the following keys: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| grpc | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-grpc-server][] to connect to | +| http | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-http-server][] to connect to | + +### Returns + +| Type | Description | +| -------- | -------- | +| `object` | An instance of the client | + +### Example + +```js +const createClient = require('ipfs-client') + +const client = createClient({ + grpc: '/ipv4/127.0.0.1/tcp/5003/ws', + http: '/ipv4/127.0.0.1/tcp/5002/http' +}) + +const id = await client.id() +``` + +[ipfs]: https://www.npmjs.com/package/ipfs +[ipfs-grpc-client]: https://www.npmjs.com/package/ipfs-grpc-client +[ipfs-http-client]: https://www.npmjs.com/package/ipfs-http-client +[ipfs-grpc-server]: https://www.npmjs.com/package/ipfs-grpc-server +[ipfs-http-server]: https://www.npmjs.com/package/ipfs-http-server diff --git a/packages/ipfs-client/package.json b/packages/ipfs-client/package.json new file mode 100644 index 0000000000..1d3816a805 --- /dev/null +++ b/packages/ipfs-client/package.json @@ -0,0 +1,49 @@ +{ + "name": "ipfs-client", + "version": "0.1.0", + "description": "A client library to talk to local IPFS daemons", + "keywords": [ + "ipfs" + ], + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-client#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "main": "src/index.js", + "typesVersions": { + "*": { + "*": [ + "dist/*", + "dist/*/index" + ] + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "test": "aegir test", + "lint": "aegir lint", + "build": "npm run build:js && npm run build:types", + "build:js": "aegir build", + "build:types": "tsc --build", + "coverage": "npx nyc -r html npm run test:node -- --bail", + "clean": "rimraf ./dist", + "dep-check": "aegir dep-check -i aegir -i typescript -i rimraf" + }, + "dependencies": { + "ipfs-grpc-client": "0.0.0", + "ipfs-http-client": "^48.1.0", + "merge-options": "^2.0.0" + }, + "devDependencies": { + "aegir": "^29.2.2", + "rimraf": "^3.0.2", + "typescript": "4.0.x" + } +} diff --git a/packages/ipfs-client/src/index.js b/packages/ipfs-client/src/index.js new file mode 100644 index 0000000000..4f391e28fa --- /dev/null +++ b/packages/ipfs-client/src/index.js @@ -0,0 +1,26 @@ +'use strict' + +const httpClient = require('ipfs-http-client') +const grpcClient = require('ipfs-grpc-client') +const mergeOptions = require('merge-options') + +module.exports = function createClient (opts = {}) { + const clients = [] + + if (opts.http) { + clients.push(httpClient({ + ...opts, + url: opts.http + })) + } + + if (opts.grpc) { + clients.push(grpcClient({ + ...opts, + url: opts.grpc + })) + } + + // override http methods with grpc if address is supplied + return mergeOptions.apply({ ignoreUndefined: true }, clients) +} diff --git a/packages/ipfs-client/tsconfig.json b/packages/ipfs-client/tsconfig.json new file mode 100644 index 0000000000..979a39adab --- /dev/null +++ b/packages/ipfs-client/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "package.json" + ] +} diff --git a/packages/ipfs-core/.aegir.js b/packages/ipfs-core/.aegir.js index 5b46973aa3..41e0cda3b0 100644 --- a/packages/ipfs-core/.aegir.js +++ b/packages/ipfs-core/.aegir.js @@ -8,7 +8,7 @@ let preloadNode = MockPreloadNode.createNode() let ipfsdServer module.exports = { - bundlesize: { maxSize: '550kB' }, + bundlesize: { maxSize: '560kB' }, karma: { files: [{ pattern: 'node_modules/interface-ipfs-core/test/fixtures/**/*', diff --git a/packages/ipfs-core/package.json b/packages/ipfs-core/package.json index 0f39fad8a3..cf1968cc83 100644 --- a/packages/ipfs-core/package.json +++ b/packages/ipfs-core/package.json @@ -121,7 +121,7 @@ "delay": "^4.4.0", "go-ipfs": "^0.7.0", "interface-ipfs-core": "^0.142.3", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "ipld-git": "^0.6.1", "iso-url": "^1.0.0", "nanoid": "^3.1.12", diff --git a/packages/ipfs-core/src/runtime/config-browser.js b/packages/ipfs-core/src/runtime/config-browser.js index 5793ac4acd..af04e1f652 100644 --- a/packages/ipfs-core/src/runtime/config-browser.js +++ b/packages/ipfs-core/src/runtime/config-browser.js @@ -7,6 +7,7 @@ module.exports = () => ({ Announce: [], API: '', Gateway: '', + RPC: '', Delegates: [ '/dns4/node0.delegate.ipfs.io/tcp/443/https', '/dns4/node1.delegate.ipfs.io/tcp/443/https', diff --git a/packages/ipfs-core/src/runtime/config-nodejs.js b/packages/ipfs-core/src/runtime/config-nodejs.js index 345ff72276..c93048cd82 100644 --- a/packages/ipfs-core/src/runtime/config-nodejs.js +++ b/packages/ipfs-core/src/runtime/config-nodejs.js @@ -9,6 +9,7 @@ module.exports = () => ({ Announce: [], API: '/ip4/127.0.0.1/tcp/5002', Gateway: '/ip4/127.0.0.1/tcp/9090', + RPC: '/ip4/127.0.0.1/tcp/5003', Delegates: [ '/dns4/node0.delegate.ipfs.io/tcp/443/https', '/dns4/node1.delegate.ipfs.io/tcp/443/https', diff --git a/packages/ipfs-daemon/package.json b/packages/ipfs-daemon/package.json index d6e055957c..5cbda33abc 100644 --- a/packages/ipfs-daemon/package.json +++ b/packages/ipfs-daemon/package.json @@ -32,6 +32,7 @@ "debug": "^4.1.1", "dlv": "^1.1.3", "ipfs-core": "^0.3.1", + "ipfs-grpc-server": "0.0.0", "ipfs-http-client": "^48.1.3", "ipfs-http-gateway": "^0.1.4", "ipfs-http-server": "^0.1.4", @@ -46,7 +47,8 @@ "devDependencies": { "aegir": "^29.2.2", "node-fetch": "^2.6.1", - "typescript": "4.0.x" + "typescript": "4.0.x", + "ws": "^7.3.1" }, "optionalDependencies": { "prom-client": "^12.0.0", diff --git a/packages/ipfs-daemon/src/index.js b/packages/ipfs-daemon/src/index.js index 46b623f39c..8c36a2c435 100644 --- a/packages/ipfs-daemon/src/index.js +++ b/packages/ipfs-daemon/src/index.js @@ -11,6 +11,7 @@ const ipfsHttpClient = require('ipfs-http-client') const IPFS = require('ipfs-core') const HttpApi = require('ipfs-http-server') const HttpGateway = require('ipfs-http-gateway') +const gRPCServer = require('ipfs-grpc-server') const createRepo = require('ipfs-core/src/runtime/repo-nodejs') const { isElectron } = require('ipfs-utils/src/env') @@ -59,6 +60,8 @@ class Daemon { await repo.apiAddr.set(this._httpApi._apiServers[0].info.ma) } + this._grpcServer = await gRPCServer(ipfs, ipfsOpts) + log('started') return this } @@ -68,6 +71,7 @@ class Daemon { await Promise.all([ this._httpApi && this._httpApi.stop(), this._httpGateway && this._httpGateway.stop(), + this._grpcServer && this._grpcServer.stop(), // @ts-ignore - may not have stop if init was false this._ipfs && this._ipfs.stop() ]) diff --git a/packages/ipfs-daemon/test/index.spec.js b/packages/ipfs-daemon/test/index.spec.js index 0b88f7ef2b..eb6cedb143 100644 --- a/packages/ipfs-daemon/test/index.spec.js +++ b/packages/ipfs-daemon/test/index.spec.js @@ -4,6 +4,7 @@ const { expect } = require('aegir/utils/chai') const Daemon = require('../') const fetch = require('node-fetch') +const WebSocket = require('ws') describe('daemon', () => { let daemon @@ -46,6 +47,41 @@ describe('daemon', () => { await daemon.stop() }) + it('should start a gRPC server', async () => { + daemon = new Daemon({}) + + await daemon.start() + + const { + uri + } = daemon._grpcServer.info + + const socket = new WebSocket(`${uri}/ipfs.Root/id`) + let received = Buffer.alloc(0) + + await new Promise((resolve) => { + socket.on('open', () => { + socket.send(Buffer.from('Y29udGVudC10eXBlOiBhcHBsaWNhdGlvbi9ncnBjLXdlYitwcm90bw0KeC1ncnBjLXdlYjogMQ0K', 'base64')) + socket.send(Buffer.from('AAAAAAAA', 'base64')) + }) + + socket.on('message', (data) => { + received = Buffer.concat([received, data], received.byteLength + data.byteLength) + }) + + socket.on('close', () => { + resolve() + }) + }) + + const apiId = await daemon._ipfs.id() + + // don't try to decode protobuf, just look for embedded string + expect(received.toString('utf8')).to.include(apiId.id) + + await daemon.stop() + }) + it('should stop', async () => { daemon = new Daemon({}) diff --git a/packages/ipfs-daemon/tsconfig.json b/packages/ipfs-daemon/tsconfig.json index f84b5f655c..2111e28905 100644 --- a/packages/ipfs-daemon/tsconfig.json +++ b/packages/ipfs-daemon/tsconfig.json @@ -14,6 +14,9 @@ { "path": "../ipfs-core-utils" }, + { + "path": "../ipfs-grpc-server" + }, { "path": "../ipfs-http-client" }, diff --git a/packages/ipfs-grpc-client/.aegir.js b/packages/ipfs-grpc-client/.aegir.js new file mode 100644 index 0000000000..4173cb261e --- /dev/null +++ b/packages/ipfs-grpc-client/.aegir.js @@ -0,0 +1,7 @@ +'use strict' + +module.exports = { + bundlesize: { + maxSize: '48kB' + } +} diff --git a/packages/ipfs-grpc-client/README.md b/packages/ipfs-grpc-client/README.md new file mode 100644 index 0000000000..1c2be2b0d9 --- /dev/null +++ b/packages/ipfs-grpc-client/README.md @@ -0,0 +1,57 @@ +# ipfs-grpc-client + +> A client for the [ipfs-grpc-server][] module + +This module implements part of the [IPFS Core API](https://github.com/ipfs/js-ipfs/tree/master/docs/core-api) using gRPC over websockets to achieve the bidirectional streaming necessary to have full duplex streams running in the browser. + +It's not recommended you use this directly, instead use the [ipfs-client](https://www.npmjs.com/package/ipfs-client) to combine this with the [ipfs-http-client](https://www.npmjs.com/package/ipfs-http-client) in order to have HTTP fallback for the missing parts of the API. + +## Why? + +The [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) and [XHR](https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest) APIs do not allow for full-duplex streaming, that is, allowing the client to receive bytes from the response while also adding more bytes to the outgoing request. + +This limits what we can do in browsers in terms of the API, for example streaming arbitrarily sized payloads or exposing libp2p duplex streams. + +gPRC over websockets has no such limitations so allows us to harness the full power of a remote IPFS node in the browser without the need to work around browser behaviour. + +## Install + +```console +$ npm install ipfs-grpc-client +``` + +## API + +### `createClient([options])` + +### Parameters + +None + +### Options + +An optional object which may have the following keys: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| url | `Multiaddr` or `string` or `URL` | `undefined` | The address of a [ipfs-grpc-server][] to connect to | + +### Returns + +| Type | Description | +| -------- | -------- | +| `object` | An instance of the client | + +### Example + +```js +const createClient = require('ipfs-gprc-client') + +const client = createClient({ + url: '/ipv4/127.0.0.1/tcp/1234/ws' +}) + +const id = await client.id() +``` + +[ipfs-grpc-server]: https://www.npmjs.com/package/ipfs-grpc-server diff --git a/packages/ipfs-grpc-client/package.json b/packages/ipfs-grpc-client/package.json new file mode 100644 index 0000000000..6d4b0e392b --- /dev/null +++ b/packages/ipfs-grpc-client/package.json @@ -0,0 +1,64 @@ +{ + "name": "ipfs-grpc-client", + "version": "0.0.0", + "description": "A client library for the IPFS gRPC API", + "keywords": [ + "ipfs" + ], + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-grpc-client#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "main": "src/index.js", + "browser": { + "./src/grpc/transport.js": "./src/grpc/transport.browser.js" + }, + "typesVersions": { + "*": { + "*": [ + "dist/*", + "dist/*/index" + ] + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "test": "aegir test", + "lint": "aegir lint", + "build": "npm run build:js && npm run build:types", + "build:js": "aegir build", + "build:types": "tsc --build", + "coverage": "npx nyc -r html npm run test:node -- --bail", + "clean": "rimraf ./dist", + "dep-check": "aegir dep-check -i aegir -i typescript -i rimraf -i ipfs-grpc-protocol" + }, + "dependencies": { + "@improbable-eng/grpc-web": "^0.13.0", + "change-case": "^4.1.1", + "cids": "^1.0.0", + "debug": "^4.1.1", + "err-code": "^2.0.3", + "ipfs-core-utils": "^0.5.0", + "ipfs-grpc-protocol": "0.0.0", + "it-first": "^1.0.4", + "it-pushable": "^1.4.0", + "protobufjs": "^6.10.2", + "multiaddr": "^8.0.0", + "wherearewe": "0.0.1", + "ws": "^7.3.1" + }, + "devDependencies": { + "aegir": "^29.2.2", + "it-all": "^1.0.4", + "rimraf": "^3.0.2", + "sinon": "^9.0.3", + "typescript": "4.0.x" + } +} diff --git a/packages/ipfs-grpc-client/src/core-api/add-all.js b/packages/ipfs-grpc-client/src/core-api/add-all.js new file mode 100644 index 0000000000..af11d3e4c5 --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/add-all.js @@ -0,0 +1,118 @@ +'use strict' + +const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') +const CID = require('cids') +const bidiToDuplex = require('../utils/bidi-to-duplex') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +function sendDirectory (index, sink, path, mode, mtime) { + const message = { + index, + type: 'DIRECTORY', + path + } + + if (mtime) { + message.mtime = mtime.secs + message.mtimeNsecs = mtime.nsecs + } + + if (mode != null) { + message.mode = mode + } + + sink.push(message) +} + +async function sendFile (index, sink, content, path, mode, mtime) { + for await (const buf of content) { + const message = { + index, + type: 'FILE', + path + } + + if (mtime) { + message.mtime = mtime.secs + message.mtimeNsecs = mtime.nsecs + } + + if (mode != null) { + message.mode = mode + } + + message.content = new Uint8Array(buf, buf.byteOffset, buf.byteLength) + + sink.push(message) + } + + // signal that the file data has finished + const message = { + index, + type: 'FILE', + path + } + + sink.push(message) +} + +async function sendFiles (stream, sink) { + let i = 1 + + for await (const { path, content, mode, mtime } of normaliseInput(stream)) { + const index = i + i++ + + if (content) { + await sendFile(index, sink, content, path, mode, mtime) + } else { + sendDirectory(index, sink, path, mode, mtime) + } + } +} + +module.exports = function grpcAddAll (grpc, service, opts = {}) { + async function * addAll (stream, options = {}) { + const { + source, + sink + } = bidiToDuplex(grpc, service, { + host: opts.url, + debug: Boolean(process.env.DEBUG), + metadata: options + }) + + sendFiles(stream, sink) + .catch(err => { + sink.end(err) + }) + .finally(() => { + sink.end() + }) + + for await (const result of source) { + // received progress result + if (result.type === 'PROGRESS') { + if (options.progress) { + options.progress(result.bytes, result.path) + } + + continue + } + + // received file/dir import result + yield { + path: result.path, + cid: new CID(result.cid), + mode: result.mode, + mtime: { + secs: result.mtime || 0, + nsecs: result.mtimeNsecs || 0 + }, + size: result.size + } + } + } + + return withTimeoutOption(addAll) +} diff --git a/packages/ipfs-grpc-client/src/core-api/files/ls.js b/packages/ipfs-grpc-client/src/core-api/files/ls.js new file mode 100644 index 0000000000..b3be56c0b0 --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/files/ls.js @@ -0,0 +1,33 @@ +'use strict' + +const CID = require('cids') +const serverStreamToIterator = require('../../utils/server-stream-to-iterator') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') + +module.exports = function grpcMfsLs (grpc, service, opts = {}) { + async function * mfsLs (path, options = {}) { + const request = { + path + } + + for await (const result of serverStreamToIterator(grpc, service, request, { + host: opts.url, + debug: Boolean(process.env.DEBUG), + metadata: options + })) { + yield { + name: result.name, + type: result.type.toLowerCase(), + size: result.size, + cid: new CID(result.cid), + mode: result.mode, + mtime: { + secs: result.mtime || 0, + nsecs: result.mtimeNsecs || 0 + } + } + } + } + + return withTimeoutOption(mfsLs) +} diff --git a/packages/ipfs-grpc-client/src/core-api/files/write.js b/packages/ipfs-grpc-client/src/core-api/files/write.js new file mode 100644 index 0000000000..30856247a2 --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/files/write.js @@ -0,0 +1,46 @@ +'use strict' + +const clientStreamToPromise = require('../../utils/client-stream-to-promise') +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') +const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') +const { mtimeToObject, modeToNumber } = require('ipfs-core-utils/src/files/normalise-input/utils') + +async function * stream (path, content) { + for await (const { content: bufs } of normaliseInput(content)) { + if (!bufs) { + return + } + + for await (const content of bufs) { + yield { path, content } + } + } +} + +module.exports = function grpcMfsWrite (grpc, service, opts = {}) { + async function mfsWrite (path, content, options = {}) { + const mtime = mtimeToObject(options.mtime) + + if (mtime != null) { + options = { + ...options, + mtime: mtime.secs, + mtimeNsecs: mtime.nsecs + } + } + + const mode = modeToNumber(options.mode) + + if (mode != null) { + options.mode = mode + } + + await clientStreamToPromise(grpc, service, stream(path, content), { + host: opts.url, + debug: Boolean(process.env.DEBUG), + metadata: options + }) + } + + return withTimeoutOption(mfsWrite) +} diff --git a/packages/ipfs-grpc-client/src/core-api/id.js b/packages/ipfs-grpc-client/src/core-api/id.js new file mode 100644 index 0000000000..b5486cbb68 --- /dev/null +++ b/packages/ipfs-grpc-client/src/core-api/id.js @@ -0,0 +1,24 @@ +'use strict' + +const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option') +const toHeaders = require('../utils/to-headers') +const unaryToPromise = require('../utils/unary-to-promise') +const multiaddr = require('multiaddr') + +module.exports = function grpcId (grpc, service, opts = {}) { + async function id (options = {}) { + const request = {} + + const res = await unaryToPromise(grpc, service, request, { + host: opts.url, + metadata: toHeaders(options) + }) + + return { + ...res, + addresses: (res.addresses || []).map(multiaddr) + } + } + + return withTimeoutOption(id) +} diff --git a/packages/ipfs-grpc-client/src/grpc/transport.browser.js b/packages/ipfs-grpc-client/src/grpc/transport.browser.js new file mode 100644 index 0000000000..4981987221 --- /dev/null +++ b/packages/ipfs-grpc-client/src/grpc/transport.browser.js @@ -0,0 +1,5 @@ +'use strict' + +const { grpc } = require('@improbable-eng/grpc-web') + +module.exports = grpc.WebsocketTransport diff --git a/packages/ipfs-grpc-client/src/grpc/transport.js b/packages/ipfs-grpc-client/src/grpc/transport.js new file mode 100644 index 0000000000..9fcea8d428 --- /dev/null +++ b/packages/ipfs-grpc-client/src/grpc/transport.js @@ -0,0 +1,10 @@ +'use strict' + +const { isElectronRenderer } = require('wherearewe') + +// In electron-renderer we use the browser transport +if (isElectronRenderer) { + module.exports = require('./transport.browser') +} else { + module.exports = require('./transport.node') +} diff --git a/packages/ipfs-grpc-client/src/grpc/transport.node.js b/packages/ipfs-grpc-client/src/grpc/transport.node.js new file mode 100644 index 0000000000..21a3dd53aa --- /dev/null +++ b/packages/ipfs-grpc-client/src/grpc/transport.node.js @@ -0,0 +1,125 @@ +'use strict' + +// copied from https://github.com/improbable-eng/grpc-web/blob/master/client/grpc-web/src/transports/websocket/websocket.ts +// but uses the ws implementation of WebSockets +// see: https://github.com/improbable-eng/grpc-web/issues/796 + +const WebSocket = require('ws') +const debug = require('debug')('ipfs:grpc-client:websocket-transport') + +const WebsocketSignal = { + FINISH_SEND: 1 +} + +const finishSendFrame = new Uint8Array([1]) + +function WebsocketTransport () { + return (opts) => { + return websocketRequest(opts) + } +} + +function websocketRequest (options) { + const webSocketAddress = constructWebSocketAddress(options.url) + + let sendQueue = [] + let ws + + function sendToWebsocket (toSend) { + if (toSend === WebsocketSignal.FINISH_SEND) { + ws.send(finishSendFrame) + } else { + const byteArray = toSend + const c = new Int8Array(byteArray.byteLength + 1) + c.set(new Uint8Array([0])) + c.set(byteArray, 1) + + ws.send(c) + } + } + + return { + sendMessage: (msgBytes) => { + if (!ws || ws.readyState === ws.CONNECTING) { + sendQueue.push(msgBytes) + } else { + sendToWebsocket(msgBytes) + } + }, + finishSend: () => { + if (!ws || ws.readyState === ws.CONNECTING) { + sendQueue.push(WebsocketSignal.FINISH_SEND) + } else { + sendToWebsocket(WebsocketSignal.FINISH_SEND) + } + }, + start: (metadata) => { + ws = new WebSocket(webSocketAddress, ['grpc-websockets']) + ws.binaryType = 'arraybuffer' + ws.onopen = function () { + options.debug && debug('websocketRequest.onopen') + ws.send(headersToBytes(metadata)) + + // send any messages that were passed to sendMessage before the connection was ready + sendQueue.forEach(toSend => { + sendToWebsocket(toSend) + }) + sendQueue = [] + } + + ws.onclose = function (closeEvent) { + options.onEnd() + } + + ws.onerror = function (error) { + options.debug && debug('websocketRequest.onerror', error) + } + + ws.onmessage = function (e) { + options.onChunk(new Uint8Array(e.data, 0, e.data.byteLength)) + } + }, + cancel: () => { + ws.close() + } + } +} + +function constructWebSocketAddress (url) { + if (url.startsWith('wss://') || url.startsWith('ws://')) { + return url + } else if (url.substr(0, 8) === 'https://') { + return `wss://${url.substr(8)}` + } else if (url.substr(0, 7) === 'http://') { + return `ws://${url.substr(7)}` + } + throw new Error('Websocket transport constructed with non-https:// or http:// host.') +} + +function headersToBytes (headers) { + let asString = '' + headers.forEach((key, values) => { + asString += `${key}: ${values.join(', ')}\r\n` + }) + return encodeASCII(asString) +} + +function encodeASCII (input) { + const encoded = new Uint8Array(input.length) + for (let i = 0; i !== input.length; ++i) { + const charCode = input.charCodeAt(i) + if (!isValidHeaderAscii(charCode)) { + throw new Error('Metadata contains invalid ASCII') + } + encoded[i] = charCode + } + return encoded +} + +const isAllowedControlChars = (char) => char === 0x9 || char === 0xa || char === 0xd + +function isValidHeaderAscii (val) { + return isAllowedControlChars(val) || (val >= 0x20 && val <= 0x7e) +} + +module.exports = WebsocketTransport diff --git a/packages/ipfs-grpc-client/src/index.js b/packages/ipfs-grpc-client/src/index.js new file mode 100644 index 0000000000..3b0c5d2088 --- /dev/null +++ b/packages/ipfs-grpc-client/src/index.js @@ -0,0 +1,40 @@ +'use strict' + +const transport = require('./grpc/transport') +const toUrlString = require('ipfs-core-utils/src/to-url-string') +const loadServices = require('./utils/load-services') +const { grpc } = require('@improbable-eng/grpc-web') +grpc.setDefaultTransport(transport()) + +const service = loadServices() + +const protocols = { + 'ws://': 'http://', + 'wss://': 'https://' +} + +function normaliseUrls (opts) { + Object.keys(protocols).forEach(protocol => { + if (opts.url.startsWith(protocol)) { + opts.url = protocols[protocol] + opts.url.substring(protocol.length) + } + }) +} + +module.exports = function createClient (opts = {}) { + opts.url = toUrlString(opts.url) + + // @improbable-eng/grpc-web requires http:// protocol URLs, not ws:// + normaliseUrls(opts) + + const client = { + addAll: require('./core-api/add-all')(grpc, service.Root.add, opts), + id: require('./core-api/id')(grpc, service.Root.id, opts), + files: { + ls: require('./core-api/files/ls')(grpc, service.MFS.ls, opts), + write: require('./core-api/files/write')(grpc, service.MFS.write, opts) + } + } + + return client +} diff --git a/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js b/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js new file mode 100644 index 0000000000..2bb0c16ab6 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/bidi-to-duplex.js @@ -0,0 +1,69 @@ +'use strict' + +const pushable = require('it-pushable') +const errCode = require('err-code') +const toHeaders = require('./to-headers') + +async function sendMessages (service, client, source) { + for await (const obj of source) { + client.send({ + serializeBinary: () => service.requestType.serializeBinary(obj) + }) + } +} + +/** + * Bidirectional streams are many-to-many operations so returns a sink + * for the caller to write client messages into and a source to read + * server messages from. + * + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {{ source: AsyncIterable, sink: { push: Function, end: Function } }} + **/ +module.exports = function bidiToDuplex (grpc, service, options) { + // @ts-ignore + const source = pushable() + + // @ts-ignore + const sink = pushable() + + const client = grpc.client(service, options) + client.onMessage(message => { + sink.push(message) + }) + client.onEnd((status, message, trailers) => { + let err + + if (status) { + const error = new Error(message) + + err = errCode(error, trailers.get('grpc-code')[0], { + status + }) + + err.stack = trailers.get('grpc-stack')[0] || error.stack + } + + sink.end(err) + }) + + sendMessages(service, client, source) + .catch(err => { + sink.end(err) + }) + .finally(() => { + client.finishSend() + }) + + client.start(toHeaders(options.metadata)) + + return { + sink: source, + source: sink + } +} diff --git a/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js b/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js new file mode 100644 index 0000000000..ba107f23a1 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/client-stream-to-promise.js @@ -0,0 +1,32 @@ +'use strict' + +const first = require('it-first') +const bidiToDuplex = require('./bidi-to-duplex') + +/** + * Client streaming methods are a many-to-one operation so this + * function takes a source that can emit multiple messages and + * returns a promise that resolves to the server response. + * + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {AsyncIterable} source - a source of objects to send + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {Promise} - A promise that resolves to a response object + **/ +module.exports = async function clientStreamToPromise (grpc, service, source, options) { + const { + source: serverSource, sink + } = bidiToDuplex(grpc, service, options) + + for await (const obj of source) { + sink.push(obj) + } + + sink.end() + + return first(serverSource) +} diff --git a/packages/ipfs-grpc-client/src/utils/load-services.js b/packages/ipfs-grpc-client/src/utils/load-services.js new file mode 100644 index 0000000000..d929136ad6 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/load-services.js @@ -0,0 +1,91 @@ +'use strict' + +// @ts-ignore +const protocol = require('ipfs-grpc-protocol') +const protobuf = require('protobufjs/light') +const { Service } = protobuf + +const CONVERSION_OPTS = { + keepCase: false, + longs: String, // long.js is required + enums: String, + defaults: false, + oneofs: true +} + +/** + * Converts protobufjs service definitions into the format expected + * by @improbable-eng/grpc-web. This is to let us use the same + * service definition on both the server and the client. + * + * @returns {object} + */ +module.exports = function loadServices () { + const root = protobuf.Root.fromJSON(protocol) + const output = {} + + Object + // @ts-ignore + .keys(root.nested.ipfs) + // @ts-ignore + .filter(key => root.nested.ipfs[key] instanceof Service) + // @ts-ignore + .map(key => root.nested.ipfs[key]) + .forEach(service => { + const serviceDef = {} + + output[service.name] = serviceDef + + Object.keys(service.methods) + .forEach(methodName => { + const method = service.methods[methodName].resolve() + + serviceDef[methodName] = { + service: { + serviceName: `ipfs.${service.name}` + }, + methodName, + requestStream: method.requestStream, + responseStream: method.responseStream, + requestType: { + serializeBinary: (obj) => { + const message = method.resolvedRequestType.fromObject(obj) + return method.resolvedRequestType.encode(message).finish() + }, + deserializeBinary: (buf) => { + const message = method.resolvedRequestType.decode(buf) + const obj = method.resolvedRequestType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + } + }, + responseType: { + serializeBinary: (obj) => { + const message = method.resolvedResponseType.fromObject(obj) + return method.resolvedResponseType.encode(message).finish() + }, + deserializeBinary: (buf) => { + const message = method.resolvedResponseType.decode(buf) + const obj = method.resolvedResponseType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + } + } + } + }) + }) + + return output +} diff --git a/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js b/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js new file mode 100644 index 0000000000..295408755e --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/server-stream-to-iterator.js @@ -0,0 +1,27 @@ +'use strict' + +const bidiToDuplex = require('./bidi-to-duplex') + +/** + * Server stream methods are one-to-many operations so this + * function accepts a client message and returns a source + * from which multiple server messages can be read. + * + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {object} request - a request object + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {AsyncIterable} + **/ +module.exports = function serverStreamToIterator (grpc, service, request, options) { + const { + source, sink + } = bidiToDuplex(grpc, service, options) + + sink.push(request) + + return source +} diff --git a/packages/ipfs-grpc-client/src/utils/to-headers.js b/packages/ipfs-grpc-client/src/utils/to-headers.js new file mode 100644 index 0000000000..9cffb3c766 --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/to-headers.js @@ -0,0 +1,21 @@ +'use strict' + +const { paramCase } = require('change-case') + +/** + * @param {object} [object] - key/value pairs to turn into HTTP headers + * @returns {object} - HTTP headers + **/ +module.exports = (object) => { + const output = {} + + Object.keys(object || {}).forEach(key => { + if (typeof object[key] === 'function') { + return + } + + output[paramCase(key)] = object[key] + }) + + return output +} diff --git a/packages/ipfs-grpc-client/src/utils/unary-to-promise.js b/packages/ipfs-grpc-client/src/utils/unary-to-promise.js new file mode 100644 index 0000000000..7404d9e8ec --- /dev/null +++ b/packages/ipfs-grpc-client/src/utils/unary-to-promise.js @@ -0,0 +1,28 @@ +'use strict' + +const first = require('it-first') +const bidiToDuplex = require('./bidi-to-duplex') + +/** + * Unary calls are one-to-one operations so this function + * takes a client message and returns a promise that resolves + * to the server response. + * + * @param {object} grpc - an @improbable-eng/grpc-web instance + * @param {object} service - an @improbable-eng/grpc-web service + * @param {object} request - a request object + * @param {object} options - RPC options + * @param {string} options.host - The remote host + * @param {boolean} [options.debug] - Whether to print debug messages + * @param {object} [options.metadata] - Metadata sent as headers + * @returns {Promise} - A promise that resolves to a response object + **/ +module.exports = function unaryToPromise (grpc, service, request, options) { + const { + source, sink + } = bidiToDuplex(grpc, service, options) + + sink.push(request) + + return first(source) +} diff --git a/packages/ipfs-grpc-client/test/utils.spec.js b/packages/ipfs-grpc-client/test/utils.spec.js new file mode 100644 index 0000000000..6bdf5ddfba --- /dev/null +++ b/packages/ipfs-grpc-client/test/utils.spec.js @@ -0,0 +1,96 @@ +/* eslint-env mocha */ +'use strict' + +const { expect } = require('aegir/utils/chai') +const all = require('it-all') +const bidiToDuplex = require('../src/utils/bidi-to-duplex') +const toHeaders = require('../src/utils/to-headers') +const sinon = require('sinon') + +describe('utils', () => { + describe('bidi-to-duplex', () => { + it('should transform a bidirectional client into an async iterable', async () => { + const service = 'service' + const options = { + metadata: { + foo: 'bar' + } + } + + const client = { + onMessage: sinon.stub(), + onEnd: sinon.stub(), + start: sinon.stub() + } + + const grpc = { + client: sinon.stub().withArgs(service, options).returns(client) + } + + const { + source + } = bidiToDuplex(grpc, service, options) + + expect(client.start.calledWith(options.metadata)).to.be.true() + + client.onMessage.getCall(0).args[0]('hello') + client.onMessage.getCall(0).args[0]('world') + client.onEnd.getCall(0).args[0]() + + await expect(all(source)).to.eventually.deep.equal(['hello', 'world']) + }) + + it('should propagate client errors', async () => { + const service = 'service' + const options = { + metadata: { + foo: 'bar' + } + } + + const client = { + onMessage: sinon.stub(), + onEnd: sinon.stub(), + start: sinon.stub() + } + + const grpc = { + client: sinon.stub().withArgs(service, options).returns(client) + } + + const { + source + } = bidiToDuplex(grpc, service, options) + + expect(client.start.calledWith(options.metadata)).to.be.true() + + client.onEnd.getCall(0).args[0](1, 'Erp!', { get: () => [] }) + + await expect(all(source)).to.eventually.be.rejectedWith(/Erp!/) + }) + }) + + describe('to-headers', () => { + it('should rename property fields', () => { + const input = { + propSimple: 'foo' + } + + const output = toHeaders(input) + + expect(output.propSimple).to.be.undefined() + expect(output['prop-simple']).to.equal(input.propSimple) + }) + + it('should remove function fields', () => { + const input = { + funcProp: () => {} + } + + const output = toHeaders(input) + + expect(output.funcProp).to.be.undefined() + expect(output['func-prop']).to.be.undefined() + }) + }) +}) diff --git a/packages/ipfs-grpc-client/tsconfig.json b/packages/ipfs-grpc-client/tsconfig.json new file mode 100644 index 0000000000..a42a1aac5d --- /dev/null +++ b/packages/ipfs-grpc-client/tsconfig.json @@ -0,0 +1,18 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "package.json" + ], + "references": [ + { + "path": "../ipfs-core" + }, + { + "path": "../ipfs-core-utils" + } + ] +} diff --git a/packages/ipfs-grpc-protocol/README.md b/packages/ipfs-grpc-protocol/README.md new file mode 100644 index 0000000000..75e7e478de --- /dev/null +++ b/packages/ipfs-grpc-protocol/README.md @@ -0,0 +1,3 @@ +# ipfs-grpc-protocol + +Contains `.proto` files that define the IPFS gRPC interface. diff --git a/packages/ipfs-grpc-protocol/package.json b/packages/ipfs-grpc-protocol/package.json new file mode 100644 index 0000000000..ca7c9d1275 --- /dev/null +++ b/packages/ipfs-grpc-protocol/package.json @@ -0,0 +1,31 @@ +{ + "name": "ipfs-grpc-protocol", + "version": "0.0.0", + "description": "Protobuf definitions for the IPFS gRPC API", + "keywords": [ + "ipfs" + ], + "main": "dist/ipfs.json", + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-grpc-protocol#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "test": "echo 'No tests here'", + "clean": "rimraf ./dist", + "build": "npm run clean && mkdirp ./dist && pbjs ./src/*.proto -t json -o ./dist/ipfs.json" + }, + "devDependencies": { + "mkdirp": "^1.0.4", + "protobufjs": "^6.10.2", + "rimraf": "^3.0.2" + } +} diff --git a/packages/ipfs-grpc-protocol/src/common.proto b/packages/ipfs-grpc-protocol/src/common.proto new file mode 100644 index 0000000000..175f672e77 --- /dev/null +++ b/packages/ipfs-grpc-protocol/src/common.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package ipfs; + +enum FileType { + DIRECTORY = 0; + FILE = 1; +} diff --git a/packages/ipfs-grpc-protocol/src/mfs.proto b/packages/ipfs-grpc-protocol/src/mfs.proto new file mode 100644 index 0000000000..8a9652933e --- /dev/null +++ b/packages/ipfs-grpc-protocol/src/mfs.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +import "common.proto"; + +package ipfs; + +service MFS { + rpc ls (LsRequest) returns (stream LsResponse) {} + rpc write (stream WriteRequest) returns (WriteResponse) {} +} + +message LsRequest { + string path = 1; +} + +message LsResponse { + string name = 1; + FileType type = 2; + uint32 size = 3; + string cid = 4; + uint32 mode = 5; + int32 mtime = 6; + uint32 mtime_nsecs = 7; +} + +message WriteRequest { + string path = 1; + bytes content = 2; +} + +message WriteResponse { + +} diff --git a/packages/ipfs-grpc-protocol/src/root.proto b/packages/ipfs-grpc-protocol/src/root.proto new file mode 100644 index 0000000000..c5eca6a18c --- /dev/null +++ b/packages/ipfs-grpc-protocol/src/root.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +import "common.proto"; + +package ipfs; + +service Root { + rpc id (IdRequest) returns (IdResponse) {} + rpc add (stream AddRequest) returns (stream AddResponse) {} +} + +enum AddResponseType { + PROGRESS = 0; + RESULT = 1; +} + +message IdRequest { + string peer_id = 1; +} + +message IdResponse { + string id = 1; + string public_key = 2; + repeated string addresses = 3; + string agent_version = 4; + string protocol_version = 5; + repeated string protocols = 6; +} + +message AddRequest { + int32 index = 1; + FileType type = 2; + string path = 3; + uint32 mode = 4; + int32 mtime = 5; + uint32 mtime_nsecs = 6; + bytes content = 7; +} + +message AddResponse { + AddResponseType type = 1; + string path = 2; + int32 bytes = 3; + string cid = 4; + uint32 mode = 5; + int32 mtime = 6; + uint32 mtime_nsecs = 7; + uint32 size = 8; +} diff --git a/packages/ipfs-grpc-server/README.md b/packages/ipfs-grpc-server/README.md new file mode 100644 index 0000000000..183dc516fa --- /dev/null +++ b/packages/ipfs-grpc-server/README.md @@ -0,0 +1,182 @@ +# ipfs-grpc-server + +> A gRPC server that runs over a websocket + +## Why? + +[gRPC-web](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md) allows us to form HTTP requests out of gRPC invocations, but the [official implementation](https://github.com/grpc/grpc-web) supports only unary calls and server streaming, which in terms of functionality doesn't give us much over the existing [ipfs-http-client](https://www.npmjs.com/package/ipfs-http-client). + +In order to support streaming file uploads with errors, pubsub, etc, bi-directional streaming is required. We can either use Websockets for this, or use two connections, one for upload and one for download though this involves two requests for every operation and some orchestration on the server side to match one up with the other. + +Websockets are a cheaper and simpler way of accomplishing the same thing though sadly the official gRPC implementation has [no plans](https://github.com/grpc/grpc-web/blob/master/doc/streaming-roadmap.md#issues-with-websockets) to implement full-duplex streaming in this way. + +This module implements a Websocket proxy for a gRPC-web server. It's a js port of the [grpcwebproxy](https://github.com/improbable-eng/grpc-web/tree/master/go/grpcwebproxy) project from [improbable-eng/grpc-web](https://github.com/improbable-eng/grpc-web). + +## Protocol + +Every RPC invocation opens a new WebSocket connection, the invocation is completed and the socket connection is closed. + +The connection is opened against the path of the RPC method the client wishes to invoke. The path is created from the protobuf service definition package, service and procedure name. + +E.g. given the following service definition: + +```protobuf +package ipfs; + +service Root { + rpc id (Req) returns (Res) {} +} +``` + +A path of `/ipfs.Root/id` would be created. + +There are three parts to the communication, metadata, messages and trailers. Communication is symmetrical; that is, the client sends metadata, one or more messages and finally some trailers and the server responds with metadata, one or more messages and finally some trailers. + +The amount of messages a client/server can send is dictated by if the RPC method is unary or streaming and if so in which direction. + +Unary will result in one message sent and one received, client streaming is many sent and one received, server streaming is one sent and many received and finally bidirectional is many sent and many received. + +### 1. Metadata + +Metadata is sent as the first websocket message. It is a utf8 encoded list in the same format as [HTTP Headers][] + +### 2. Messages + +One ore more messages will be sent. Messages are sent as a single websocket message and contain a signal, a header, some message data and an optional trailer. + +Every message sent to or received from the server will have the following format: + +| byte index | Notes | +|---|---| +| 0 | Signal | +| 1-5 | Header | +| n1-n2 | Message data | +| n3-n3+5 | Trailer + +#### Signal + +A one-byte field. + +| Value | Meaning | +|---|---| +| 0 | START_SEND: Further messages will be sent as part of this context | +| 1 | FINISH_SEND: This is the final message, no further data will be sent | + +#### Header + +A five-byte field that contains one byte signifying if it's a Header or a Trailer and four bytes that contain the length of the following data. + +| byte index | Meaning | +|--------------|---| +| 0 | 0: This is a header, 128: This is a footer | +| 1-4 | An unsigned big-endian 32-bit integer that specifies the length of the message | + +#### Message data + +A protocol buffer message, the length of which is defined in the header + +#### Trailer + +A five-byte field that contains one byte signifying if it's a Header or a Trailer and four bytes that contain the length of the following data. + +| byte index | Meaning | +|--------------|---| +| 0 | 0: This is a header, 128: This is a footer | +| 1-4 | A big-endian 32-bit integer that specifies the length of the trailer | + +The trailer contains [HTTP headers][] as a utf8 encoded string in the same way as invocation metadata. + +## Handlers + +Method handlers come in four flavours - unary, server streaming, client streaming, bidirectional streaming and accept metadata as an argument. + +### Metadata + +All methods accept metadata which are sent as the equivalent of HTTP headers as part of every request. These are accepted by the client as options to a given method. + +E.g.: + +```js +ipfs.addAll(source, options) +// `source` will be turned into a message stream +// `options` will be sent as metadata +``` + +### Unary + +The simplest case, one request message and one response message. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function handler (request, metadata) { + const response = { + //... some fields here + } + + return response + } + + return handler +} +``` + +### Server streaming + +Where the server sends multiple messages. `sink` is an [it-pushable][]. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function serverStreamingHandler (request, sink, metadata) { + sink.push(..) + sink.push(..) + + sink.end() + } + + return clientStreamingHandler +} +``` + +### Client streaming + +Where the client sends multiple messages. `source` is an [AsyncIterator][]. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function clientStreamingHandler (source, metadata) { + const response = { + //... some fields here + } + + for await (const thing of source) { + // do something with `thing` + } + + return response + } + + return handler +} +``` + +### Bidirectional streaming + +Where the client and the server both send multiple messages. `source` is an [AsyncIterator][] and `sink` is an [it-pushable][]. + +```javascript +module.exports = function grpcFunction (ipfs, options = {}) { + async function bidirectionalHandler (source, sink, metadata) { + for await (const thing of source) { + sink.push(sink) + } + + sink.end() + } + + return bidirectionalHandler +} +``` + +[HTTP headers]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers +[it-pushable]: https://www.npmjs.com/package/it-pushable +[AsyncIterator]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Symbol/asyncIterator diff --git a/packages/ipfs-grpc-server/package.json b/packages/ipfs-grpc-server/package.json new file mode 100644 index 0000000000..117050de84 --- /dev/null +++ b/packages/ipfs-grpc-server/package.json @@ -0,0 +1,63 @@ +{ + "name": "ipfs-grpc-server", + "version": "0.0.0", + "description": "A server library for the IPFS gRPC API", + "keywords": [ + "ipfs" + ], + "homepage": "https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs-grpc-server#readme", + "bugs": "https://github.com/ipfs/js-ipfs/issues", + "license": "(Apache-2.0 OR MIT)", + "leadMaintainer": "Alex Potsides ", + "files": [ + "src", + "dist" + ], + "main": "src/index.js", + "browser": {}, + "typesVersions": { + "*": { + "*": [ + "dist/*", + "dist/*/index" + ] + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/js-ipfs.git" + }, + "scripts": { + "lint": "aegir lint", + "build": "npm run build:types", + "build:types": "tsc --build", + "test": "aegir test -t node", + "coverage": "nyc --reporter=text --reporter=lcov npm run test:node", + "clean": "rimraf ./dist", + "dep-check": "aegir dep-check -i ipfs-grpc-protocol -i typescript -i aegir -i ipfs-core -i rimraf" + }, + "dependencies": { + "@grpc/grpc-js": "^1.1.8", + "protobufjs": "^6.10.2", + "change-case": "^4.1.1", + "coercer": "^1.1.2", + "debug": "^4.1.1", + "ipfs-grpc-protocol": "0.0.0", + "it-first": "^1.0.4", + "it-map": "^1.0.4", + "it-peekable": "^1.0.1", + "it-pipe": "^1.1.0", + "it-pushable": "^1.4.0", + "ws": "^7.3.1" + }, + "devDependencies": { + "aegir": "^29.2.2", + "ipfs-core": "^0.2.1", + "it-all": "^1.0.4", + "it-drain": "^1.0.3", + "rimraf": "^3.0.2", + "sinon": "^9.0.3", + "typescript": "4.0.x", + "uint8arrays": "^1.1.0" + } +} diff --git a/packages/ipfs-grpc-server/src/endpoints/add.js b/packages/ipfs-grpc-server/src/endpoints/add.js new file mode 100644 index 0000000000..5f39f01413 --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/add.js @@ -0,0 +1,108 @@ +'use strict' + +// @ts-ignore +const pushable = require('it-pushable') +const { pipe } = require('it-pipe') +const encodeMtime = require('../utils/encode-mtime') + +module.exports = function grpcAdd (ipfs, options = {}) { + async function add (source, sink, metadata) { + const opts = { + ...metadata, + progress: (bytes = 0, path = '') => { + sink.push({ + type: 'PROGRESS', + bytes, + path + }) + } + } + + await pipe( + async function * toInput () { + // @ts-ignore + const fileInputStream = pushable() + + setTimeout(async () => { + const streams = [] + + try { + for await (const { index, type, path, mode, mtime, mtimeNsecs, content } of source) { + let mtimeObj + + if (mtime != null) { + mtimeObj = { + secs: mtime, + nsecs: undefined + } + + if (mtimeNsecs != null) { + mtimeObj.nsecs = mtimeNsecs + } + } + + if (!type || type === 'DIRECTORY') { + // directory + fileInputStream.push({ + path, + mode: mode !== 0 ? mode : undefined, + mtime: mtimeObj + }) + + continue + } + + let stream = streams[index] + + if (!stream) { + // start of new file + // @ts-ignore + stream = streams[index] = pushable() + + fileInputStream.push({ + path, + mode: mode !== 0 ? mode : undefined, + mtime: mtimeObj, + content: stream + }) + } + + if (content && content.length) { + // file is in progress + stream.push(content) + } else { + // file is finished + stream.end() + + streams[index] = null + } + } + + fileInputStream.end() + } catch (err) { + fileInputStream.end(err) + } finally { + // clean up any open streams + streams.filter(Boolean).forEach(stream => stream.end()) + } + }, 0) + + yield * fileInputStream + }, + async function (source) { + for await (const result of ipfs.addAll(source, opts)) { + sink.push({ + ...result, + type: 'RESULT', + cid: result.cid.toString(), + ...encodeMtime(result.mtime) + }) + } + + sink.end() + } + ) + } + + return add +} diff --git a/packages/ipfs-grpc-server/src/endpoints/id.js b/packages/ipfs-grpc-server/src/endpoints/id.js new file mode 100644 index 0000000000..8bdd210fce --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/id.js @@ -0,0 +1,16 @@ +'use strict' + +const { callbackify } = require('util') + +module.exports = function grpcId (ipfs, options = {}) { + function id (request, metadata) { + const opts = { + ...request, + ...metadata + } + + return ipfs.id(opts) + } + + return callbackify(id) +} diff --git a/packages/ipfs-grpc-server/src/endpoints/mfs/ls.js b/packages/ipfs-grpc-server/src/endpoints/mfs/ls.js new file mode 100644 index 0000000000..1432553d93 --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/mfs/ls.js @@ -0,0 +1,24 @@ +'use strict' + +const encodeMtime = require('../../utils/encode-mtime') + +module.exports = function grpcMfsLs (ipfs, options = {}) { + async function mfsLs (request, sink, metadata) { + const opts = { + ...metadata + } + + for await (const result of ipfs.files.ls(request.path, opts)) { + sink.push({ + ...result, + cid: result.cid.toString(), + type: result.type.toUpperCase(), + ...encodeMtime(result.mtime) + }) + } + + sink.end() + } + + return mfsLs +} diff --git a/packages/ipfs-grpc-server/src/endpoints/mfs/write.js b/packages/ipfs-grpc-server/src/endpoints/mfs/write.js new file mode 100644 index 0000000000..92eb61b600 --- /dev/null +++ b/packages/ipfs-grpc-server/src/endpoints/mfs/write.js @@ -0,0 +1,38 @@ +'use strict' + +const peekable = require('it-peekable') +const map = require('it-map') +const { callbackify } = require('util') + +module.exports = function grpcMfsWrite (ipfs, options = {}) { + async function mfsWrite (source, metadata) { + const opts = { + ...metadata + } + + if (opts.mtime) { + opts.mtime = { + secs: opts.mtime, + nsecs: opts.mtimeNsecs + } + } + + // path is sent with content messages + const content = peekable(source) + const result = await content.peek() + const { + value: { + // @ts-ignore + path + } + } = result + content.push(result.value) + + // @ts-ignore + await ipfs.files.write(path, map(content, ({ content }) => content), opts) + + return {} + } + + return callbackify(mfsWrite) +} diff --git a/packages/ipfs-grpc-server/src/index.js b/packages/ipfs-grpc-server/src/index.js new file mode 100644 index 0000000000..1a98a79950 --- /dev/null +++ b/packages/ipfs-grpc-server/src/index.js @@ -0,0 +1,111 @@ +'use strict' + +const grpc = require('@grpc/grpc-js') +const first = require('it-first') +const debug = require('debug')('ipfs:grpc-server') +const webSocketServer = require('./utils/web-socket-server') +const loadServices = require('./utils/load-services') + +const { + Root, + MFS +} = loadServices() + +module.exports = async function createServer (ipfs, options = {}) { + options = options || {} + + const server = new grpc.Server() + server.addService(Root, { + add: require('./endpoints/add')(ipfs, options), + // @ts-ignore + id: require('./endpoints/id')(ipfs, options) + }) + server.addService(MFS, { + ls: require('./endpoints/mfs/ls')(ipfs, options), + // @ts-ignore + write: require('./endpoints/mfs/write')(ipfs, options) + }) + + const socket = options.socket || await webSocketServer(ipfs, options) + + socket.on('error', (error) => debug(error)) + + socket.on('data', async ({ path, metadata, channel }) => { + // @ts-ignore + const handler = server.handlers.get(path) + + if (!handler) { + channel.end(new Error(`Request path ${path} unimplemented`)) + return + } + + channel.handler = handler + + switch (handler.type) { + case 'bidi': + handler.func(channel.source, channel.sink, metadata) + .catch(err => { + channel.end(err) + }) + + channel.sendMetadata({}) + + for await (const output of channel.sink) { + channel.sendMessage(output) + } + + channel.end() + + break + case 'unary': + handler.func(await first(channel.source), metadata, (err, value, metadata, flags) => { + if (err) { + return channel.end(err) + } + + channel.sendMetadata(metadata || {}) + + if (value) { + channel.sendMessage(value) + } + + channel.end() + }) + break + case 'clientStream': + handler.func(channel.source, metadata, (err, value, metadata, flags) => { + if (err) { + return channel.end(err) + } + + channel.sendMetadata(metadata || {}) + + if (value) { + channel.sendMessage(value) + } + + channel.end() + }) + break + case 'serverStream': + handler.func(await first(channel.source), channel.sink, metadata) + .catch(err => { + channel.end(err) + }) + + channel.sendMetadata({}) + + for await (const output of channel.sink) { + channel.sendMessage(output) + } + + channel.end() + + break + default: + debug(`Invalid handler type ${handler.type}`) + } + }) + + return socket +} diff --git a/packages/ipfs-grpc-server/src/utils/encode-mtime.js b/packages/ipfs-grpc-server/src/utils/encode-mtime.js new file mode 100644 index 0000000000..12d9bb0eb9 --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/encode-mtime.js @@ -0,0 +1,26 @@ +'use strict' + +function encodeMtime (mtime) { + const output = {} + + if (!mtime) { + return output + } + + const { + secs, + nsecs + } = mtime + + if (secs != null) { + output.mtime = secs + + if (nsecs != null) { + output.mtime_nsecs = nsecs + } + } + + return output +} + +module.exports = encodeMtime diff --git a/packages/ipfs-grpc-server/src/utils/load-services.js b/packages/ipfs-grpc-server/src/utils/load-services.js new file mode 100644 index 0000000000..c901d4e1d0 --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/load-services.js @@ -0,0 +1,77 @@ +'use strict' + +// @ts-ignore +const protocol = require('ipfs-grpc-protocol') +const protobuf = require('protobufjs/light') +const { Service } = protobuf + +const CONVERSION_OPTS = { + keepCase: false, + longs: String, // long.js is required + enums: String, + defaults: false, + oneofs: true +} + +module.exports = function loadServices () { + const root = protobuf.Root.fromJSON(protocol) + const output = {} + + Object + // @ts-ignore + .keys(root.nested.ipfs) + // @ts-ignore + .filter(key => root.nested.ipfs[key] instanceof Service) + // @ts-ignore + .map(key => root.nested.ipfs[key]) + .forEach(service => { + const serviceDef = {} + + output[service.name] = serviceDef + + Object.keys(service.methods) + .forEach(methodName => { + const method = service.methods[methodName].resolve() + + serviceDef[methodName] = { + path: `/ipfs.${service.name}/${methodName}`, + requestStream: method.requestStream, + responseStream: method.responseStream, + requestSerialize: (obj) => { + const message = method.resolvedRequestType.fromObject(obj) + return method.resolvedRequestType.encode(message).finish() + }, + requestDeserialize: (buf) => { + const message = method.resolvedRequestType.decode(buf) + const obj = method.resolvedRequestType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + }, + responseSerialize: (obj) => { + const message = method.resolvedResponseType.fromObject(obj) + return method.resolvedResponseType.encode(message).finish() + }, + responseDeserialize: (buf) => { + const message = method.resolvedResponseType.decode(buf) + const obj = method.resolvedResponseType.toObject(message, CONVERSION_OPTS) + + Object.defineProperty(obj, 'toObject', { + enumerable: false, + configurable: false, + value: () => obj + }) + + return obj + } + } + }) + }) + + return output +} diff --git a/packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js b/packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js new file mode 100644 index 0000000000..1322f94b7a --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/web-socket-message-channel.js @@ -0,0 +1,139 @@ +'use strict' + +// @ts-ignore +const pushable = require('it-pushable') +const { paramCase } = require('change-case') + +const WebsocketSignal = { + START_SEND: 0, + FINISH_SEND: 1 +} + +const HEADER_SIZE = 5 +const TRAILER_BYTES = 0x80 + +/** + * @param {object} object - key/value pairs to turn into HTTP headers + * @returns {Uint8Array} - HTTP headers + **/ +const objectToHeaders = (object) => { + const output = {} + + Object.keys(object).forEach(key => { + if (typeof object[key] === 'function') { + return + } + + output[paramCase(key)] = object[key] + }) + + return Buffer.from( + Object.entries(object) + .filter(([, value]) => value != null) + .map(([key, value]) => `${key}: ${JSON.stringify(value)}`) + .join('\r\n') + ) +} + +class WebsocketMessageChannel { + constructor (ws) { + this._ws = ws + + this.handler = { + deserialize: (buf) => ({}), + serialize: (message) => Buffer.from([]) + } + + // @ts-ignore + this.source = pushable() + + // @ts-ignore + this.sink = pushable() + + ws.on('message', (buf) => { + const flag = buf[0] + + if (flag === WebsocketSignal.FINISH_SEND) { + this.source.end() + + return + } + + let offset = 1 + + if (buf.length < (HEADER_SIZE + offset)) { + return + } + + const header = buf.slice(offset, HEADER_SIZE + offset) + const length = header.readUInt32BE(1, 4) + offset += HEADER_SIZE + + if (buf.length < (length + offset)) { + return + } + + const message = buf.slice(offset, offset + length) + const deserialized = this.handler.deserialize(message) + this.source.push(deserialized) + }) + + ws.once('end', () => { + this.source.end() + this.sink.end() + }) + } + + sendMetadata (metadata) { + this._ws.send(objectToHeaders(metadata)) + } + + /** + * @param {object} message - A message object to send to the client + * @returns {void} + */ + sendMessage (message) { + const response = this.handler.serialize(message) + + const header = new DataView(new ArrayBuffer(HEADER_SIZE)) + header.setUint32(1, response.byteLength) + + this._ws.send( + Buffer.concat([ + new Uint8Array(header.buffer, header.byteOffset, header.byteLength), + response + ], header.byteLength + response.byteLength) + ) + + this.sendTrailer() + } + + sendTrailer (err) { + const trailerBuffer = objectToHeaders({ + 'grpc-status': err ? 1 : 0, + 'grpc-message': err ? err.message : undefined, + 'grpc-stack': err ? err.stack : undefined, + 'grpc-code': err ? err.code : undefined + }) + + const trailer = new DataView(new ArrayBuffer(HEADER_SIZE)) + trailer.setUint8(0, TRAILER_BYTES) + trailer.setUint32(1, trailerBuffer.byteLength) + + this._ws.send( + Buffer.concat([ + new Uint8Array(trailer.buffer, trailer.byteOffset, trailer.byteLength), + trailerBuffer + ], trailer.byteLength + trailerBuffer.byteLength) + ) + } + + end (err) { + this.sendTrailer(err) + this.source.end() + this.sink.end() + this._ws.close() + } +} + +module.exports = WebsocketMessageChannel diff --git a/packages/ipfs-grpc-server/src/utils/web-socket-server.js b/packages/ipfs-grpc-server/src/utils/web-socket-server.js new file mode 100644 index 0000000000..85415aef8b --- /dev/null +++ b/packages/ipfs-grpc-server/src/utils/web-socket-server.js @@ -0,0 +1,90 @@ +'use strict' + +const { Server: WebSocketServer } = require('ws') +const EventEmitter = require('events').EventEmitter +const WebSocketMessageChannel = require('./web-socket-message-channel') +const debug = require('debug')('ipfs:grpc-server:utils:web-socket-server') +const coerce = require('coercer') +const { camelCase } = require('change-case') + +/** + * @param {Buffer} buf - e.g. `Buffer.from('foo-bar: baz\r\n')` + * @returns {object} - e.g. `{ foorBar: 'baz' }` + **/ +const fromHeaders = (buf) => { + const headers = buf.toString('utf8') + .trim() + .split('\r\n') + .map(s => s.split(':').map(s => s.trim())) + .reduce((acc, curr) => { + if (curr[0] !== 'content-type' && curr[0] !== 'x-grpc-web') { + acc[camelCase(curr[0])] = curr[1] + } + + return acc + }, {}) + + return coerce(headers) +} + +class Messages extends EventEmitter { + constructor (wss) { + super() + + this._wss = wss + this.multiaddr = '' + + wss.on('connection', (ws, request) => { + ws.on('error', error => debug(`WebSocket Error: ${error.stack}`)) + + ws.once('message', (buf) => { + const path = request.url + const metadata = fromHeaders(buf) + const channel = new WebSocketMessageChannel(ws) + + this.emit('data', { + path, + metadata, + channel + }) + }) + }) + + wss.on('error', error => this.emit('error', error)) + } + + stop () { + return new Promise((resolve) => { + this._wss.close(() => resolve()) + }) + } + + ready () { + return new Promise((resolve) => { + this._wss.on('listening', () => { + this.info = this._wss.address() + this.info.uri = `http://${this.info.address}:${this.info.port}` + this.multiaddr = `/ip4/${this._wss.address().address}/tcp/${this._wss.address().port}/ws` + + resolve(this) + }) + }) + } +} + +module.exports = async (ipfs, options = {}) => { + const config = await ipfs.config.getAll() + const grpcAddr = config.Addresses.RPC + const [,, host, , port] = grpcAddr.split('/') + + debug(`starting ws server on ${host}:${port}`) + + const wss = new WebSocketServer({ + host, + port + }) + + const messages = new Messages(wss) + + return messages.ready() +} diff --git a/packages/ipfs-grpc-server/test/add.spec.js b/packages/ipfs-grpc-server/test/add.spec.js new file mode 100644 index 0000000000..5262a62118 --- /dev/null +++ b/packages/ipfs-grpc-server/test/add.spec.js @@ -0,0 +1,95 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('./utils/server') +const { expect } = require('aegir/utils/chai') +const uint8ArrayFromString = require('uint8arrays/from-string') +const all = require('it-all') +const drain = require('it-drain') + +describe('Root.add', () => { + let ipfs + let socket + + before(() => { + ipfs = { + addAll: sinon.stub() + } + socket = server({ ipfs }) + }) + + it('should add files', async () => { + const path1 = '/path/file-1.txt' + const cid1 = 'cid-1' + const path2 = '/path/file-1.txt' + const cid2 = 'cid-2' + + const results = [{ + type: 'RESULT', + path: path1, + cid: cid1 + }, { + type: 'RESULT', + path: path2, + cid: cid2 + }] + + ipfs.addAll.returns(results) + + const requests = [ + { index: 1, type: 'FILE', path: path1, content: uint8ArrayFromString('hello world') }, + { index: 1, type: 'FILE', path: path1 }, + { index: 2, type: 'FILE', path: path2, content: uint8ArrayFromString('hello world') }, + { index: 2, type: 'FILE', path: path2 } + ] + + const channel = socket.send('/ipfs.Root/add', {}) + requests.forEach(request => channel.clientSend(request)) + channel.clientEnd() + + await expect(all(channel.clientSink)).to.eventually.deep.equal(results) + }) + + it('should propagate error when adding files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.addAll.throws(err) + + const channel = socket.send('/ipfs.Root/add', {}) + channel.clientSend({ + index: 1, + type: 'DIRECTORY', + path + }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) + + it('should propagate async error when adding files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.addAll.returns(async function * () { + yield { + type: 'file', + name: 'name', + cid: 'cid' + } + await Promise.resolve(true) + throw err + }()) + + const channel = socket.send('/ipfs.Root/add', {}) + channel.clientSend({ + index: 1, + type: 'DIRECTORY', + path + }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/id.spec.js b/packages/ipfs-grpc-server/test/id.spec.js new file mode 100644 index 0000000000..761170e853 --- /dev/null +++ b/packages/ipfs-grpc-server/test/id.spec.js @@ -0,0 +1,65 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('./utils/server') +const { expect } = require('aegir/utils/chai') +const all = require('it-all') +const drain = require('it-drain') + +describe('Root.id', () => { + let ipfs + let socket + + beforeEach(() => { + ipfs = { + id: sinon.stub() + } + socket = server({ ipfs }) + }) + + it('should get the node id', async () => { + const id = 'hello world ' + Math.random() + + ipfs.id.withArgs({}).resolves(id) + + const channel = socket.send('/ipfs.Root/id', {}) + channel.clientSend({}) + channel.clientEnd() + + const messages = await all(channel.clientSink) + expect(messages).to.have.lengthOf(1) + expect(messages).to.have.nested.property('[0]', id) + }) + + it('should get a different node id', async () => { + const peerId = 'peer-id ' + Math.random() + const id = 'hello world ' + Math.random() + + ipfs.id.withArgs({ + peerId + }).resolves(id) + + const channel = socket.send('/ipfs.Root/id', {}) + channel.clientSend({ + peerId + }) + channel.clientEnd() + + const messages = await all(channel.clientSink) + expect(messages).to.have.lengthOf(1) + expect(messages).to.have.nested.property('[0]', id) + }) + + it('should propagate error when getting id', async () => { + const err = new Error('halp!') + + ipfs.id.rejects(err) + + const channel = socket.send('/ipfs.Root/id', {}) + channel.clientSend({}) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/mfs/ls.spec.js b/packages/ipfs-grpc-server/test/mfs/ls.spec.js new file mode 100644 index 0000000000..37aca5fd4f --- /dev/null +++ b/packages/ipfs-grpc-server/test/mfs/ls.spec.js @@ -0,0 +1,80 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('../utils/server') +const { expect } = require('aegir/utils/chai') +const all = require('it-all') +const drain = require('it-drain') + +describe('MFS.ls', () => { + let ipfs + let socket + + before(() => { + ipfs = { + files: { + ls: sinon.stub() + } + } + socket = server({ ipfs }) + }) + + it('should list files', async () => { + const path = '/path' + const results = [{ + type: 'file', + name: 'file', + cid: 'cid-1' + }, { + type: 'directory', + name: 'dir', + cid: 'cid-2' + }] + + ipfs.files.ls.withArgs(path).returns(results) + + const channel = socket.send('/ipfs.MFS/ls', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(all(channel.clientSink)).to.eventually.deep.equal(results.map(result => ({ + ...result, + type: result.type.toUpperCase() + }))) + }) + + it('should propagate error when listing files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.files.ls.withArgs(path).throws(err) + + const channel = socket.send('/ipfs.MFS/ls', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) + + it('should propagate async error when listing files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.files.ls.withArgs(path).returns(async function * () { + yield { + type: 'file', + name: 'name', + cid: 'cid' + } + await Promise.resolve(true) + throw err + }()) + + const channel = socket.send('/ipfs.MFS/ls', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/mfs/write.spec.js b/packages/ipfs-grpc-server/test/mfs/write.spec.js new file mode 100644 index 0000000000..7bc9bb9343 --- /dev/null +++ b/packages/ipfs-grpc-server/test/mfs/write.spec.js @@ -0,0 +1,46 @@ +/* eslint-env mocha */ +'use strict' + +const sinon = require('sinon') +const server = require('../utils/server') +const { expect } = require('aegir/utils/chai') +const drain = require('it-drain') + +describe('MFS.write', () => { + let ipfs + let socket + + before(() => { + ipfs = { + files: { + write: sinon.stub() + } + } + socket = server({ ipfs }) + }) + + it('should write a file', async () => { + const path = '/path' + + const channel = socket.send('/ipfs.MFS/write', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await drain(channel.clientSink) + + expect(ipfs.files.write.calledWith(path)).to.be.true() + }) + + it('should propagate error when writing files', async () => { + const path = '/path' + const err = new Error('halp!') + + ipfs.files.write.withArgs(path).throws(err) + + const channel = socket.send('/ipfs.MFS/write', {}) + channel.clientSend({ path }) + channel.clientEnd() + + await expect(drain(channel.clientSink)).to.eventually.be.rejectedWith(/halp!/) + }) +}) diff --git a/packages/ipfs-grpc-server/test/utils/channel.js b/packages/ipfs-grpc-server/test/utils/channel.js new file mode 100644 index 0000000000..7258fb62a4 --- /dev/null +++ b/packages/ipfs-grpc-server/test/utils/channel.js @@ -0,0 +1,48 @@ +'use strict' + +const pushable = require('it-pushable') + +class MessageChannel { + constructor () { + this.source = pushable() + this.sink = pushable() + + this.clientSink = pushable() + } + + sendMetadata (metadata) { + this.metadata = metadata + } + + sendMessage (message) { + setTimeout(() => { + this.clientSink.push(message) + }, 0) + } + + sendTrailers (trailers) { + this.trailers = trailers + } + + end (error) { + setTimeout(() => { + this.clientSink.end(error) + }, 0) + } + + clientSend (message) { + setTimeout(() => { + this.source.push(message) + }, 0) + } + + clientEnd (err) { + setTimeout(() => { + this.source.end(err) + }, 0) + } +} + +module.exports = () => { + return new MessageChannel() +} diff --git a/packages/ipfs-grpc-server/test/utils/server.js b/packages/ipfs-grpc-server/test/utils/server.js new file mode 100644 index 0000000000..d2b0b45945 --- /dev/null +++ b/packages/ipfs-grpc-server/test/utils/server.js @@ -0,0 +1,23 @@ +'use strict' + +const createServer = require('../../src') +const EventEmitter = require('events').EventEmitter +const createChannel = require('./channel') + +module.exports = ({ ipfs, options }) => { + const socket = new EventEmitter() + + createServer(ipfs, { + socket + }) + + return { + send: (path, metadata) => { + const channel = createChannel() + + socket.emit('data', { path, metadata, channel }) + + return channel + } + } +} diff --git a/packages/ipfs-grpc-server/tsconfig.json b/packages/ipfs-grpc-server/tsconfig.json new file mode 100644 index 0000000000..979a39adab --- /dev/null +++ b/packages/ipfs-grpc-server/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "package.json" + ] +} diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 2566d4e57c..988be4cb83 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -80,7 +80,7 @@ "aegir": "^29.2.2", "go-ipfs": "^0.7.0", "ipfs-core": "^0.3.1", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "it-all": "^1.0.4", "it-concat": "^1.0.1", "nock": "^13.0.2", diff --git a/packages/ipfs/.aegir.js b/packages/ipfs/.aegir.js index 622a7ac601..4a3cf8c709 100644 --- a/packages/ipfs/.aegir.js +++ b/packages/ipfs/.aegir.js @@ -15,7 +15,7 @@ let sigServerB let ipfsdServer module.exports = { - bundlesize: { maxSize: '530kB' }, + bundlesize: { maxSize: '560kB' }, karma: { files: [{ pattern: 'node_modules/interface-ipfs-core/test/fixtures/**/*', @@ -86,6 +86,9 @@ module.exports = { }, { go: { ipfsBin: require('go-ipfs').path() + }, + js: { + ipfsClientModule: require('ipfs-client') } }).start() diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 7c65fc8a14..05e2dca59a 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -28,6 +28,7 @@ "build:types": "tsc --build", "test": "echo 'Only interface tests live here'", "test:interface:core": "aegir test -f test/interface-core.js", + "test:interface:client": "aegir test -f test/interface-client.js", "test:interface:http-js": "aegir test -f test/interface-http-js.js", "test:interface:http-go": "aegir test -f test/interface-http-go.js", "test:interop": "cross-env IPFS_JS_EXEC=$PWD/src/cli.js IPFS_JS_MODULE=$PWD IPFS_JS_HTTP_MODULE=$PWD/../ipfs-http-client IPFS_REUSEPORT=false ipfs-interop", @@ -50,10 +51,11 @@ "electron-webrtc": "^0.3.0", "go-ipfs": "^0.7.0", "interface-ipfs-core": "^0.142.3", + "ipfs-client": "^0.1.0", "ipfs-http-client": "^48.1.3", "ipfs-interop": "^3.0.0", "ipfs-utils": "^5.0.0", - "ipfsd-ctl": "^7.1.1", + "ipfsd-ctl": "^7.2.0", "iso-url": "^1.0.0", "libp2p-webrtc-star": "^0.20.1", "merge-options": "^2.0.0", diff --git a/packages/ipfs/test/interface-client.js b/packages/ipfs/test/interface-client.js new file mode 100644 index 0000000000..ff6ae19223 --- /dev/null +++ b/packages/ipfs/test/interface-client.js @@ -0,0 +1,147 @@ +/* eslint-env mocha, browser */ +'use strict' + +const tests = require('interface-ipfs-core') +const { isNode } = require('ipfs-utils/src/env') +const factory = require('./utils/factory') + +describe('interface-ipfs-core ipfs-client tests', () => { + const commonFactory = factory({ + type: 'js', + ipfsClientModule: require('ipfs-client') + }) + + tests.files(commonFactory, { + skip: [{ + name: '.files.chmod', + reason: 'not implemented' + }, { + name: '.files.cp', + reason: 'not implemented' + }, { + name: '.files.mkdir', + reason: 'not implemented' + }, { + name: '.files.stat', + reason: 'not implemented' + }, { + name: '.files.touch', + reason: 'not implemented' + }, { + name: '.files.rm', + reason: 'not implemented' + }, { + name: '.files.read', + reason: 'not implemented' + }, { + name: '.files.mv', + reason: 'not implemented' + }, { + name: '.files.flush', + reason: 'not implemented' + }].concat(isNode ? [] : [{ + name: 'should make directory and specify mtime as hrtime', + reason: 'Not designed to run in the browser' + }, { + name: 'should set mtime as hrtime', + reason: 'Not designed to run in the browser' + }, { + name: 'should write file and specify mtime as hrtime', + reason: 'Not designed to run in the browser' + }]) + }) + + tests.root(commonFactory, { + skip: [ + { + name: 'add', + reason: 'not implemented' + }, + { + name: 'should add with only-hash=true', + reason: 'ipfs.object.get is not implemented' + }, + { + name: 'should add a directory with only-hash=true', + reason: 'ipfs.object.get is not implemented' + }, + { + name: 'should add with mtime as hrtime', + reason: 'process.hrtime is not a function in browser' + }, + { + name: 'should add from a URL with only-hash=true', + reason: 'ipfs.object.get is not implemented' + }, + { + name: 'should cat with a Uint8Array multihash', + reason: 'Passing CID as Uint8Array is not supported' + }, + { + name: 'should add from a HTTP URL', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a HTTP URL with redirection', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a URL with only-hash=true', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a URL with wrap-with-directory=true', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should add from a URL with wrap-with-directory=true and URL-escaped file name', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should not add from an invalid url', + reason: 'https://github.com/ipfs/js-ipfs/issues/3195' + }, + { + name: 'should be able to add dir without sharding', + reason: 'Cannot spawn IPFS with different args' + }, + { + name: 'with sharding', + reason: 'TODO: allow spawning new daemons with different config' + }, + { + name: 'get', + reason: 'Not implemented' + }, + { + name: 'refs', + reason: 'Not implemented' + }, + { + name: 'refsLocal', + reason: 'Not implemented' + } + ] + }) + + tests.miscellaneous(commonFactory, { + skip: [ + { + name: '.dns', + reason: 'Not implemented' + }, + { + name: '.resolve', + reason: 'Not implemented' + }, + { + name: '.stop', + reason: 'Not implemented' + }, + { + name: '.version', + reason: 'Not implemented' + } + ] + }) +}) diff --git a/packages/ipfs/test/interface-http-go.js b/packages/ipfs/test/interface-http-go.js index 6d56c3800b..565875f09a 100644 --- a/packages/ipfs/test/interface-http-go.js +++ b/packages/ipfs/test/interface-http-go.js @@ -58,6 +58,10 @@ describe('interface-ipfs-core over ipfs-http-client tests against go-ipfs', () = name: 'should override raw leaves when file is smaller than one block and metadata is present', reason: 'TODO not implemented in go-ipfs yet' }, + { + name: 'should add directories with metadata', + reason: 'TODO not implemented in go-ipfs yet' + }, { name: 'should support bidirectional streaming', reason: 'Not supported by http' diff --git a/tsconfig.json b/tsconfig.json index a5f6d1a969..d065abadd8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -35,6 +35,9 @@ "ipfs-cli/*": [ "ipfs-cli/*" ], + "ipfs-client/*": [ + "ipfs-client/*" + ], "ipfs-core/*": [ "ipfs-core/*" ], @@ -44,6 +47,12 @@ "ipfs-daemon/*": [ "ipfs-daemon/*" ], + "ipfs-grpc-client/*": [ + "ipfs-grpc-client/*" + ], + "ipfs-grpc-server/*": [ + "ipfs-grpc-server/*" + ], "ipfs-http-client/*": [ "ipfs-http-client/*" ],