feat: add grpc server and client

Adds a server running a gRPC endpoint over websockets, a client to access the server and a `ipfs-client` module that uses the gRPC client with HTTP fallback.

So far only supports `ipfs.addAll` but the idea is to implement all streaming methods over websockets instead of HTTP, to give us bidirectional streaming and errors that work in the browser.

Fixes:

Depends on:

- [ ] ipfs/js-ipfsd-ctl#561

Author: achingbrain

.travis.yml

@@ -164,6 +164,41 @@ jobs:
       script:
         - npm run test:interface:core -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
 
+    - stage: test
+    name: js-ipfs interface tests - ipfs-client - node
+    script:
+      - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t node
+
+  - stage: test
+    name: js-ipfs interface tests - ipfs-client - chrome
+    script:
+      - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser
+
+  - stage: test
+    name: js-ipfs interface tests - ipfs-client - chrome webworker
+    script:
+      - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --timeout 60000
+
+  - stage: test
+    name: js-ipfs interface tests - ipfs-client - firefox
+    script:
+      - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t browser --browsers FirefoxHeadless
+
+  - stage: test
+    name: js-ipfs interface tests - ipfs-client - firefox webworker
+    script:
+      - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t webworker --browsers FirefoxHeadless --timeout 60000
+
+  - stage: test
+    name: js-ipfs interface tests - ipfs-client - electron main
+    script:
+      - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-main --timeout 60000
+
+  - stage: test
+    name: js-ipfs interface tests - ipfs-client - electron renderer
+    script:
+      - npm run test:interface:client -- $RUN_SINCE -- -- --bail -t electron-renderer --timeout 60000
+
     - stage: test
       name: http-api-client interface tests vs go-ipfs - node
       script:

examples/browser-ipns-publish/package.json

@@ -27,7 +27,7 @@
   "devDependencies": {
     "delay": "^4.4.0",
     "execa": "^4.0.3",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "go-ipfs": "^0.7.0",
     "parcel-bundler": "^1.12.4",
     "path": "^0.12.7",

examples/explore-ethereum-blockchain/package.json

@@ -12,7 +12,7 @@
   "devDependencies": {
     "ipfs": "^0.52.1",
     "ipfs-http-client": "^48.1.1",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "ipld-ethereum": "^5.0.1",
     "test-ipfs-example": "^2.0.3"
   }

examples/http-client-browser-pubsub/package.json

@@ -21,7 +21,7 @@
     "execa": "^4.0.3",
     "go-ipfs": "^0.7.0",
     "ipfs": "^0.52.1",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "parcel-bundler": "^1.12.4",
     "test-ipfs-example": "^2.0.3"
   }

examples/http-client-bundle-webpack/package.json

@@ -25,7 +25,7 @@
     "copy-webpack-plugin": "^5.0.4",
     "execa": "^4.0.3",
     "ipfs": "^0.52.1",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "react-hot-loader": "^4.12.21",
     "test-ipfs-example": "^2.0.3",
     "webpack": "^4.43.0",

examples/http-client-name-api/package.json

@@ -18,7 +18,7 @@
   "devDependencies": {
     "execa": "^4.0.3",
     "go-ipfs": "^0.7.0",
-    "ipfsd-ctl": "^7.0.2",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
     "parcel-bundler": "^1.12.4",
     "test-ipfs-example": "^2.0.3"
   },

examples/ipfs-client-add-files/README.md

@@ -0,0 +1,21 @@
+# JS IPFS API - Example Browser - Name
+
+## Setup
+
+```sh
+npm install -g ipfs
+jsipfs init
+# Configure CORS to allow ipfs-http-client to access this IPFS node
+jsipfs config --json API.HTTPHeaders.Access-Control-Allow-Origin '["http://127.0.0.1:8888"]'
+# Start the IPFS node
+jsipfs daemon
+```
+
+Then in this folder run
+
+```bash
+> npm install
+> npm start
+```
+
+and open your browser at `http://127.0.0.1:8888`.

examples/ipfs-client-add-files/index.html

@@ -0,0 +1,36 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>JS IPFS Client example</title>
+    <style>
+      .hidden {
+        opacity: 0;
+      }
+
+      form {
+        padding-bottom: 1em;
+      }
+    </style>
+  </head>
+
+  <body>
+    <h1>ipfs-client</h1>
+    <form id="connect-to-api">
+      <h3>Enter IPFS API details</h3>
+      <label for="grpc-input">
+        GRPC:
+        <input id="grpc-input" name="grpc-input" type="text" value="/ip4/127.0.0.1/tcp/5003" required>
+      </label>
+      <label for="http-input">
+        HTTP:
+        <input id="http-input" name="text" type="text" value="/ip4/127.0.0.1/tcp/5001" required>
+      </label>
+      <button id="connect-submit" type="submit">Connect</button>
+    </form>
+    <div id="output">
+    </div>
+
+    <script src="index.js"></script>
+  </body>
+</html>

examples/ipfs-client-add-files/index.js

@@ -0,0 +1,81 @@
+/* eslint-disable no-console */
+'use strict'
+
+const ipfsClient = require('ipfs-client')
+let ipfs
+
+const COLORS = {
+  active: 'blue',
+  success: 'green',
+  error: 'red'
+}
+
+const showStatus = (text, bg) => {
+  console.info(text)
+
+  const log = document.getElementById('output')
+
+  if (!log) {
+    return
+  }
+
+  const line = document.createElement('p')
+  line.innerText = text
+  line.style.color = bg
+
+  log.appendChild(line)
+}
+
+async function * streamFiles () {
+  for (let i = 0; i < 100; i++) {
+    await new Promise((resolve) => {
+      setTimeout(() => resolve(), 100)
+    })
+
+    showStatus(`Sending /file-${i}.txt`, COLORS.active)
+
+    yield {
+      path: `/file-${i}.txt`,
+      content: `file ${i}`
+    }
+  }
+}
+
+async function main (grpcApi, httpApi) {
+  showStatus(`Connecting to ${grpcApi} using ${httpApi} as fallback`, COLORS.active)
+
+  ipfs = ipfsClient({
+    grpc: grpcApi,
+    http: httpApi
+  })
+
+  const id = await ipfs.id()
+  showStatus(`Daemon active\nID: ${id.id}`, COLORS.success)
+
+  for await (const file of ipfs.addAll(streamFiles(), {
+    wrapWithDirectory: true,
+    // this is just to show the interleaving of uploads and progress events
+    // otherwise we'd have to upload 50 files before we see any response from
+    // the server. do not specify this so low in production as you'll have
+    // greatly degraded import performance
+    fileImportConcurrency: 1,
+    progress: (bytes, file) => {
+      showStatus(`File progress ${file} ${bytes}`, COLORS.active)
+    }
+  })) {
+    showStatus(`Added file: ${file.path} ${file.cid}`, COLORS.success)
+  }
+
+  showStatus('Finished!', COLORS.success)
+}
+
+// Event listeners
+document.getElementById('connect-submit').onclick = (e) => {
+  e.preventDefault()
+
+  main(document.getElementById('grpc-input').value, document.getElementById('http-input').value)
+    .catch(err => {
+      showStatus(err.message, COLORS.error)
+      console.error(err)
+    })
+}

examples/ipfs-client-add-files/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "example-ipfs-client-add-files",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "private": true,
+  "scripts": {
+    "clean": "rm -rf ./dist",
+    "build": "parcel build index.html --public-url '.'",
+    "start": "parcel index.html -p 8888",
+    "test": "test-ipfs-example"
+  },
+  "dependencies": {
+    "ipfs-client": "^0.1.0"
+  },
+  "devDependencies": {
+    "execa": "^4.0.3",
+    "ipfs": "^0.52.0",
+    "ipfsd-ctl": "ipfs/js-ipfsd-ctl#feat/expose-grpc-addr",
+    "parcel-bundler": "^1.12.4",
+    "test-ipfs-example": "^2.0.3"
+  },
+  "browserslist": [
+    "last 2 versions and not dead and > 2%"
+  ]
+}

examples/ipfs-client-add-files/test.js

@@ -0,0 +1,82 @@
+'use strict'
+
+const path = require('path')
+const execa = require('execa')
+const { createFactory } = require('ipfsd-ctl')
+const df = createFactory({
+  ipfsClientModule: require('ipfs-client'),
+  ipfsBin: require.resolve('ipfs/src/cli.js')
+})
+const {
+  startServer
+} = require('test-ipfs-example/utils')
+const pkg = require('./package.json')
+
+async function testUI (url, http, grpc, id) {
+  const proc = execa(require.resolve('test-ipfs-example/node_modules/.bin/nightwatch'), ['--config', require.resolve('test-ipfs-example/nightwatch.conf.js'), path.join(__dirname, 'test.js')], {
+    cwd: path.resolve(__dirname, '../'),
+    env: {
+      ...process.env,
+      CI: true,
+      IPFS_EXAMPLE_TEST_URL: url,
+      IPFS_GRPC_API_MULTIADDR: grpc,
+      IPFS_HTTP_API_MULTIADDR: http
+    },
+    all: true
+  })
+  proc.all.on('data', (data) => {
+    process.stdout.write(data)
+  })
+
+  await proc
+}
+
+async function runTest () {
+  const app = await startServer(__dirname)
+  const daemon = await df.spawn({
+    type: 'js',
+    test: true,
+    ipfsOptions: {
+      config: {
+        Addresses: {
+          API: '/ip4/127.0.0.1/tcp/0',
+          RPC: '/ip4/127.0.0.1/tcp/0'
+        },
+        API: {
+          HTTPHeaders: {
+            'Access-Control-Allow-Origin': [
+              app.url
+            ]
+          }
+        }
+      }
+    }
+  })
+
+  try {
+    await testUI(app.url, daemon.apiAddr, daemon.grpcAddr, daemon.api.peerId.id)
+  } finally {
+    await daemon.stop()
+    await app.stop()
+  }
+}
+
+module.exports = runTest
+
+module.exports[pkg.name] = function (browser) {
+  browser
+    .url(process.env.IPFS_EXAMPLE_TEST_URL)
+    .waitForElementVisible('#grpc-input')
+    .clearValue('#grpc-input')
+    .setValue('#grpc-input', process.env.IPFS_GRPC_API_MULTIADDR)
+    .pause(1000)
+    .waitForElementVisible('#http-input')
+    .clearValue('#http-input')
+    .setValue('#http-input', process.env.IPFS_HTTP_API_MULTIADDR)
+    .pause(1000)
+    .click('#connect-submit')
+
+  browser.expect.element('#output').text.to.contain('Added file: file-0.txt QmUDLiEJwL3vUhhXNXDF2RrCnVkSB2LemWYffpCCPcQCeU')
+
+  browser.end()
+}

packages/interface-ipfs-core/src/add-all.js

@@ -420,5 +420,58 @@ module.exports = (common, options) => {
       expect(files[0].cid.codec).to.equal('dag-pb')
       expect(files[0].size).to.equal(18)
     })
+
+    it('should support bidirectional streaming', async function () {
+      let progressInvoked
+
+      const handler = (bytes, path) => {
+        progressInvoked = true
+      }
+
+      const source = async function * () {
+        yield {
+          content: 'hello',
+          path: '/file'
+        }
+
+        await new Promise((resolve) => {
+          const interval = setInterval(() => {
+            // we've received a progress result, that means we've received some
+            // data from the server before we're done sending data to the server
+            // so the streaming is bidirectional and we can finish up
+            if (progressInvoked) {
+              clearInterval(interval)
+              resolve()
+            }
+          }, 10)
+        })
+      }
+
+      await drain(ipfs.addAll(source(), {
+        progress: handler,
+        fileImportConcurrency: 1
+      }))
+
+      expect(progressInvoked).to.be.true()
+    })
+
+    it('should error during add-all stream', async function () {
+      const source = async function * () {
+        yield {
+          content: 'hello',
+          path: '/file'
+        }
+
+        yield {
+          content: 'hello',
+          path: '/file'
+        }
+      }
+
+      await expect(drain(ipfs.addAll(source(), {
+        fileImportConcurrency: 1,
+        chunker: 'rabin-2048--50' // invalid chunker parameters, validated after the stream starts moving
+      }))).to.eventually.be.rejectedWith(/Chunker parameter avg must be an integer/)
+    })
   })
 }