feat: interoperable DHT (#856)

* feat: dht ready

* test: enable dht core test

* feat: add dht cli

* chore: enable dht by default

* fix: code review

* chore: increase silent timeout even more

* fix: code review

* fix: ci tests

* fix: code review

* fix: make ci happy again

Author: daviddias

README.md

@@ -34,11 +34,6 @@ You can check the development status at the [Kanban Board](https://waffle.io/ipf
 
 [![Throughput Graph](https://graphs.waffle.io/ipfs/js-ipfs/throughput.svg)](https://waffle.io/ipfs/js-ipfs/metrics/throughput)
 
-**Please read this:** DHT (automatic content discovery) and Circuit Relay (pierce through NATs and dial between any node in the network) are two fundamental pieces that are not finalized yet. There are multiple applications that can be built without these two services but nevertheless they are fundamental to get that magic IPFS experience. If you want to track progress or contribute, please follow:
-
-- DHT: https://github.com/ipfs/js-ipfs/pull/856
-- ✅ Relay: https://github.com/ipfs/js-ipfs/pull/1063
-
 [**`Weekly Core Dev Calls`**](https://github.com/ipfs/pm/issues/650)
 
 ## Tech Lead
@@ -319,7 +314,6 @@ Enable and configure experimental features.
 - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`)
 - `ipnsPubsub` (boolean): Enable pub-sub on IPNS. (Default: `false`)
 - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`)
-- `dht` (boolean): Enable KadDHT. **This is currently not interoperable with `go-ipfs`.**
 
 ##### `options.config`
 
@@ -600,7 +594,13 @@ The core API is grouped into several areas:
   - [`ipfs.bootstrap.add(addr, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BOOTSTRAP.md#bootstrapadd)
   - [`ipfs.bootstrap.rm(peer, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BOOTSTRAP.md#bootstraprm)
 
-- dht (not implemented yet)
+- [dht](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/)
+  - [`ipfs.dht.findPeer(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindpeer)
+  - [`ipfs.dht.findProvs(multihash, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindprovs)
+  - [`ipfs.dht.get(key, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtget)
+  - [`ipfs.dht.provide(cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtprovide)
+  - [`ipfs.dht.put(key, value, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtput)
+  - [`ipfs.dht.query(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtquery)
 
 - [pubsub](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md)
   - [`ipfs.pubsub.subscribe(topic, handler, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubsubscribe)

package.json

@@ -129,7 +129,7 @@
     "joi": "^14.3.0",
     "joi-browser": "^13.4.0",
     "joi-multiaddr": "^4.0.0",
-    "libp2p": "~0.24.1",
+    "libp2p": "~0.25.0-rc.0",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.16.0",
     "libp2p-kad-dht": "~0.14.4",
@@ -141,7 +141,7 @@
     "libp2p-tcp": "~0.13.0",
     "libp2p-webrtc-star": "~0.15.5",
     "libp2p-websocket-star-multi": "~0.4.0",
-    "libp2p-websockets": "~0.12.0",
+    "libp2p-websockets": "~0.12.2",
     "lodash": "^4.17.11",
     "mafmt": "^6.0.2",
     "mime-types": "^2.1.21",

src/cli/commands/daemon.js

@@ -18,10 +18,6 @@ module.exports = {
         type: 'boolean',
         default: false
       })
-      .option('enable-dht-experiment', {
-        type: 'boolean',
-        default: false
-      })
       .option('offline', {
         desc: 'Run offline. Do not connect to the rest of the network but provide local API.',
         default: false

src/cli/commands/dht.js

@@ -0,0 +1,14 @@
+'use strict'
+
+module.exports = {
+  command: 'dht <command>',
+
+  description: 'Issue commands directly through the DHT.',
+
+  builder (yargs) {
+    return yargs.commandDir('dht')
+  },
+
+  handler (argv) {
+  }
+}

src/cli/commands/dht/find-peer.js

@@ -0,0 +1,23 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'findpeer <peerID>',
+
+  describe: 'Find the multiaddresses associated with a Peer ID.',
+
+  builder: {},
+
+  handler ({ getIpfs, peerID, resolve }) {
+    resolve((async () => {
+      const ipfs = await getIpfs()
+      const peers = await ipfs.dht.findPeer(peerID)
+      const addresses = peers.multiaddrs.toArray().map((ma) => ma.toString())
+
+      addresses.forEach((addr) => {
+        print(addr)
+      })
+    })())
+  }
+}

src/cli/commands/dht/find-providers.js

@@ -0,0 +1,33 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'findprovs <key>',
+
+  describe: 'Find peers that can provide a specific value, given a key.',
+
+  builder: {
+    'num-providers': {
+      alias: 'n',
+      describe: 'The number of providers to find. Default: 20.',
+      default: 20
+    }
+  },
+
+  handler (argv) {
+    const { getIpfs, key, resolve } = argv
+    const opts = {
+      maxNumProviders: argv['num-providers']
+    }
+
+    resolve((async () => {
+      const ipfs = await getIpfs()
+      const provs = await ipfs.dht.findProvs(key, opts)
+
+      provs.forEach((element) => {
+        print(element.id.toB58String())
+      })
+    })())
+  }
+}

src/cli/commands/dht/get.js

@@ -0,0 +1,20 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'get <key>',
+
+  describe: 'Given a key, query the routing system for its best value.',
+
+  builder: {},
+
+  handler ({ getIpfs, key, resolve }) {
+    resolve((async () => {
+      const ipfs = await getIpfs()
+      const value = await ipfs.dht.get(key)
+
+      print(value)
+    })())
+  }
+}

src/cli/commands/dht/provide.js

@@ -0,0 +1,26 @@
+'use strict'
+
+module.exports = {
+  command: 'provide <key>',
+
+  describe: 'Announce to the network that you are providing given values.',
+
+  builder: {
+    recursive: {
+      alias: 'r',
+      recursive: 'Recursively provide entire graph.',
+      default: false
+    }
+  },
+
+  handler ({ getIpfs, key, recursive, resolve }) {
+    const opts = {
+      recursive
+    }
+
+    resolve((async () => {
+      const ipfs = await getIpfs()
+      await ipfs.dht.provide(key, opts)
+    })())
+  }
+}

src/cli/commands/dht/put.js

@@ -0,0 +1,16 @@
+'use strict'
+
+module.exports = {
+  command: 'put <key> <value>',
+
+  describe: 'Write a key/value pair to the routing system.',
+
+  builder: {},
+
+  handler ({ getIpfs, key, value, resolve }) {
+    resolve((async () => {
+      const ipfs = await getIpfs()
+      await ipfs.dht.put(key, value)
+    })())
+  }
+}

src/cli/commands/dht/query.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const print = require('../../utils').print
+
+module.exports = {
+  command: 'query <peerID>',
+
+  describe: 'Find the closest Peer IDs to a given Peer ID by querying the DHT.',
+
+  builder: {},
+
+  handler ({ getIpfs, peerID, resolve }) {
+    resolve((async () => {
+      const ipfs = await getIpfs()
+      const result = await ipfs.dht.query(peerID)
+
+      result.forEach((peerID) => {
+        print(peerID.id.toB58String())
+      })
+    })())
+  }
+}