feat: allow daemon to init and start in a single cmd (#2428)

`ipfs daemon` will now init the repo automatically if one does not exist

Author: hugomrdias

.travis.yml

@@ -40,26 +40,58 @@ jobs:
       addons:
         chrome: stable
       script:
-        - npx aegir test -t browser
-        - npx aegir test -t webworker
+        - npx aegir test -t browser -t webworker
 
     - stage: test
       name: firefox
       addons:
         firefox: latest
       script:
-        - npx aegir test -t browser -- --browsers FirefoxHeadless
-        - npx aegir test -t webworker -- --browsers FirefoxHeadless
+        - npx aegir test -t browser -t webworker -- --browsers FirefoxHeadless
 
     - stage: test
       name: electron-main
+      os: osx
       script:
-        - xvfb-run npx aegir test -t electron-main -- --bail --timeout 10000
+        - npx aegir test -t electron-main --bail --timeout 10000
 
     - stage: test
       name: electron-renderer
+      os: osx
       script:
-        - xvfb-run npx aegir test -t electron-renderer -- --bail --timeout 10000
+        - npx aegir test -t electron-renderer --bail --timeout 10000
+
+    - stage: test
+      name: interop node
+      script:
+        - cd node_modules/ipfs-interop
+        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t node --bail
+
+    - stage: test
+      name: interop browser
+      script:
+        - cd node_modules/ipfs-interop
+        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t browser --bail
+
+    - stage: test
+      name: interop electron-main
+      os: osx
+      script:
+        - cd node_modules/ipfs-interop
+        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t electron-main -f ./test/node.js --bail --timeout 10000
+
+    - stage: test
+      name: interop electron-renderer
+      os: osx
+      script:
+        - cd node_modules/ipfs-interop
+        - IPFS_JS_EXEC=./../../src/cli/bin.js IPFS_REUSEPORT=false npx aegir test -t electron-renderer -f ./test/browser.js --bail --timeout 10000
+
+    - stage: tag
+      if: branch = master AND type = push AND fork = false
+      name: update-last-successful-build
+      script:
+        - npx aegir update-last-successful-build
 
     - stage: test
       name: interop

package.json

@@ -97,15 +97,15 @@
     "ipfs-bitswap": "~0.25.1",
     "ipfs-block": "~0.8.1",
     "ipfs-block-service": "~0.15.2",
-    "ipfs-http-client": "^35.1.0",
+    "ipfs-http-client": "^37.0.2",
     "ipfs-http-response": "~0.3.1",
     "ipfs-mfs": "^0.12.2",
     "ipfs-multipart": "^0.2.0",
     "ipfs-repo": "~0.26.6",
     "ipfs-unixfs": "~0.1.16",
     "ipfs-unixfs-exporter": "~0.37.7",
     "ipfs-unixfs-importer": "~0.39.11",
-    "ipfs-utils": "~0.2.0",
+    "ipfs-utils": "~0.4.0",
     "ipld": "~0.24.1",
     "ipld-bitcoin": "~0.3.0",
     "ipld-dag-cbor": "~0.15.0",
@@ -124,7 +124,7 @@
     "it-to-stream": "^0.1.1",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
-    "ky": "~0.13.0",
+    "ky": "~0.14.0",
     "ky-universal": "~0.3.0",
     "libp2p": "~0.26.1",
     "libp2p-bootstrap": "~0.9.3",
@@ -188,7 +188,7 @@
     "yargs-promise": "^1.1.0"
   },
   "devDependencies": {
-    "aegir": "^20.1.0",
+    "aegir": "^20.3.1",
     "base64url": "^3.0.1",
     "chai": "^4.2.0",
     "clear-module": "^4.0.0",
@@ -199,16 +199,17 @@
     "execa": "^2.0.4",
     "form-data": "^2.5.1",
     "hat": "0.0.3",
-    "interface-ipfs-core": "^0.113.0",
+    "interface-ipfs-core": "~0.114.0",
     "ipfs-interop": "~0.1.0",
-    "ipfsd-ctl": "~0.46.0",
+    "ipfsd-ctl": "^0.47.2",
     "libp2p-websocket-star": "~0.10.2",
     "ncp": "^2.0.0",
     "p-event": "^4.1.0",
     "qs": "^6.5.2",
     "rimraf": "^3.0.0",
     "sinon": "^7.4.2",
-    "stream-to-promise": "^2.2.0"
+    "stream-to-promise": "^2.2.0",
+    "temp-write": "^4.0.0"
   },
   "optionalDependencies": {
     "prom-client": "^11.5.3",

src/cli/commands/daemon.js

@@ -1,9 +1,11 @@
 'use strict'
 
 const os = require('os')
+const fs = require('fs')
 const toUri = require('multiaddr-to-uri')
 const { ipfsPathHelp } = require('../utils')
 const { isTest } = require('ipfs-utils/src/env')
+const debug = require('debug')('ipfs:cli:daemon')
 
 module.exports = {
   command: 'daemon',
@@ -13,6 +15,10 @@ module.exports = {
   builder (yargs) {
     return yargs
       .epilog(ipfsPathHelp)
+      .option('init-config', {
+        type: 'string',
+        desc: 'Path to existing configuration file to be loaded during --init.'
+      })
       .option('enable-sharding-experiment', {
         type: 'boolean',
         default: false
@@ -42,9 +48,22 @@ module.exports = {
 
       const repoPath = argv.getRepoPath()
 
+      let config = {}
+      // read and parse config file
+      if (argv.initConfig) {
+        try {
+          const raw = fs.readFileSync(argv.initConfig)
+          config = JSON.parse(raw)
+        } catch (error) {
+          debug(error)
+          throw new Error('Default config couldn\'t be found or content isn\'t valid JSON.')
+        }
+      }
+
       // Required inline to reduce startup time
       const Daemon = require('../../cli/daemon')
       const daemon = new Daemon({
+        config,
         silent: argv.silent,
         repo: process.env.IPFS_PATH,
         offline: argv.offline,

src/cli/commands/init.js

@@ -1,15 +1,17 @@
 'use strict'
 
+const fs = require('fs')
+const debug = require('debug')('ipfs:cli:init')
 const { ipfsPathHelp } = require('../utils')
 
 module.exports = {
-  command: 'init [config] [options]',
+  command: 'init [default-config] [options]',
   describe: 'Initialize a local IPFS node',
   builder (yargs) {
     return yargs
       .epilog(ipfsPathHelp)
-      .positional('config', {
-        describe: 'Node config, this should JSON and will be merged with the default config. Check https://github.com/ipfs/js-ipfs#optionsconfig',
+      .positional('default-config', {
+        describe: 'Initialize with the given configuration. Path to the config file. Check https://github.com/ipfs/js-ipfs#optionsconfig',
         type: 'string'
       })
       .option('bits', {
@@ -34,6 +36,18 @@ module.exports = {
     argv.resolve((async () => {
       const path = argv.getRepoPath()
 
+      let config = {}
+      // read and parse config file
+      if (argv.defaultConfig) {
+        try {
+          const raw = fs.readFileSync(argv.defaultConfig)
+          config = JSON.parse(raw)
+        } catch (error) {
+          debug(error)
+          throw new Error('Default config couldn\'t be found or content isn\'t valid JSON.')
+        }
+      }
+
       argv.print(`initializing ipfs node at ${path}`)
 
       // Required inline to reduce startup time
@@ -44,7 +58,7 @@ module.exports = {
         repo: new Repo(path),
         init: false,
         start: false,
-        config: argv.config || {}
+        config
       })
 
       try {

src/cli/daemon.js

@@ -55,7 +55,7 @@ class Daemon {
     }
 
     // start the daemon
-    const ipfsOpts = Object.assign({ init: false }, this._options, { start: true, libp2p })
+    const ipfsOpts = Object.assign({ }, this._options, { init: true, start: true, libp2p })
     const ipfs = new IPFS(ipfsOpts)
 
     await new Promise((resolve, reject) => {

test/cli/daemon.js

@@ -9,10 +9,35 @@ const os = require('os')
 const path = require('path')
 const hat = require('hat')
 const fs = require('fs')
+const tempWrite = require('temp-write')
 const pkg = require('../../package.json')
 
 const skipOnWindows = isWindows() ? it.skip : it
+const daemonReady = (daemon, cb) => {
+  let r = null
+  const p = new Promise((resolve, reject) => {
+    daemon.stdout.on('data', async (data) => {
+      if (data.toString().includes('Daemon is ready')) {
+        try {
+          r = await cb()
+        } catch (err) {
+          reject(err)
+        }
+        daemon.cancel()
+      }
+    })
+    daemon.stderr.on('data', () => reject(new Error('Daemon didnt start')))
+    daemon.then(() => resolve(r)).catch(err => {
+      if (r && err.killed) {
+        return resolve(r)
+      }
+
+      reject(err)
+    })
+  })
 
+  return p
+}
 const checkLock = (repo) => {
   // skip on windows
   // https://github.com/ipfs/js-ipfsd-ctl/pull/155#issuecomment-326983530
@@ -190,14 +215,6 @@ describe('daemon', () => {
     checkLock(repoPath)
   })
 
-  it('gives error if user hasn\'t run init before', async function () {
-    this.timeout(100 * 1000)
-
-    const err = await ipfs.fail('daemon')
-
-    expect(err.stdout).to.include('no initialized ipfs repo found in ' + repoPath)
-  })
-
   it('should be silent', async function () {
     this.timeout(100 * 1000)
     await ipfs('init')
@@ -265,4 +282,39 @@ describe('daemon', () => {
       expect(err.stdout).to.include(`Node.js version: ${process.versions.node}`)
     }
   })
+
+  it('should init by default', async function () {
+    this.timeout(100 * 1000)
+
+    expect(fs.existsSync(repoPath)).to.be.false()
+
+    const daemon = ipfs('daemon')
+    let stdout = ''
+
+    daemon.stdout.on('data', (data) => {
+      stdout += data.toString('utf8')
+
+      if (stdout.includes('Daemon is ready')) {
+        daemon.kill()
+      }
+    })
+
+    try {
+      await daemon
+      throw new Error('Did not kill process')
+    } catch (err) {
+      expect(err.killed).to.be.true()
+    }
+
+    expect(fs.existsSync(repoPath)).to.be.true()
+  })
+
+  it('should init with custom config', async function () {
+    this.timeout(100 * 1000)
+    const configPath = tempWrite.sync('{"Addresses": {"API": "/ip4/127.0.0.1/tcp/9999"}}', 'config.json')
+    const daemon = ipfs(`daemon --init-config ${configPath}`)
+
+    const r = await daemonReady(daemon, () => ipfs('config \'Addresses.API\''))
+    expect(r).to.be.eq('/ip4/127.0.0.1/tcp/9999\n')
+  })
 })

test/cli/init.js

@@ -8,9 +8,10 @@ const clean = require('../utils/clean')
 const hat = require('hat')
 const ipfsExec = require('../utils/ipfs-exec')
 const os = require('os')
+const tempWrite = require('temp-write')
 
 describe('init', function () {
-  this.timeout(40 * 1000)
+  this.timeout(100 * 1000)
 
   let repoPath
   let ipfs
@@ -33,8 +34,6 @@ describe('init', function () {
   afterEach(() => clean(repoPath))
 
   it('basic', function () {
-    this.timeout(40 * 1000)
-
     return ipfs('init').then((out) => {
       expect(repoDirSync('blocks')).to.have.length.above(2)
       expect(repoExistsSync('config')).to.equal(true)
@@ -48,8 +47,6 @@ describe('init', function () {
   })
 
   it('bits', function () {
-    this.timeout(40 * 1000)
-
     return ipfs('init --bits 1024').then(() => {
       expect(repoDirSync('blocks')).to.have.length.above(2)
       expect(repoExistsSync('config')).to.equal(true)
@@ -58,8 +55,6 @@ describe('init', function () {
   })
 
   it('empty', function () {
-    this.timeout(40 * 1000)
-
     return ipfs('init --bits 1024 --empty-repo true').then(() => {
       expect(repoDirSync('blocks')).to.have.length(2)
       expect(repoExistsSync('config')).to.equal(true)
@@ -68,11 +63,18 @@ describe('init', function () {
   })
 
   it('should present ipfs path help when option help is received', function (done) {
-    this.timeout(100 * 1000)
-
     ipfs('init --help').then((res) => {
       expect(res).to.have.string('export IPFS_PATH=/path/to/ipfsrepo')
       done()
     })
   })
+
+  it('default config argument', () => {
+    const configPath = tempWrite.sync('{"Addresses": {"API": "/ip4/127.0.0.1/tcp/9999"}}', 'config.json')
+    return ipfs(`init ${configPath}`).then((res) => {
+      const configRaw = fs.readFileSync(path.join(repoPath, 'config')).toString()
+      const config = JSON.parse(configRaw)
+      expect(config.Addresses.API).to.be.eq('/ip4/127.0.0.1/tcp/9999')
+    })
+  })
 })

test/core/bitswap.spec.js

@@ -13,7 +13,7 @@ const waterfall = require('async/waterfall')
 const parallel = require('async/parallel')
 const Block = require('ipfs-block')
 const multiaddr = require('multiaddr')
-const isNode = require('detect-node')
+const { isNode } = require('ipfs-utils/src/env')
 const multihashing = require('multihashing-async')
 const CID = require('cids')
 const path = require('path')