feat: add mssing dag put and dag resolve cli commands (#2521)

* feat: add mssing `dag put` and `dag resolve` cli commands

Also allows passing input to `ipfs-exec` to simulate piping in
cli tests.

* chore: update interop dep

* fix: do pinning inside core with gc lock

* fix: wrap put in gc lock

Author: achingbrain

src/cli/commands/dag/put.js

@@ -0,0 +1,124 @@
+'use strict'
+
+const mh = require('multihashes')
+const multibase = require('multibase')
+const dagCBOR = require('ipld-dag-cbor')
+const dagPB = require('ipld-dag-pb')
+const { cidToString } = require('../../../utils/cid')
+
+const inputDecoders = {
+  json: (buf) => JSON.parse(buf.toString()),
+  cbor: (buf) => dagCBOR.util.deserialize(buf),
+  protobuf: (buf) => dagPB.util.deserialize(buf),
+  raw: (buf) => buf
+}
+
+const formats = {
+  cbor: 'dag-cbor',
+  raw: 'raw',
+  protobuf: 'dag-pb',
+  'dag-cbor': 'dag-cbor',
+  'dag-pb': 'dag-pb'
+}
+
+module.exports = {
+  command: 'put [data]',
+
+  describe: 'accepts input from a file or stdin and parses it into an object of the specified format',
+
+  builder: {
+    data: {
+      type: 'string'
+    },
+    format: {
+      type: 'string',
+      alias: 'f',
+      default: 'cbor',
+      describe: 'Format that the object will be added as',
+      choices: ['dag-cbor', 'dag-pb', 'raw', 'cbor', 'protobuf']
+    },
+    'input-encoding': {
+      type: 'string',
+      alias: 'input-enc',
+      default: 'json',
+      describe: 'Format that the input object will be',
+      choices: ['json', 'cbor', 'raw', 'protobuf']
+    },
+    pin: {
+      type: 'boolean',
+      default: true,
+      describe: 'Pin this object when adding'
+    },
+    'hash-alg': {
+      type: 'string',
+      alias: 'hash',
+      default: 'sha2-256',
+      describe: 'Hash function to use',
+      choices: Object.keys(mh.names)
+    },
+    'cid-version': {
+      type: 'integer',
+      describe: 'CID version. Defaults to 0 unless an option that depends on CIDv1 is passed',
+      default: 0
+    },
+    'cid-base': {
+      describe: 'Number base to display CIDs in.',
+      type: 'string',
+      choices: multibase.names
+    },
+    preload: {
+      type: 'boolean',
+      default: true,
+      describe: 'Preload this object when adding'
+    },
+    'only-hash': {
+      type: 'boolean',
+      default: false,
+      describe: 'Only hash the content, do not write to the underlying block store'
+    }
+  },
+
+  handler ({ data, format, inputEncoding, pin, hashAlg, cidVersion, cidBase, preload, onlyHash, getIpfs, print, resolve }) {
+    resolve((async () => {
+      const ipfs = await getIpfs()
+
+      if (inputEncoding === 'cbor') {
+        format = 'dag-cbor'
+      } else if (inputEncoding === 'protobuf') {
+        format = 'dag-pb'
+      }
+
+      format = formats[format]
+
+      if (format !== 'dag-pb') {
+        cidVersion = 1
+      }
+
+      let source = data
+
+      if (!source) {
+        // pipe from stdin
+        source = Buffer.alloc(0)
+
+        for await (const buf of process.stdin) {
+          source = Buffer.concat([source, buf])
+        }
+      } else {
+        source = Buffer.from(source)
+      }
+
+      source = inputDecoders[inputEncoding](source)
+
+      const cid = await ipfs.dag.put(source, {
+        format,
+        hashAlg,
+        version: cidVersion,
+        onlyHash,
+        preload,
+        pin
+      })
+
+      print(cidToString(cid, { base: cidBase }))
+    })())
+  }
+}

src/cli/commands/dag/resolve.js

@@ -0,0 +1,45 @@
+'use strict'
+
+const CID = require('cids')
+
+module.exports = {
+  command: 'resolve <ref>',
+
+  describe: 'fetches a dag node from ipfs, prints its address and remaining path',
+
+  builder: {
+    ref: {
+      type: 'string'
+    }
+  },
+
+  handler ({ ref, getIpfs, print, resolve }) {
+    resolve((async () => {
+      const ipfs = await getIpfs()
+      const options = {}
+
+      try {
+        const result = await ipfs.dag.resolve(ref, options)
+        let lastCid
+
+        for (const res of result) {
+          if (CID.isCID(res.value)) {
+            lastCid = res.value
+          }
+        }
+
+        if (!lastCid) {
+          if (ref.startsWith('/ipfs/')) {
+            ref = ref.substring(6)
+          }
+
+          lastCid = ref.split('/').shift()
+        }
+
+        print(lastCid.toString())
+      } catch (err) {
+        return print(`dag get resolve: ${err}`)
+      }
+    })())
+  }
+}

src/core/components/dag.js

@@ -6,6 +6,50 @@ const all = require('async-iterator-all')
 const errCode = require('err-code')
 const multicodec = require('multicodec')
 
+function parseArgs (cid, path, options) {
+  options = options || {}
+
+  // Allow options in path position
+  if (path !== undefined && typeof path !== 'string') {
+    options = path
+    path = undefined
+  }
+
+  if (typeof cid === 'string') {
+    if (cid.startsWith('/ipfs/')) {
+      cid = cid.substring(6)
+    }
+
+    const split = cid.split('/')
+
+    try {
+      cid = new CID(split[0])
+    } catch (err) {
+      throw errCode(err, 'ERR_INVALID_CID')
+    }
+
+    split.shift()
+
+    if (split.length > 0) {
+      path = split.join('/')
+    } else {
+      path = path || '/'
+    }
+  } else if (Buffer.isBuffer(cid)) {
+    try {
+      cid = new CID(cid)
+    } catch (err) {
+      throw errCode(err, 'ERR_INVALID_CID')
+    }
+  }
+
+  return [
+    cid,
+    path,
+    options
+  ]
+}
+
 module.exports = function dag (self) {
   return {
     put: callbackify.variadic(async (dagNode, options) => {
@@ -44,50 +88,38 @@ module.exports = function dag (self) {
         }
       }
 
-      const cid = await self._ipld.put(dagNode, options.format, {
-        hashAlg: options.hashAlg,
-        cidVersion: options.version
-      })
+      let release
 
-      if (options.preload !== false) {
-        self._preload(cid)
+      if (options.pin) {
+        release = await self._gcLock.readLock()
       }
 
-      return cid
-    }),
-
-    get: callbackify.variadic(async (cid, path, options) => {
-      options = options || {}
+      try {
+        const cid = await self._ipld.put(dagNode, options.format, {
+          hashAlg: options.hashAlg,
+          cidVersion: options.version
+        })
 
-      // Allow options in path position
-      if (path !== undefined && typeof path !== 'string') {
-        options = path
-        path = undefined
-      }
-
-      if (typeof cid === 'string') {
-        const split = cid.split('/')
-
-        try {
-          cid = new CID(split[0])
-        } catch (err) {
-          throw errCode(err, 'ERR_INVALID_CID')
+        if (options.pin) {
+          await self.pin.add(cid, {
+            lock: false
+          })
         }
 
-        split.shift()
-
-        if (split.length > 0) {
-          path = split.join('/')
-        } else {
-          path = path || '/'
+        if (options.preload !== false) {
+          self._preload(cid)
         }
-      } else if (Buffer.isBuffer(cid)) {
-        try {
-          cid = new CID(cid)
-        } catch (err) {
-          throw errCode(err, 'ERR_INVALID_CID')
+
+        return cid
+      } finally {
+        if (release) {
+          release()
         }
       }
+    }),
+
+    get: callbackify.variadic(async (cid, path, options) => {
+      [cid, path, options] = parseArgs(cid, path, options)
 
       if (options.preload !== false) {
         self._preload(cid)
@@ -116,37 +148,23 @@ module.exports = function dag (self) {
     }),
 
     tree: callbackify.variadic(async (cid, path, options) => { // eslint-disable-line require-await
-      options = options || {}
+      [cid, path, options] = parseArgs(cid, path, options)
 
-      // Allow options in path position
-      if (path !== undefined && typeof path !== 'string') {
-        options = path
-        path = undefined
+      if (options.preload !== false) {
+        self._preload(cid)
       }
 
-      if (typeof cid === 'string') {
-        const split = cid.split('/')
-
-        try {
-          cid = new CID(split[0])
-        } catch (err) {
-          throw errCode(err, 'ERR_INVALID_CID')
-        }
-
-        split.shift()
+      return all(self._ipld.tree(cid, path, options))
+    }),
 
-        if (split.length > 0) {
-          path = split.join('/')
-        } else {
-          path = undefined
-        }
-      }
+    resolve: callbackify.variadic(async (cid, path, options) => { // eslint-disable-line require-await
+      [cid, path, options] = parseArgs(cid, path, options)
 
       if (options.preload !== false) {
         self._preload(cid)
       }
 
-      return all(self._ipld.tree(cid, path, options))
+      return all(self._ipld.resolve(cid, path))
     })
   }
 }

test/cli/commands.js

@@ -4,7 +4,7 @@
 const { expect } = require('interface-ipfs-core/src/utils/mocha')
 const runOnAndOff = require('../utils/on-and-off')
 
-const commandCount = 98
+const commandCount = 100
 describe('commands', () => runOnAndOff((thing) => {
   let ipfs
 

test/cli/daemon.js

@@ -27,6 +27,10 @@ const daemonReady = (daemon) => {
         reject(new Error('Daemon didn\'t start ' + data.toString('utf8')))
       }
     })
+
+    daemon.catch(err => {
+      reject(err)
+    })
   })
 }
 const checkLock = (repo) => {
@@ -126,8 +130,8 @@ describe('daemon', () => {
     ]
 
     await ipfs('init')
-    await ipfs('config', 'Addresses.API', JSON.stringify(apiAddrs), '--json')
-    await ipfs('config', 'Addresses.Gateway', JSON.stringify(gatewayAddrs), '--json')
+    await ipfs(`config Addresses.API ${JSON.stringify(apiAddrs)} --json`)
+    await ipfs(`config Addresses.Gateway ${JSON.stringify(gatewayAddrs)} --json`)
 
     const daemon = ipfs('daemon')
     let stdout = ''
@@ -154,8 +158,8 @@ describe('daemon', () => {
     this.timeout(100 * 1000)
 
     await ipfs('init')
-    await ipfs('config', 'Addresses.API', '[]', '--json')
-    await ipfs('config', 'Addresses.Gateway', '[]', '--json')
+    await ipfs('config Addresses.API [] --json')
+    await ipfs('config Addresses.Gateway [] --json')
 
     const daemon = ipfs('daemon')
     let stdout = ''