feat: add mfs implementation

Author: achingbrain

README.md

@@ -57,6 +57,7 @@ You can check the development status at the [Waffle Board](https://waffle.io/ipf
   - [Use in Node.js](#use-in-nodejs)
   - [Through command line tool](#through-command-line-tool)
   - [Use in the browser](#use-in-the-browser)
+  - [Use with Web Workers](#use-with-web-workers)
 - [Usage](#usage)
   - [IPFS CLI](#ipfs-cli)
   - [IPFS Daemon](#ipfs-daemon)
@@ -135,6 +136,20 @@ You can also load it using a `<script>` using the [unpkg](https://unpkg.com) CDN
 ```
 Inserting one of the above lines will make an `Ipfs` object available in the global namespace.
 
+### Use with Web Workers
+
+Some of the `ipfs.files.*` [MFS](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#mutable-file-system) commands mutate the CID that corresponds to the current root of the file system. A read/write lock on the main thread prevents these mutations from conflicting. With single-process and [Node.js cluster](https://nodejs.org/api/cluster.html) based apps this is completely transparent but [Web Workers](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API) require a little extra setup.  Use the [`observable-webworkers`](https://www.npmjs.com/package/observable-webworkers) module to allow communication with any Web Workers your application creates:
+
+```javascript
+const observe = require('observable-webworkers')
+ 
+const worker = new Worker('my-worker-script.js')
+ 
+observe(worker)
+```
+
+`js-ipfs` will send extra messages to your Web Workers to orchestrate the read/write locks so please be aware of this in your Worker code.
+
 ## Usage
 
 ### IPFS CLI

examples/browser-script-tag/index.html

@@ -30,7 +30,7 @@ <h2>Some suggestions</h2>
   <code style="display:block; white-space:pre-wrap; background-color:#d7d6d6">
     node.files.add(new node.types.Buffer('Hello world!'), (err, filesAdded) => {
       if (err) {
-        return console.error('Error - ipfs files add', err, res)
+        return console.error('Error - ipfs add', err, res)
       }
 
       filesAdded.forEach((file) => console.log('successfully stored', file.hash))

examples/circuit-relaying/package.json

@@ -18,7 +18,7 @@
     "ipfs-pubsub-room": "~0.3.0"
   },
   "devDependencies": {
-    "aegir": "^13.0.5",
+    "aegir": "^14.0.0",
     "http-server": "~0.10.0",
     "ipfs-css": "~0.2.0",
     "parcel-bundler": "^1.6.2",

package.json

@@ -66,6 +66,7 @@
     "chai": "^4.1.2",
     "delay": "^2.0.0",
     "detect-node": "^2.0.3",
+    "detect-webworker": "^1.0.0",
     "dir-compare": "^1.4.0",
     "dirty-chai": "^2.0.1",
     "eslint-plugin-react": "^7.7.0",
@@ -107,11 +108,12 @@
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
     "interface-datastore": "~0.4.1",
-    "ipfs-api": "^22.1.1",
+    "ipfs-api": "^22.2.1",
     "ipfs-bitswap": "~0.20.0",
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
     "ipfs-http-response": "~0.1.2",
+    "ipfs-mfs": "~0.0.13",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.22.1",
     "ipfs-unixfs": "~0.1.15",
@@ -170,7 +172,8 @@
     "through2": "^2.0.3",
     "update-notifier": "^2.5.0",
     "yargs": "^11.0.0",
-    "yargs-parser": "^10.0.0"
+    "yargs-parser": "^10.0.0",
+    "yargs-promise": "^1.1.0"
   },
   "optionalDependencies": {
     "prom-client": "^11.0.0",

src/cli/bin.js

@@ -2,13 +2,14 @@
 
 'use strict'
 
+const YargsPromise = require('yargs-promise')
 const yargs = require('yargs')
 const updateNotifier = require('update-notifier')
 const readPkgUp = require('read-pkg-up')
-const fs = require('fs')
-const path = require('path')
 const utils = require('./utils')
 const print = utils.print
+const mfs = require('ipfs-mfs/cli')
+const debug = require('debug')('ipfs:cli')
 
 const pkg = readPkgUp.sync({cwd: __dirname}).pkg
 updateNotifier({
@@ -18,10 +19,6 @@ updateNotifier({
 
 const args = process.argv.slice(2)
 
-// Determine if the first argument is a sub-system command
-const commandNames = fs.readdirSync(path.join(__dirname, 'commands'))
-const isCommand = commandNames.includes(`${args[0]}.js`)
-
 const cli = yargs
   .option('silent', {
     desc: 'Write no output',
@@ -34,14 +31,6 @@ const cli = yargs
     type: 'string',
     default: ''
   })
-  .commandDir('commands', {
-    // Only include the commands for the sub-system we're using, or include all
-    // if no sub-system command has been passed.
-    include (path, filename) {
-      if (!isCommand) return true
-      return `${args[0]}.js` === filename
-    }
-  })
   .epilog(utils.ipfsPathHelp)
   .demandCommand(1)
   .fail((msg, err, yargs) => {
@@ -56,27 +45,15 @@ const cli = yargs
     yargs.showHelp()
   })
 
-// If not a sub-system command then load the top level aliases
-if (!isCommand) {
-  // NOTE: This creates an alias of
-  // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
-  // This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
-  const addCmd = require('./commands/files/add')
-  const catCmd = require('./commands/files/cat')
-  const getCmd = require('./commands/files/get')
-  const aliases = [addCmd, catCmd, getCmd]
-  aliases.forEach((alias) => {
-    cli.command(alias.command, alias.describe, alias.builder, alias.handler)
-  })
-}
-
 // Need to skip to avoid locking as these commands
 // don't require a daemon
 if (args[0] === 'daemon' || args[0] === 'init') {
   cli
     .help()
     .strict()
     .completion()
+    .command(require('./commands/daemon'))
+    .command(require('./commands/init'))
     .parse(args)
 } else {
   // here we have to make a separate yargs instance with
@@ -86,20 +63,58 @@ if (args[0] === 'daemon' || args[0] === 'init') {
     if (err) {
       throw err
     }
+
     utils.getIPFS(argv, (err, ipfs, cleanup) => {
-      if (err) { throw err }
+      if (err) {
+        throw err
+      }
+
+      // add mfs commands
+      mfs(cli)
+
+      // NOTE: This creates an alias of
+      // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
+      // This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
+      const addCmd = require('./commands/files/add')
+      const catCmd = require('./commands/files/cat')
+      const getCmd = require('./commands/files/get')
+      const aliases = [addCmd, catCmd, getCmd]
+      aliases.forEach((alias) => {
+        cli.command(alias)
+      })
 
       cli
+        .commandDir('commands')
         .help()
         .strict()
         .completion()
-        .parse(args, { ipfs: ipfs }, (err, argv, output) => {
-          if (output) { print(output) }
 
-          cleanup(() => {
-            if (err) { throw err }
-          })
+      let exitCode = 0
+
+      const parser = new YargsPromise(cli, { ipfs })
+      parser.parse(args)
+        .then(({ data, argv }) => {
+          if (data) {
+            print(data)
+          }
+        })
+        .catch((arg) => {
+          debug(arg)
+
+          // the argument can have a different shape depending on where the error came from
+          if (arg.message) {
+            print(arg.message)
+          } else if (arg.error && arg.error.message) {
+            print(arg.error.message)
+          } else {
+            print('Unknown error, please re-run the command with DEBUG=ipfs:cli to see debug output')
+          }
+
+          exitCode = 1
         })
+        .then(() => cleanup())
+        .catch(() => {})
+        .then(() => process.exit(exitCode))
     })
   })
 }

src/cli/commands/files.js

@@ -30,6 +30,7 @@ function fileHandler (dir) {
         callback(err)
       } else {
         const fullFilePath = path.join(dir, file.path)
+
         if (file.content) {
           file.content
             .pipe(fs.createWriteStream(fullFilePath))

src/cli/commands/files/get.js

@@ -9,6 +9,7 @@ const log = debug('cli')
 log.error = debug('cli:error')
 const Progress = require('progress')
 const byteman = require('byteman')
+const promisify = require('promisify-es6')
 
 exports = module.exports
 
@@ -40,7 +41,7 @@ function getAPICtl (apiAddr) {
 
 exports.getIPFS = (argv, callback) => {
   if (argv.api || isDaemonOn()) {
-    return callback(null, getAPICtl(argv.api), (cb) => cb())
+    return callback(null, getAPICtl(argv.api), promisify((cb) => cb()))
   }
 
   // Required inline to reduce startup time
@@ -55,13 +56,13 @@ exports.getIPFS = (argv, callback) => {
     }
   })
 
-  const cleanup = (cb) => {
+  const cleanup = promisify((cb) => {
     if (node && node._repo && !node._repo.closed) {
-      node._repo.close(() => cb())
+      node._repo.close((err) => cb(err))
     } else {
       cb()
     }
-  }
+  })
 
   node.on('error', (err) => {
     throw err

src/cli/utils.js

@@ -26,3 +26,4 @@ exports.dht = require('./dht')
 exports.dns = require('./dns')
 exports.key = require('./key')
 exports.stats = require('./stats')
+exports.mfs = require('ipfs-mfs/core')

src/core/components/index.js

@@ -133,6 +133,13 @@ class IPFS extends EventEmitter {
       isIPFS: isIPFS
     }
 
+    // ipfs.files
+    const mfs = components.mfs(this)
+
+    Object.keys(mfs).forEach(key => {
+      this.files[key] = mfs[key]
+    })
+
     boot(this)
   }
 }

src/core/index.js

@@ -34,7 +34,8 @@ exports.parseKey = (request, reply) => {
   if (!request.query.arg) {
     return reply({
       Message: "Argument 'key' is required",
-      Code: 0
+      Code: 0,
+      Type: 'error'
     }).code(400).takeover()
   }
 
@@ -54,7 +55,8 @@ exports.parseKey = (request, reply) => {
     log.error(err)
     return reply({
       Message: 'invalid ipfs ref path',
-      Code: 0
+      Code: 0,
+      Type: 'error'
     }).code(500).takeover()
   }
 
@@ -81,9 +83,9 @@ exports.cat = {
       if (err) {
         log.error(err)
         if (err.message === 'No such file') {
-          reply({Message: 'No such file'}).code(500)
+          reply({Message: 'No such file', Code: 0, Type: 'error'}).code(500)
         } else {
-          reply({Message: 'Failed to cat file: ' + err, Code: 0}).code(500)
+          reply({Message: 'Failed to cat file: ' + err, Code: 0, Type: 'error'}).code(500)
         }
         return
       }
@@ -177,7 +179,8 @@ exports.add = {
     if (!request.payload) {
       return reply({
         Message: 'Array, Buffer, or String is required.',
-        code: 0
+        Code: 0,
+        Type: 'error'
       }).code(400).takeover()
     }
 
@@ -211,7 +214,8 @@ exports.add = {
       if (!filesParsed) {
         return reply({
           Message: "File argument 'data' is required.",
-          code: 0
+          Code: 0,
+          Type: 'error'
         }).code(400).takeover()
       }
       fileAdder.end()
@@ -302,7 +306,8 @@ exports.immutableLs = {
       if (err) {
         return reply({
           Message: 'Failed to list dir: ' + err.message,
-          Code: 0
+          Code: 0,
+          Type: 'error'
         }).code(500).takeover()
       }
 

src/http/api/resources/files.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const resources = require('./../resources')
+const mfs = require('ipfs-mfs/http')
 
 module.exports = (server) => {
   const api = server.select('API')
@@ -54,4 +55,6 @@ module.exports = (server) => {
       handler: resources.files.immutableLs.handler
     }
   })
+
+  mfs(api)
 }