chore: move mfs and multipart files into core (ipfs#2811)

BREAKING CHANGE:

When the path passed to `ipfs.files.stat(path)` was a hamt sharded dir, the resovled
value returned by js-ipfs previously had a `type` property of with a value of
`'hamt-sharded-directory'`.  To bring it in line with go-ipfs this value is now
`'directory'`.

Author: achingbrain

README.md

@@ -1016,7 +1016,6 @@ Listing of the main packages used in the IPFS ecosystem. There are also three sp
 | [`ipfs-http-client`](//github.com/ipfs/js-ipfs) | [![npm](https://img.shields.io/npm/v/ipfs-http-client.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/js-ipfs/releases) | [![Deps](https://david-dm.org/ipfs/js-ipfs.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs) | [![Travis CI](https://flat.badgen.net/travis/ipfs/js-ipfs/master)](https://travis-ci.com/ipfs/js-ipfs) | [![codecov](https://codecov.io/gh/ipfs/js-ipfs/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs) | [Alan Shaw](mailto:[email protected]) |
 | [`ipfs-http-response`](//github.com/ipfs/js-ipfs-http-response) | [![npm](https://img.shields.io/npm/v/ipfs-http-response.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/js-ipfs-http-response/releases) | [![Deps](https://david-dm.org/ipfs/js-ipfs-http-response.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs-http-response) | [![Travis CI](https://flat.badgen.net/travis/ipfs/js-ipfs-http-response/master)](https://travis-ci.com/ipfs/js-ipfs-http-response) | [![codecov](https://codecov.io/gh/ipfs/js-ipfs-http-response/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-http-response) | [Vasco Santos](mailto:[email protected]) |
 | [`ipfsd-ctl`](//github.com/ipfs/js-ipfsd-ctl) | [![npm](https://img.shields.io/npm/v/ipfsd-ctl.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/js-ipfsd-ctl/releases) | [![Deps](https://david-dm.org/ipfs/js-ipfsd-ctl.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfsd-ctl) | [![Travis CI](https://flat.badgen.net/travis/ipfs/js-ipfsd-ctl/master)](https://travis-ci.com/ipfs/js-ipfsd-ctl) | [![codecov](https://codecov.io/gh/ipfs/js-ipfsd-ctl/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfsd-ctl) | [Hugo Dias](mailto:[email protected]) |
-| [`ipfs-multipart`](//github.com/ipfs/js-ipfs) | [![npm](https://img.shields.io/npm/v/ipfs-multipart.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/js-ipfs/releases) | [![Deps](https://david-dm.org/ipfs/js-ipfs.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs) | [![Travis CI](https://flat.badgen.net/travis/ipfs/js-ipfs/master)](https://travis-ci.com/ipfs/js-ipfs) | [![codecov](https://codecov.io/gh/ipfs/js-ipfs/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs) | [Hugo Dias](mailto:[email protected]) |
 | [`is-ipfs`](//github.com/ipfs/is-ipfs) | [![npm](https://img.shields.io/npm/v/is-ipfs.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/is-ipfs/releases) | [![Deps](https://david-dm.org/ipfs/is-ipfs.svg?style=flat-square)](https://david-dm.org/ipfs/is-ipfs) | [![Travis CI](https://flat.badgen.net/travis/ipfs/is-ipfs/master)](https://travis-ci.com/ipfs/is-ipfs) | [![codecov](https://codecov.io/gh/ipfs/is-ipfs/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/ipfs/is-ipfs) | [Marcin Rataj](mailto:[email protected]) |
 | [`aegir`](//github.com/ipfs/aegir) | [![npm](https://img.shields.io/npm/v/aegir.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/aegir/releases) | [![Deps](https://david-dm.org/ipfs/aegir.svg?style=flat-square)](https://david-dm.org/ipfs/aegir) | [![Travis CI](https://flat.badgen.net/travis/ipfs/aegir/master)](https://travis-ci.com/ipfs/aegir) | [![codecov](https://codecov.io/gh/ipfs/aegir/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/ipfs/aegir) | [Hugo Dias](mailto:[email protected]) |
 | [`ipfs-repo-migrations`](//github.com/ipfs/js-ipfs-repo-migrations) | [![npm](https://img.shields.io/npm/v/ipfs-repo-migrations.svg?maxAge=86400&style=flat-square)](//github.com/ipfs/js-ipfs-repo-migrations/releases) | [![Deps](https://david-dm.org/ipfs/js-ipfs-repo-migrations.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs-repo-migrations) | [![Travis CI](https://flat.badgen.net/travis/ipfs/js-ipfs-repo-migrations/master)](https://travis-ci.com/ipfs/js-ipfs-repo-migrations) | [![codecov](https://codecov.io/gh/ipfs/js-ipfs-repo-migrations/branch/master/graph/badge.svg?style=flat-square)](https://codecov.io/gh/ipfs/js-ipfs-repo-migrations) | N/A |

package-list.json

@@ -30,7 +30,6 @@
     ["ipfs/js-ipfs", "ipfs-http-client"],
     ["ipfs/js-ipfs-http-response", "ipfs-http-response"],
     ["ipfs/js-ipfsd-ctl", "ipfsd-ctl"],
-    ["ipfs/js-ipfs", "ipfs-multipart"],
     ["ipfs/is-ipfs", "is-ipfs"],
     ["ipfs/aegir", "aegir"],
     ["ipfs/js-ipfs-repo-migrations", "ipfs-repo-migrations"],

package.json

@@ -43,6 +43,8 @@
     "test": "cross-env ECHO_SERVER_PORT=37480 aegir test",
     "test:node": "cross-env ECHO_SERVER_PORT=37481 aegir test -t node",
     "test:browser": "cross-env ECHO_SERVER_PORT=37482 aegir test -t browser",
+    "test:browser:http": "cross-env ECHO_SERVER_PORT=37489 aegir test -t browser -f test/http-api/index.js",
+    "test:browser:interface:http": "cross-env ECHO_SERVER_PORT=37489 aegir test -t browser -f test/http-api/interface.js",
     "test:webworker": "cross-env ECHO_SERVER_PORT=37483 aegir test -t webworker",
     "test:electron": "cross-env ECHO_SERVER_PORT=37484 aegir test -t electron-main  -t electron-renderer",
     "test:electron-main": "cross-env ECHO_SERVER_PORT=37485 aegir test -t electron-main",
@@ -66,6 +68,7 @@
   "dependencies": {
     "@hapi/ammo": "^3.1.2",
     "@hapi/boom": "^7.4.3",
+    "@hapi/content": "^4.1.0",
     "@hapi/hapi": "^18.4.0",
     "@hapi/joi": "^15.1.0",
     "abort-controller": "^3.0.0",
@@ -89,6 +92,7 @@
     "file-type": "^12.0.1",
     "fnv1a": "^1.0.1",
     "get-folder-size": "^2.0.0",
+    "hamt-sharding": "^1.0.0",
     "hapi-pino": "^6.1.0",
     "hashlru": "^2.3.0",
     "interface-datastore": "^0.8.0",
@@ -97,8 +101,6 @@
     "ipfs-block-service": "^0.16.0",
     "ipfs-http-client": "^42.0.0",
     "ipfs-http-response": "^0.5.0",
-    "ipfs-mfs": "^1.0.0",
-    "ipfs-multipart": "^0.3.0",
     "ipfs-repo": "^0.30.1",
     "ipfs-unixfs": "^1.0.0",
     "ipfs-unixfs-exporter": "^1.0.1",
@@ -117,16 +119,17 @@
     "is-ipfs": "^0.6.1",
     "it-all": "^1.0.1",
     "it-concat": "^1.0.0",
+    "it-drain": "^1.0.0",
     "it-glob": "0.0.7",
     "it-last": "^1.0.1",
+    "it-map": "^1.0.0",
+    "it-multipart": "^1.0.1",
     "it-pipe": "^1.1.0",
     "it-tar": "^1.2.1",
     "it-to-stream": "^0.1.1",
     "iterable-ndjson": "^1.1.0",
     "jsondiffpatch": "^0.3.11",
     "just-safe-set": "^2.1.0",
-    "ky": "^0.15.0",
-    "ky-universal": "^0.3.0",
     "libp2p": "^0.27.2",
     "libp2p-bootstrap": "^0.10.2",
     "libp2p-crypto": "^0.17.1",
@@ -172,19 +175,20 @@
     "yargs-promise": "^1.1.0"
   },
   "devDependencies": {
-    "aegir": "^21.3.0",
+    "aegir": "21.3.0",
     "base64url": "^3.0.1",
     "clear-module": "^4.0.0",
     "cross-env": "^7.0.0",
     "delay": "^4.3.0",
     "execa": "^3.0.0",
     "form-data": "^3.0.0",
     "go-ipfs-dep": "0.4.23-3",
-    "hat": "0.0.3",
     "interface-ipfs-core": "^0.132.0",
     "ipfs-interop": "ipfs/interop#fix/name-pubsub",
     "ipfsd-ctl": "^3.0.0",
+    "iso-random-stream": "^1.1.1",
     "it-first": "^1.0.1",
+    "nanoid": "^2.1.11",
     "ncp": "^2.0.0",
     "p-event": "^4.1.0",
     "p-map": "^3.0.0",

src/cli/commands/files.js

@@ -0,0 +1,15 @@
+'use strict'
+
+module.exports = {
+  command: 'files <command>',
+
+  description: 'Operations over mfs files (ls, mkdir, rm, etc)',
+
+  builder (yargs) {
+    return yargs.commandDir('files')
+  },
+
+  handler (argv) {
+    argv.print('Type `jsipfs files --help` for more instructions')
+  }
+}

src/cli/commands/files/chmod.js

@@ -0,0 +1,68 @@
+'use strict'
+
+const {
+  asBoolean,
+  asOctal
+} = require('../../utils')
+
+module.exports = {
+  command: 'chmod [mode] [path]',
+
+  describe: 'Change file modes',
+
+  builder: {
+    path: {
+      type: 'string',
+      describe: 'The MFS path to change the mode of'
+    },
+    mode: {
+      type: 'int',
+      coerce: asOctal,
+      describe: 'The mode to use'
+    },
+    recursive: {
+      alias: 'r',
+      type: 'boolean',
+      default: false,
+      coerce: asBoolean,
+      describe: 'Whether to change modes recursively'
+    },
+    'hash-alg': {
+      alias: 'h',
+      type: 'string',
+      default: 'sha2-256',
+      describe: 'Hash function to use. Will set CID version to 1 if used'
+    },
+    flush: {
+      alias: 'f',
+      type: 'boolean',
+      default: true,
+      coerce: asBoolean,
+      describe: 'Flush the changes to disk immediately'
+    },
+    'shard-split-threshold': {
+      type: 'number',
+      default: 1000,
+      describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
+    }
+  },
+
+  handler (argv) {
+    const {
+      ctx: { ipfs },
+      path,
+      mode,
+      recursive,
+      hashAlg,
+      flush,
+      shardSplitThreshold
+    } = argv
+
+    return ipfs.files.chmod(path, mode, {
+      recursive,
+      hashAlg,
+      flush,
+      shardSplitThreshold
+    })
+  }
+}

src/cli/commands/files/cp.js

@@ -0,0 +1,58 @@
+'use strict'
+
+const {
+  asBoolean
+} = require('../../utils')
+
+module.exports = {
+  command: 'cp <source> <dest>',
+
+  describe: 'Copy files between locations in the mfs',
+
+  builder: {
+    parents: {
+      alias: 'p',
+      type: 'boolean',
+      default: false,
+      coerce: asBoolean,
+      describe: 'Create any non-existent intermediate directories'
+    },
+    'hash-alg': {
+      alias: 'h',
+      type: 'string',
+      default: 'sha2-256',
+      describe: 'Hash function to use. Will set CID version to 1 if used'
+    },
+    flush: {
+      alias: 'f',
+      type: 'boolean',
+      default: true,
+      coerce: asBoolean,
+      describe: 'Flush the changes to disk immediately'
+    },
+    'shard-split-threshold': {
+      type: 'number',
+      default: 1000,
+      describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
+    }
+  },
+
+  handler (argv) {
+    const {
+      ctx: { ipfs },
+      source,
+      dest,
+      parents,
+      flush,
+      hashAlg,
+      shardSplitThreshold
+    } = argv
+
+    return ipfs.files.cp(source, dest, {
+      parents,
+      flush,
+      hashAlg,
+      shardSplitThreshold
+    })
+  }
+}

src/cli/commands/files/flush.js

@@ -0,0 +1,31 @@
+'use strict'
+
+module.exports = {
+  command: 'flush [path]',
+
+  describe: ' Flush a given path\'s data to disk',
+
+  builder: {
+    'cid-base': {
+      describe: 'CID base to use.'
+    }
+  },
+
+  async handler (argv) {
+    const {
+      ctx: { ipfs, print },
+      path,
+      cidBase
+    } = argv
+
+    let cid = await ipfs.files.flush(path || '/', {})
+
+    if (cidBase && cidBase !== 'base58btc' && cid.version === 0) {
+      cid = cid.toV1()
+    }
+
+    print(JSON.stringify({
+      Cid: cid.toString(cidBase)
+    }))
+  }
+}

src/cli/commands/files/ls.js

@@ -0,0 +1,68 @@
+'use strict'
+
+const all = require('it-all')
+const {
+  asBoolean
+} = require('../../utils')
+const formatMode = require('ipfs-utils/src/files/format-mode')
+const formatMtime = require('ipfs-utils/src/files/format-mtime')
+
+module.exports = {
+  command: 'ls [path]',
+
+  describe: 'List mfs directories',
+
+  builder: {
+    long: {
+      alias: 'l',
+      type: 'boolean',
+      default: false,
+      coerce: asBoolean,
+      describe: 'Use long listing format.'
+    },
+    sort: {
+      alias: 's',
+      type: 'boolean',
+      default: true,
+      coerce: asBoolean,
+      describe: 'Sort entries by name'
+    },
+    'cid-base': {
+      describe: 'CID base to use.'
+    }
+  },
+
+  async handler (argv) {
+    const {
+      ctx: { ipfs, print },
+      path,
+      long,
+      sort,
+      cidBase
+    } = argv
+
+    const printListing = file => {
+      if (long) {
+        print(`${formatMode(file.mode, file.type === 1)}\t${formatMtime(file.mtime)}\t${file.name}\t${file.cid.toString(cidBase)}\t${file.size}`)
+      } else {
+        print(file.name)
+      }
+    }
+
+    // https://github.com/ipfs/go-ipfs/issues/5181
+    if (sort) {
+      let files = await all(ipfs.files.ls(path || '/'))
+
+      files = files.sort((a, b) => {
+        return a.name.localeCompare(b.name)
+      })
+
+      files.forEach(printListing)
+      return
+    }
+
+    for await (const file of ipfs.files.ls(path || '/')) {
+      printListing(file)
+    }
+  }
+}