chore: upgrade unixfs (#3827)

Upgrades to new ESM unixfs.  The new version doesn't export the `Dir` and `ShardedDir` classes so just copy the relevant parts into the codebase.

Author: achingbrain

packages/interface-ipfs-core/package.json

@@ -49,8 +49,8 @@
     "err-code": "^3.0.1",
     "interface-blockstore": "^1.0.0",
     "ipfs-core-types": "^0.7.0",
-    "ipfs-unixfs": "^5.0.0",
-    "ipfs-unixfs-importer": "^8.0.2",
+    "ipfs-unixfs": "^6.0.3",
+    "ipfs-unixfs-importer": "^9.0.3",
     "ipfs-utils": "^8.1.4",
     "ipns": "^0.13.3",
     "is-ipfs": "^6.0.1",

packages/ipfs-core-utils/package.json

@@ -46,7 +46,7 @@
     "browser-readablestream-to-it": "^1.0.1",
     "err-code": "^3.0.1",
     "ipfs-core-types": "^0.7.0",
-    "ipfs-unixfs": "^5.0.0",
+    "ipfs-unixfs": "^6.0.3",
     "ipfs-utils": "^8.1.4",
     "it-all": "^1.0.4",
     "it-map": "^1.0.4",

packages/ipfs-core/package.json

@@ -60,6 +60,7 @@
     "@ipld/car": "^3.1.0",
     "@ipld/dag-cbor": "^6.0.5",
     "@ipld/dag-pb": "^2.1.3",
+    "@multiformats/murmur3": "^1.0.1",
     "abort-controller": "^3.0.0",
     "array-shuffle": "^2.0.0",
     "blockstore-datastore-adapter": "^1.0.0",
@@ -79,9 +80,9 @@
     "ipfs-core-utils": "^0.10.1",
     "ipfs-http-client": "^52.0.1",
     "ipfs-repo": "^11.0.1",
-    "ipfs-unixfs": "^5.0.0",
-    "ipfs-unixfs-exporter": "^6.0.2",
-    "ipfs-unixfs-importer": "^8.0.2",
+    "ipfs-unixfs": "^6.0.3",
+    "ipfs-unixfs-exporter": "^7.0.3",
+    "ipfs-unixfs-importer": "^9.0.3",
     "ipfs-utils": "^8.1.4",
     "ipns": "^0.13.3",
     "is-domain-name": "^1.0.1",

packages/ipfs-core/src/components/files/chmod.js

@@ -17,8 +17,7 @@ const { recursive } = require('ipfs-unixfs-exporter')
 const last = require('it-last')
 const cp = require('./cp')
 const rm = require('./rm')
-// @ts-ignore - TODO: refactor this so it does not require a deep require
-const persist = require('ipfs-unixfs-importer/src/utils/persist')
+const persist = require('./utils/persist')
 const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option')
 
 /**

packages/ipfs-core/src/components/files/cp.js

@@ -208,7 +208,8 @@ const addSourceToParent = async (context, source, childName, parent, options) =>
   const sourceBlock = await context.repo.blocks.get(source.cid)
   const {
     node,
-    cid
+    cid,
+    size
   } = await addLink(context, {
     parentCid: parent.cid,
     size: sourceBlock.length,
@@ -222,7 +223,7 @@ const addSourceToParent = async (context, source, childName, parent, options) =>
 
   parent.node = node
   parent.cid = cid
-  parent.size = node.size
+  parent.size = size
 
   return parent
 }

packages/ipfs-core/src/components/files/utils/add-link.js

@@ -1,15 +1,10 @@
 'use strict'
 
-// @ts-ignore
 const dagPb = require('@ipld/dag-pb')
-const { sha256, sha512 } = require('multiformats/hashes/sha2')
 const { CID } = require('multiformats/cid')
 const log = require('debug')('ipfs:mfs:core:utils:add-link')
 const { UnixFS } = require('ipfs-unixfs')
-// @ts-ignore - refactor this to not need deep require
-const DirSharded = require('ipfs-unixfs-importer/src/dir-sharded')
-// @ts-ignore - refactor this to not need deep require
-const defaultImporterOptions = require('ipfs-unixfs-importer/src/options')
+const DirSharded = require('./dir-sharded')
 const {
   updateHamtDirectory,
   recreateHamtLevel,
@@ -223,6 +218,11 @@ const addToShardedDirectory = async (context, options) => {
     shard, path
   } = await addFileToShardedDirectory(context, options)
   const result = await last(shard.flush(context.repo.blocks))
+
+  if (!result) {
+    throw new Error('No result from flushing shard')
+  }
+
   const block = await context.repo.blocks.get(result.cid)
   const node = dagPb.decode(block)
 
@@ -269,44 +269,24 @@ const addFileToShardedDirectory = async (context, options) => {
   // start at the root bucket and descend, loading nodes as we go
   const rootBucket = await recreateInitialHamtLevel(options.parent.Links)
   const node = UnixFS.unmarshal(options.parent.Data)
-  const importerOptions = defaultImporterOptions()
-
-  // NOTE vmx 2021-04-01: in ipfs the hash algorithm is a constant in unixfs
-  // it's an implementation. Do the option conversion at the boundary between
-  // ipfs and unixfs.
-  let hasher
-  switch (options.hashAlg) {
-    case 'sha2-256':
-      hasher = sha256
-      break
-    case 'sha2-512':
-      hasher = sha512
-      break
-    default:
-      throw new Error(`TODO vmx 2021-03-31: Proper error message for unsupported hash algorithms like ${options.hashAlg}`)
-  }
 
   const shard = new DirSharded({
     root: true,
     dir: true,
-    parent: null,
-    parentKey: null,
+    parent: undefined,
+    parentKey: undefined,
     path: '',
     dirty: true,
     flat: false,
     mode: node.mode
-  }, {
-    hamtHashFn: importerOptions.hamtHashFn,
-    hamtHashCode: importerOptions.hamtHashCode,
-    hamtBucketBits: importerOptions.hamtBucketBits,
-    hasher,
-    ...options
-  })
+  }, options)
   shard._bucket = rootBucket
 
   if (node.mtime) {
     // update mtime if previously set
-    shard.mtime = new Date()
+    shard.mtime = {
+      secs: Math.round(Date.now() / 1000)
+    }
   }
 
   // load subshards until the bucket & position no longer changes