chore: remove node buffers from runtime code

Node `Buffers` are subclasses of `Uint8Array` and we don't use any
`Buffer`-specific functions, so do not demand `Buffer`s where
`Uint8Array`s will do.

`Buffer`s are still used in the tests because the `Buffer` requirement
needs pull out of (at least) the `cids` and `multihash` modules first.

Author: achingbrain

packages/ipfs-unixfs-exporter/README.md

@@ -50,7 +50,7 @@ const files = []
 
 for await (const file of importer([{
   path: '/foo/bar.txt',
-  content: Buffer.from(0, 1, 2, 3)
+  content: new Uint8Array([0, 1, 2, 3])
 }], ipld)) {
   files.push(file)
 }
@@ -74,7 +74,7 @@ for await (const buf of entry.content({
   bytes.push(buf)
 }
 
-const content = Buffer.concat(bytes)
+const content = new Uint8Array(bytes)
 
 console.info(content) // 0, 1, 2, 3
 ```
@@ -185,7 +185,7 @@ for await (const chunk of entry.content({
 }
 
 // `data` contains the first 5 bytes of the file
-const data = Buffer.concat(bufs)
+const data = new Uint8Array(bufs)
 ```
 
 If `entry` is a directory or hamt shard, passing `offset` and/or `length` to `entry.content()` will limit the number of files returned from the directory.

packages/ipfs-unixfs-exporter/package-lock.json

@@ -36,15 +36,16 @@
   "homepage": "https://github.com/ipfs/js-ipfs-unixfs#readme",
   "devDependencies": {
     "abort-controller": "^3.0.0",
-    "aegir": "^22.0.0",
+    "aegir": "^25.0.0",
+    "buffer": "^5.6.0",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "detect-node": "^2.0.4",
     "dirty-chai": "^2.0.1",
     "ipfs-unixfs-importer": "^2.0.1",
     "ipld": "^0.26.1",
     "ipld-dag-pb": "^0.19.0",
-    "ipld-in-memory": "^4.0.0",
+    "ipld-in-memory": "^5.0.0",
     "it-all": "^1.0.1",
     "it-buffer-stream": "^1.0.2",
     "it-first": "^1.0.1",
@@ -53,12 +54,11 @@
     "sinon": "^9.0.1"
   },
   "dependencies": {
-    "buffer": "^5.6.0",
     "cids": "^0.8.0",
     "err-code": "^2.0.0",
     "hamt-sharding": "^1.0.0",
     "ipfs-unixfs": "^1.0.2",
     "it-last": "^1.0.1",
-    "multihashing-async": "^0.8.0"
+    "multihashing-async": "^1.0.0"
   }
 }

packages/ipfs-unixfs-exporter/package.json

@@ -1,6 +1,5 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const errCode = require('err-code')
 const CID = require('cids')
 const resolve = require('./resolvers')
@@ -15,7 +14,7 @@ const toPathComponents = (path = '') => {
 }
 
 const cidAndRest = (path) => {
-  if (Buffer.isBuffer(path)) {
+  if (path instanceof Uint8Array) {
     return {
       cid: new CID(path),
       toResolve: []

packages/ipfs-unixfs-exporter/src/index.js

@@ -1,14 +1,13 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const extractDataFromBlock = require('../../../utils/extract-data-from-block')
 const validateOffsetAndLength = require('../../../utils/validate-offset-and-length')
 const UnixFS = require('ipfs-unixfs')
 const errCode = require('err-code')
 
 async function * emitBytes (ipld, node, start, end, streamPosition = 0, options) {
   // a `raw` node
-  if (Buffer.isBuffer(node)) {
+  if (node instanceof Uint8Array) {
     const buf = extractDataFromBlock(node, streamPosition, start, end)
 
     if (buf.length) {

packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js

@@ -1,5 +1,4 @@
 'use strict'
-const { Buffer } = require('buffer')
 
 module.exports = function extractDataFromBlock (block, blockStart, requestedStart, requestedEnd) {
   const blockLength = block.length
@@ -8,7 +7,7 @@ module.exports = function extractDataFromBlock (block, blockStart, requestedStar
   if (requestedStart >= blockEnd || requestedEnd < blockStart) {
     // If we are looking for a byte range that is starts after the start of the block,
     // return an empty block.  This can happen when internal nodes contain data
-    return Buffer.alloc(0)
+    return Uint8Array.from([])
   }
 
   if (requestedEnd >= blockStart && requestedEnd < blockEnd) {

packages/ipfs-unixfs-exporter/src/utils/extract-data-from-block.js

@@ -1,20 +1,20 @@
 'use strict'
 
-const { Buffer } = require('buffer')
 const Bucket = require('hamt-sharding/src/bucket')
 const multihashing = require('multihashing-async')
 
 // FIXME: this is copy/pasted from ipfs-unixfs-importer/src/dir-sharded.js
 const hashFn = async function (value) {
-  const hash = await multihashing(Buffer.from(value, 'utf8'), 'murmur3-128')
+  const buf = new TextEncoder('utf-8').encode(value)
+  const hash = await multihashing(buf, 'murmur3-128')
 
   // Multihashing inserts preamble of 2 bytes. Remove it.
   // Also, murmur3 outputs 128 bit but, accidently, IPFS Go's
   // implementation only uses the first 64, so we must do the same
   // for parity..
   const justHash = hash.slice(2, 10)
   const length = justHash.length
-  const result = Buffer.alloc(length)
+  const result = new Uint8Array(length)
   // TODO: invert buffer because that's how Go impl does it
   for (let i = 0; i < length; i++) {
     result[length - i - 1] = justHash[i]

packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js

@@ -35,14 +35,15 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs-unixfs#readme",
   "devDependencies": {
-    "aegir": "^22.0.0",
+    "aegir": "^25.0.0",
+    "buffer": "^5.6.0",
     "chai": "^4.2.0",
     "cids": "^0.8.0",
     "detect-node": "^2.0.4",
     "dirty-chai": "^2.0.1",
     "ipfs-unixfs-exporter": "^2.0.1",
     "ipld": "^0.26.1",
-    "ipld-in-memory": "^4.0.0",
+    "ipld-in-memory": "^5.0.0",
     "it-buffer-stream": "^1.0.2",
     "it-last": "^1.0.1",
     "merge-options": "^2.0.0",
@@ -52,7 +53,6 @@
   },
   "dependencies": {
     "bl": "^4.0.0",
-    "buffer": "^5.6.0",
     "err-code": "^2.0.0",
     "hamt-sharding": "^1.0.0",
     "ipfs-unixfs": "^1.0.2",
@@ -62,7 +62,7 @@
     "it-first": "^1.0.1",
     "it-parallel-batch": "^1.0.3",
     "merge-options": "^2.0.0",
-    "multihashing-async": "^0.8.0",
+    "multihashing-async": "^1.0.0",
     "rabin-wasm": "^0.1.1"
   }
 }

packages/ipfs-unixfs-importer/package.json

@@ -1,7 +1,6 @@
 'use strict'
 
 const errCode = require('err-code')
-const { Buffer } = require('buffer')
 
 // make sure the content only emits buffer-a-likes
 async function * validateChunks (source) {
@@ -11,9 +10,9 @@ async function * validateChunks (source) {
     }
 
     if (typeof content === 'string' || content instanceof String) {
-      yield Buffer.from(content, 'utf8')
+      yield new TextEncoder('utf-8').encode(content)
     } else if (Array.isArray(content)) {
-      yield Buffer.from(content)
+      yield Uint8Array.from(content)
     } else {
       yield content
     }

packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.js

@@ -4,7 +4,6 @@ const {
   DAGLink,
   DAGNode
 } = require('ipld-dag-pb')
-const { Buffer } = require('buffer')
 const UnixFS = require('ipfs-unixfs')
 const multihashing = require('multihashing-async')
 const Dir = require('./dir')
@@ -13,15 +12,16 @@ const Bucket = require('hamt-sharding')
 const mergeOptions = require('merge-options').bind({ ignoreUndefined: true })
 
 const hashFn = async function (value) {
-  const hash = await multihashing(Buffer.from(value, 'utf8'), 'murmur3-128')
+  const buf = new TextEncoder('utf-8').encode(value)
+  const hash = await multihashing(buf, 'murmur3-128')
 
   // Multihashing inserts preamble of 2 bytes. Remove it.
   // Also, murmur3 outputs 128 bit but, accidently, IPFS Go's
   // implementation only uses the first 64, so we must do the same
   // for parity..
   const justHash = hash.slice(2, 10)
   const length = justHash.length
-  const result = Buffer.alloc(length)
+  const result = new Uint8Array(length)
   // TODO: invert buffer because that's how Go impl does it
   for (let i = 0; i < length; i++) {
     result[length - i - 1] = justHash[i]
@@ -142,7 +142,7 @@ async function * flush (path, bucket, block, shardRoot, options) {
 
   // go-ipfs uses little endian, that's why we have to
   // reverse the bit field before storing it
-  const data = Buffer.from(children.bitField().reverse())
+  const data = new Uint8Array(children.bitField().reverse())
   const dir = new UnixFS({
     type: 'hamt-sharded-directory',
     data,

packages/ipfs-unixfs-importer/src/dir-sharded.js

@@ -141,7 +141,7 @@ const data = new UnixFS([options])
   - `metadata`
   - `symlink`
   - `hamt-sharded-directory`
-- data (Buffer): The optional data field for this node
+- data (Uint8Array): The optional data field for this node
 - blockSizes (Array, default: `[]`): If this is a `file` node that is made up of multiple blocks, `blockSizes` is a list numbers that represent the size of the file chunks stored in each child node. It is used to calculate the total file size.
 - mode (Number, default `0644` for files, `0755` for directories/hamt-sharded-directories) file mode
 - mtime (`Date`, `{ secs, nsecs }`, `{ Seconds, FractionalNanoseconds }`, `[ secs, nsecs ]`): The modification time of this node

packages/ipfs-unixfs/README.md

@@ -35,7 +35,7 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs-unixfs#readme",
   "devDependencies": {
-    "aegir": "^22.0.0",
+    "aegir": "^25.0.0",
     "chai": "^4.2.0",
     "dirty-chai": "^2.0.1",
     "nyc": "^15.0.0"