WIP: commit what i currently have

Author: vmx

package.json

@@ -250,5 +250,8 @@
     "Jade Meskill <[email protected]>",
     "Jacob Karlsson <[email protected]>",
     "noah the goodra <[email protected]>"
-  ]
+  ],
+  "dependencies": {
+    "@mapbox/node-pre-gyp": "^1.0.0"
+  }
 }

packages/interface-ipfs-core/package.json

@@ -39,8 +39,8 @@
     "delay": "^4.4.0",
     "dirty-chai": "^2.0.1",
     "err-code": "^2.0.3",
-    "ipfs-unixfs": "^2.0.3",
-    "ipfs-unixfs-importer": "^5.0.0",
+    "ipfs-unixfs": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs?js-dag-pb",
+    "ipfs-unixfs-importer": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs-importer?js-dag-pb",
     "ipfs-utils": "^6.0.1",
     "ipld-block": "^0.11.0",
     "ipld-dag-cbor": "^0.17.0",

packages/ipfs-cli/package.json

@@ -20,7 +20,7 @@
   },
   "scripts": {
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "test": "npm run test:node",
     "test:node": "aegir test -t node",
     "coverage": "nyc --reporter=text --reporter=lcov npm run test:node",

packages/ipfs-client/package.json

@@ -32,7 +32,7 @@
   "scripts": {
     "test": "aegir test",
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "prepublishOnly": "aegir build",
     "coverage": "npx nyc -r html npm run test:node -- --bail",
     "clean": "rimraf ./dist",

packages/ipfs-core-utils/package.json

@@ -30,12 +30,13 @@
     "test:electron-renderer": "aegir test -t electron-renderer",
     "test:node": "aegir test -t node",
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "clean": "rimraf ./dist",
     "dep-check": "aegir dep-check -i rimraf -i ipfs-core-types"
   },
   "license": "MIT",
   "dependencies": {
+    "multiformats": "^4.0.0",
     "any-signal": "^2.1.2",
     "blob-to-it": "^1.0.1",
     "browser-readablestream-to-it": "^1.0.1",

packages/ipfs-core-utils/src/as-legacy-cid.js

@@ -0,0 +1,44 @@
+'use strict'
+
+const legacyCID = require('cids')
+// @ts-ignore
+const CID = require('multiformats/cid')
+
+/**
+ * Makes sure that an object only contains js-cid style CIDs.
+ *
+ * It traverses the object recursively and changes all instances of a CID from
+ * a js-multiforamts style CID to a js-cid style CID (js-cid style CIDs stay
+ * as they are). You can also pass in a CID directly.
+ *
+ * Once js-cid is no longer used in the code base, this utility function will
+ * no longer be needed.
+ *
+ * @param {any} obj - The object to do the transformation on
+ */
+const asLegacyCid = (obj) => {
+  if (legacyCID.isCID(obj)) {
+    return obj
+  }
+
+  // NOTE vmx 2021-02-22: I have no idea why TypeScript doesn't pick this up
+  // correctly => ignore it for now, deal with it later.
+  // @ts-ignore
+  const newCID = CID.asCID(obj)
+  if (newCID) {
+    const { version, code, multihash: { bytes } } = newCID
+    const { buffer, byteOffset, byteLength } = bytes
+    const multihash = Buffer.from(buffer, byteOffset, byteLength)
+    return new legacyCID(version, code, multihash)
+    }
+
+  if (obj && typeof obj === 'object') {
+    for (const [key, value] of Object.entries(obj)) {
+      obj[key] = asLegacyCid(value)
+    }
+  }
+
+  return obj
+}
+
+module.exports = asLegacyCid

packages/ipfs-core/package.json

@@ -39,7 +39,7 @@
   },
   "scripts": {
     "lint": "aegir lint",
-    "prepare": "aegir build --no-bundle",
+    "prepare": "aegir build --no-bundle --no-types",
     "prepublishOnly": "aegir build",
     "test": "aegir test",
     "test:node": "aegir test -t node",
@@ -53,6 +53,7 @@
     "dep-check": "aegir dep-check -i interface-ipfs-core -i ipfs-core-types -i abort-controller"
   },
   "dependencies": {
+    "@ipld/dag-pb": "0.0.1",
     "abort-controller": "^3.0.0",
     "array-shuffle": "^2.0.0",
     "bignumber.js": "^9.0.0",
@@ -73,9 +74,9 @@
     "ipfs-core-types": "^0.3.0",
     "ipfs-core-utils": "^0.7.1",
     "ipfs-repo": "^8.0.0",
-    "ipfs-unixfs": "^2.0.3",
-    "ipfs-unixfs-exporter": "^3.0.4",
-    "ipfs-unixfs-importer": "^5.0.0",
+    "ipfs-unixfs": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs?js-dag-pb",
+    "ipfs-unixfs-exporter": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs-exporter?js-dag-pb",
+    "ipfs-unixfs-importer": "https://gitpkg.now.sh/ipfs/js-ipfs-unixfs/packages/ipfs-unixfs-importer?js-dag-pb",
     "ipfs-utils": "^6.0.1",
     "ipld": "^0.28.0",
     "ipld-block": "^0.11.0",
@@ -109,6 +110,7 @@
     "multiaddr-to-uri": "^6.0.0",
     "multibase": "^4.0.0",
     "multicodec": "^3.0.1",
+    "multiformats": "^4.0.0",
     "multihashing-async": "^2.0.1",
     "native-abort-controller": "^1.0.3",
     "p-queue": "^6.6.1",

packages/ipfs-core/src/components/add-all/index.js

@@ -6,6 +6,7 @@ const { parseChunkerString } = require('./utils')
 const { pipe } = require('it-pipe')
 const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option')
 const mergeOptions = require('merge-options').bind({ ignoreUndefined: true })
+const asLegacyCid = require('ipfs-core-utils/src/as-legacy-cid')
 
 /**
  * @typedef {Object} Context
@@ -97,7 +98,9 @@ module.exports = ({ block, gcLock, preload, pin, options }) => {
         // do not keep file totals around forever
         delete totals[added.path]
 
-        yield added
+        const legacyAdded = added
+        legacyAdded.cid = asLegacyCid(added.cid)
+        yield legacyAdded
       }
     } finally {
       releaseLock()

packages/ipfs-core/src/components/add.js

@@ -1,6 +1,7 @@
 'use strict'
 
 const last = require('it-last')
+const asLegacyCid = require('ipfs-core-utils/src/as-legacy-cid')
 
 /**
  * @typedef {Object} Context
@@ -27,7 +28,10 @@ module.exports = ({ addAll }) => {
       throw Error('Failed to add a file, if you see this please report a bug')
     }
 
-    return result
+    let legacyResult = result
+    legacyResult.cid = asLegacyCid(result.cid)
+
+    return legacyResult
   }
 
   return add

packages/ipfs-core/src/components/cat.js

@@ -6,12 +6,12 @@ const withTimeoutOption = require('ipfs-core-utils/src/with-timeout-option')
 
 /**
  * @typedef {Object} Context
- * @property {import('.').IPLD} ipld
+ * @property {import('.').BlockService} blockService
  * @property {import('.').Preload} preload
  *
  * @param {Context} context
  */
-module.exports = function ({ ipld, preload }) {
+module.exports = function ({ blockService, preload }) {
   /**
    * Returns content of the file addressed by a valid IPFS Path or CID.
    *
@@ -27,7 +27,7 @@ module.exports = function ({ ipld, preload }) {
       preload(pathComponents[0])
     }
 
-    const file = await exporter(ipfsPath, ipld, options)
+    const file = await exporter(ipfsPath, blockService, options)
 
     // File may not have unixfs prop if small & imported with rawLeaves true
     if (file.unixfs && file.unixfs.type.includes('dir')) {

packages/ipfs-core/src/components/files/chmod.js

@@ -9,7 +9,13 @@ const toTrail = require('./utils/to-trail')
 const addLink = require('./utils/add-link')
 const updateTree = require('./utils/update-tree')
 const updateMfsRoot = require('./utils/update-mfs-root')
-const { DAGNode } = require('ipld-dag-pb')
+const dagPb = require('@ipld/dag-pb')
+// @ts-ignore
+const { sha256 } = require('multiformats/hashes/sha2')
+const Block = require('multiformats/block')
+// @ts-ignore
+const IpldBlock = require('ipld-block')
+const CID = require('cids')
 const mc = require('multicodec')
 const mh = require('multihashing-async').multihash
 const { pipe } = require('it-pipe')
@@ -193,10 +199,14 @@ module.exports = (context) => {
       // but do not reimport files, only manipulate dag-pb nodes
       const root = await pipe(
         async function * () {
-          for await (const entry of exporter.recursive(cid, context.ipld)) {
-            let node = await context.ipld.get(entry.cid)
+          for await (const entry of exporter.recursive(cid, context.blockService)) {
+            const block = await context.blockService(entry.cid)
+            let node = dagPb.decode(block.data)
             entry.unixfs.mode = calculateMode(mode, entry.unixfs)
-            node = new DAGNode(entry.unixfs.marshal(), node.Links)
+            node = prepare({
+              Data: entry.unixfs.marshal(),
+              Links: node.Links
+            })
 
             yield {
               path: entry.path,
@@ -234,20 +244,32 @@ module.exports = (context) => {
       return
     }
 
-    let node = await context.ipld.get(cid)
+    const block = await context.blockService.get(cid)
+    let node = dagPb.decode(block.data)
     const metadata = UnixFS.unmarshal(node.Data)
     metadata.mode = calculateMode(mode, metadata)
-    node = new DAGNode(metadata.marshal(), node.Links)
+    node = prepare({
+      Data: metadata.marshal(),
+      Links: node.Links
+    })
+
 
-    const updatedCid = await context.ipld.put(node, mc.DAG_PB, {
-      cidVersion: cid.version,
-      hashAlg: mh.names[opts.hashAlg || defaultOptions.hashAlg],
-      onlyHash: !opts.flush
+    const updatedBlock = await Block.encode({
+      value: node,
+      codec: dagPb,
+      // TODO vmx 2021-02-22: Add back support for other hashing algorithms
+      hasher: sha256
     })
+    updatedCid = updatedBlock.cid
+    if (settings.flush) {
+      const legacyCid = new CID(updatedBlock.cid.multihash.bytes)
+      await context.blockService.put(new IpldBlock(updatedBlock.bytes, legacyCid))
+    }
 
     const trail = await toTrail(context, mfsDirectory)
     const parent = trail[trail.length - 1]
-    const parentNode = await context.ipld.get(parent.cid)
+    const parentBlock = await context.blockService(parent.cid)
+    const parentNode = dagPb.decode(parentBlock.data)
 
     const result = await addLink(context, {
       parent: parentNode,

packages/ipfs-core/src/components/files/index.js

@@ -51,7 +51,6 @@ const wrap = ({
 
 const defaultOptions = {
   repoOwner: true,
-  ipld: null,
   repo: null
 }
 
@@ -93,17 +92,15 @@ function createMfs (options) {
 
 /**
  * @param {Object} context
- * @param {import('..').IPLD} context.ipld
  * @param {import('..').Block} context.block
  * @param {import('..').BlockService} context.blockService
  * @param {import('..').Repo} context.repo
  * @param {import('..').Preload} context.preload
  * @param {import('..').Options} context.options
  * @returns {MFS}
  */
-module.exports = ({ ipld, block, blockService, repo, preload, options: constructorOptions }) => {
+module.exports = ({ block, blockService, repo, preload, options: constructorOptions }) => {
   const methods = createMfs({
-    ipld,
     block,
     blocks: blockService,
     datastore: repo.root,

packages/ipfs-core/src/components/files/ls.js

@@ -64,7 +64,7 @@ module.exports = (context) => {
    */
   async function * mfsLs (path, options = {}) {
     const mfsPath = await toMfsPath(context, path, options)
-    const fsDir = await exporter(mfsPath.mfsPath, context.ipld)
+    const fsDir = await exporter(mfsPath.mfsPath, context.blocks)
 
     // single file/node
     if (!fsDir.unixfs || !fsDir.unixfs.type.includes('directory')) {

packages/ipfs-core/src/components/files/mkdir.js

@@ -75,7 +75,7 @@ module.exports = (context) => {
       const subPath = `/ipfs/${root}/${subPathComponents.join('/')}`
 
       try {
-        parent = await exporter(subPath, context.ipld)
+        parent = await exporter(subPath, context.blockService)
         log(`${subPath} existed`)
         log(`${subPath} had children ${parent.node.Links.map(link => link.Name)}`)
 

packages/ipfs-core/src/components/files/read.js

@@ -40,7 +40,7 @@ module.exports = (context) => {
     return {
       [Symbol.asyncIterator]: async function * read () {
         const mfsPath = await toMfsPath(context, path, options)
-        const result = await exporter(mfsPath.mfsPath, context.ipld)
+        const result = await exporter(mfsPath.mfsPath, context.blockService)
 
         if (result.unixfs.type !== 'file') {
           throw errCode(new Error(`${path} was not a file`), 'ERR_NOT_FILE')