Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 03b17f5

Browse files
achingbrainhugomrdias
achingbrain
and
hugomrdias
authored
feat: store blocks by multihash instead of CID (#3124)
Updates the `ipfs-repo` dep to a version that stores blocks by multihash instead of CID to support CIDv1 and CIDv0 access to the same block. New features: - Adds a `--multihash` argument to the cli command `ipfs refs local` which prints the base32 encoded multihash of each block BREAKING CHANGES: - `ipfs.refs.local` now returns a v1 CID with the raw codec for every block and not the original CID by which it was added to the blockstore Co-authored-by: Hugo Dias <[email protected]>
1 parent 0b64c3e commit 03b17f5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

105 files changed

+216
-184
lines changed

‎docs/core-api/REFS.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@ for await (const ref of ipfs.refs(ipfsPath, { recursive: true })) {
6767

6868
> Output all local references (CIDs of all blocks in the blockstore)
6969
70+
Blocks in the blockstore are stored by multihash and not CID so yielded CIDs are v1 CIDs with the 'raw' codec. These may not match the CID originally used to store a given block, though the multihash contained within the CID will.
71+
7072
### Parameters
7173

7274
None

‎examples/custom-ipfs-repo/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"dependencies": {
1313
"datastore-fs": "^1.1.0",
1414
"ipfs": "^0.47.0",
15-
"ipfs-repo": "^3.0.0",
15+
"ipfs-repo": "^4.0.0",
1616
"it-all": "^1.0.1"
1717
},
1818
"devDependencies": {

‎packages/interface-ipfs-core/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,15 @@
5353
"it-last": "^1.0.1",
5454
"it-pushable": "^1.3.1",
5555
"multiaddr": "^7.4.3",
56-
"multibase": "^0.7.0",
56+
"multibase": "^1.0.1",
5757
"multihashing-async": "^1.0.0",
5858
"nanoid": "^3.0.2",
5959
"peer-id": "^0.13.12",
6060
"readable-stream": "^3.4.0",
6161
"temp-write": "^4.0.0"
6262
},
6363
"devDependencies": {
64-
"aegir": "^22.1.0",
64+
"aegir": "^23.0.0",
6565
"ipfsd-ctl": "^4.1.1"
6666
},
6767
"contributors": [

‎packages/interface-ipfs-core/src/block/rm.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ module.exports = (common, options) => {
4040
// block should be present in the local store
4141
const localRefs = await all(ipfs.refs.local())
4242
expect(localRefs).to.have.property('length').that.is.greaterThan(0)
43-
expect(localRefs.find(ref => ref.ref === cid.toString())).to.be.ok()
43+
expect(localRefs.find(ref => ref.ref === new CID(1, 'raw', cid.multihash).toString())).to.be.ok()
4444

4545
const result = await all(ipfs.block.rm(cid))
4646
expect(result).to.be.an('array').and.to.have.lengthOf(1)
@@ -49,7 +49,7 @@ module.exports = (common, options) => {
4949

5050
// did we actually remove the block?
5151
const localRefsAfterRemove = await all(ipfs.refs.local())
52-
expect(localRefsAfterRemove.find(ref => ref.ref === cid.toString())).to.not.be.ok()
52+
expect(localRefsAfterRemove.find(ref => ref.ref === new CID(1, 'raw', cid.multihash).toString())).to.not.be.ok()
5353
})
5454

5555
it('should remove by CID in string', async () => {

‎packages/interface-ipfs-core/src/refs-local.js

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ const all = require('it-all')
77
const importer = require('ipfs-unixfs-importer')
88
const drain = require('it-drain')
99
const testTimeout = require('./utils/test-timeout')
10+
const CID = require('cids')
1011

1112
/** @typedef { import("ipfsd-ctl/src/factory") } Factory */
1213
/**
@@ -54,8 +55,22 @@ module.exports = (common, options) => {
5455

5556
const refs = await all(ipfs.refs.local())
5657
const cids = refs.map(r => r.ref)
57-
expect(cids).to.include('QmVwdDCY4SPGVFnNCiZnX5CtzwWDn6kAM98JXzKxE3kCmn')
58-
expect(cids).to.include('QmR4nFjTu18TyANgC65ArNWp5Yaab1gPzQ4D8zp7Kx3vhr')
58+
59+
expect(
60+
cids.find(cid => {
61+
const multihash = new CID(cid).multihash
62+
63+
return imported[0].cid.multihash.equals(multihash)
64+
})
65+
).to.be.ok()
66+
67+
expect(
68+
cids.find(cid => {
69+
const multihash = new CID(cid).multihash
70+
71+
return imported[1].cid.multihash.equals(multihash)
72+
})
73+
).to.be.ok()
5974
})
6075
})
6176
}

‎packages/interface-ipfs-core/src/repo/gc.js

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const { getDescribe, getIt, expect } = require('../utils/mocha')
66
const { DAGNode } = require('ipld-dag-pb')
77
const all = require('it-all')
88
const testTimeout = require('../utils/test-timeout')
9+
const CID = require('cids')
910

1011
/** @typedef { import("ipfsd-ctl/src/factory") } Factory */
1112
/**
@@ -58,15 +59,15 @@ module.exports = (common, options) => {
5859
// the initial list and contain hash
5960
const refsAfterAdd = await all(ipfs.refs.local())
6061
expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
61-
expect(refsAfterAdd.map(r => r.ref)).includes(cid.toString())
62+
expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(cid.multihash)
6263

6364
// Run garbage collection
6465
await all(ipfs.repo.gc())
6566

6667
// Get the list of local blocks after GC, should still contain the hash,
6768
// because the file is still pinned
6869
const refsAfterGc = await all(ipfs.refs.local())
69-
expect(refsAfterGc.map(r => r.ref)).includes(cid.toString())
70+
expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(cid.multihash)
7071

7172
// Unpin the data
7273
await ipfs.pin.rm(cid)
@@ -76,7 +77,7 @@ module.exports = (common, options) => {
7677

7778
// The list of local blocks should no longer contain the hash
7879
const refsAfterUnpinAndGc = await all(ipfs.refs.local())
79-
expect(refsAfterUnpinAndGc.map(r => r.ref)).not.includes(cid.toString())
80+
expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(cid.multihash)
8081
})
8182

8283
it('should clean up removed MFS files', async () => {
@@ -87,21 +88,20 @@ module.exports = (common, options) => {
8788
await ipfs.files.write('/test', Buffer.from('oranges'), { create: true })
8889
const stats = await ipfs.files.stat('/test')
8990
expect(stats.type).to.equal('file')
90-
const hash = stats.cid.toString()
9191

9292
// Get the list of local blocks after the add, should be bigger than
9393
// the initial list and contain hash
9494
const refsAfterAdd = await all(ipfs.refs.local())
9595
expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
96-
expect(refsAfterAdd.map(r => r.ref)).includes(hash)
96+
expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(stats.cid.multihash)
9797

9898
// Run garbage collection
9999
await all(ipfs.repo.gc())
100100

101101
// Get the list of local blocks after GC, should still contain the hash,
102102
// because the file is in MFS
103103
const refsAfterGc = await all(ipfs.refs.local())
104-
expect(refsAfterGc.map(r => r.ref)).includes(hash)
104+
expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(stats.cid.multihash)
105105

106106
// Remove the file
107107
await ipfs.files.rm('/test')
@@ -111,7 +111,7 @@ module.exports = (common, options) => {
111111

112112
// The list of local blocks should no longer contain the hash
113113
const refsAfterUnpinAndGc = await all(ipfs.refs.local())
114-
expect(refsAfterUnpinAndGc.map(r => r.ref)).not.includes(hash)
114+
expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(stats.cid.multihash)
115115
})
116116

117117
it('should clean up block only after unpinned and removed from MFS', async () => {
@@ -135,17 +135,15 @@ module.exports = (common, options) => {
135135
// the initial list and contain the data hash
136136
const refsAfterAdd = await all(ipfs.refs.local())
137137
expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
138-
const hashesAfterAdd = refsAfterAdd.map(r => r.ref)
139-
expect(hashesAfterAdd).includes(dataCid.toString())
138+
expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
140139

141140
// Run garbage collection
142141
await all(ipfs.repo.gc())
143142

144143
// Get the list of local blocks after GC, should still contain the hash,
145144
// because the file is pinned and in MFS
146145
const refsAfterGc = await all(ipfs.refs.local())
147-
const hashesAfterGc = refsAfterGc.map(r => r.ref)
148-
expect(hashesAfterGc).includes(dataCid.toString())
146+
expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
149147

150148
// Remove the file
151149
await ipfs.files.rm('/test')
@@ -156,9 +154,8 @@ module.exports = (common, options) => {
156154
// Get the list of local blocks after GC, should still contain the hash,
157155
// because the file is still pinned
158156
const refsAfterRmAndGc = await all(ipfs.refs.local())
159-
const hashesAfterRmAndGc = refsAfterRmAndGc.map(r => r.ref)
160-
expect(hashesAfterRmAndGc).not.includes(mfsFileCid.toString())
161-
expect(hashesAfterRmAndGc).includes(dataCid.toString())
157+
expect(refsAfterRmAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(mfsFileCid.multihash)
158+
expect(refsAfterRmAndGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
162159

163160
// Unpin the data
164161
await ipfs.pin.rm(dataCid)
@@ -168,9 +165,8 @@ module.exports = (common, options) => {
168165

169166
// The list of local blocks should no longer contain the hashes
170167
const refsAfterUnpinAndGc = await all(ipfs.refs.local())
171-
const hashesAfterUnpinAndGc = refsAfterUnpinAndGc.map(r => r.ref)
172-
expect(hashesAfterUnpinAndGc).not.includes(mfsFileCid.toString())
173-
expect(hashesAfterUnpinAndGc).not.includes(dataCid.toString())
168+
expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(mfsFileCid.multihash)
169+
expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(dataCid.multihash)
174170
})
175171

176172
it('should clean up indirectly pinned data after recursive pin removal', async () => {
@@ -201,9 +197,8 @@ module.exports = (common, options) => {
201197
// the initial list and contain data and object hash
202198
const refsAfterAdd = await all(ipfs.refs.local())
203199
expect(refsAfterAdd.length).to.be.gt(refsBeforeAdd.length)
204-
const hashesAfterAdd = refsAfterAdd.map(r => r.ref)
205-
expect(hashesAfterAdd).includes(objCid.toString())
206-
expect(hashesAfterAdd).includes(dataCid.toString())
200+
expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(objCid.multihash)
201+
expect(refsAfterAdd.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
207202

208203
// Recursively pin the object
209204
await ipfs.pin.add(objCid, { recursive: true })
@@ -218,7 +213,7 @@ module.exports = (common, options) => {
218213
// Get the list of local blocks after GC, should still contain the data
219214
// hash, because the data is still (indirectly) pinned
220215
const refsAfterGc = await all(ipfs.refs.local())
221-
expect(refsAfterGc.map(r => r.ref)).includes(dataCid.toString())
216+
expect(refsAfterGc.map(r => new CID(r.ref).multihash)).deep.includes(dataCid.multihash)
222217

223218
// Recursively unpin the object
224219
await ipfs.pin.rm(objCid.toString())
@@ -228,9 +223,8 @@ module.exports = (common, options) => {
228223

229224
// The list of local blocks should no longer contain the hashes
230225
const refsAfterUnpinAndGc = await all(ipfs.refs.local())
231-
const hashesAfterUnpinAndGc = refsAfterUnpinAndGc.map(r => r.ref)
232-
expect(hashesAfterUnpinAndGc).not.includes(objCid.toString())
233-
expect(hashesAfterUnpinAndGc).not.includes(dataCid.toString())
226+
expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(objCid.multihash)
227+
expect(refsAfterUnpinAndGc.map(r => new CID(r.ref).multihash)).not.deep.includes(dataCid.multihash)
234228
})
235229
})
236230
}

‎packages/ipfs-core-utils/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
"ipfs-utils": "^2.2.2"
3434
},
3535
"devDependencies": {
36-
"aegir": "^22.1.0",
36+
"aegir": "^23.0.0",
3737
"chai": "^4.2.0",
3838
"chai-as-promised": "^7.1.1",
3939
"delay": "^4.3.0",

‎packages/ipfs-http-client/package.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,16 +59,16 @@
5959
"merge-options": "^2.0.0",
6060
"multiaddr": "^7.4.3",
6161
"multiaddr-to-uri": "^5.1.0",
62-
"multibase": "^0.7.0",
62+
"multibase": "^1.0.1",
6363
"multicodec": "^1.0.0",
6464
"multihashes": "^1.0.1",
6565
"nanoid": "^3.0.2",
6666
"node-fetch": "^2.6.0",
67-
"parse-duration": "^0.1.2",
67+
"parse-duration": "^0.4.4",
6868
"stream-to-it": "^0.2.0"
6969
},
7070
"devDependencies": {
71-
"aegir": "^22.1.0",
71+
"aegir": "^23.0.0",
7272
"cross-env": "^7.0.0",
7373
"go-ipfs-dep": "^0.5.1",
7474
"interface-ipfs-core": "^0.137.0",

‎packages/ipfs-http-client/src/lib/core.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ const Multiaddr = require('multiaddr')
44
const toUri = require('multiaddr-to-uri')
55
const { isBrowser, isWebWorker } = require('ipfs-utils/src/env')
66
const { URL } = require('iso-url')
7-
const parseDuration = require('parse-duration')
7+
const parseDuration = require('parse-duration').default
88
const log = require('debug')('ipfs-http-client:lib:error-handler')
99
const HTTP = require('ipfs-utils/src/http')
1010
const merge = require('merge-options')

‎packages/ipfs/package.json

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@
9696
"ipfs-core-utils": "^0.2.4",
9797
"ipfs-http-client": "^44.3.0",
9898
"ipfs-http-response": "^0.5.0",
99-
"ipfs-repo": "^3.0.0",
99+
"ipfs-repo": "^4.0.0",
100100
"ipfs-unixfs": "^1.0.3",
101101
"ipfs-unixfs-exporter": "^2.0.2",
102102
"ipfs-unixfs-importer": "^2.0.2",
@@ -117,7 +117,7 @@
117117
"it-all": "^1.0.1",
118118
"it-concat": "^1.0.0",
119119
"it-drain": "^1.0.1",
120-
"it-glob": "0.0.7",
120+
"it-glob": "0.0.8",
121121
"it-last": "^1.0.1",
122122
"it-map": "^1.0.0",
123123
"it-multipart": "^1.0.1",
@@ -148,19 +148,19 @@
148148
"mortice": "^2.0.0",
149149
"multiaddr": "^7.4.3",
150150
"multiaddr-to-uri": "^5.1.0",
151-
"multibase": "^0.7.0",
151+
"multibase": "^1.0.1",
152152
"multicodec": "^1.0.0",
153153
"multihashing-async": "^1.0.0",
154154
"p-defer": "^3.0.0",
155155
"p-queue": "^6.1.0",
156-
"parse-duration": "^0.1.2",
156+
"parse-duration": "^0.4.4",
157157
"peer-id": "^0.13.12",
158158
"pretty-bytes": "^5.3.0",
159159
"progress": "^2.0.1",
160160
"protons": "^1.2.0",
161161
"semver": "^7.3.2",
162162
"stream-to-it": "^0.2.0",
163-
"streaming-iterables": "^4.1.1",
163+
"streaming-iterables": "^5.0.0",
164164
"temp": "^0.9.0",
165165
"timeout-abort-controller": "^1.1.0",
166166
"update-notifier": "^4.0.0",
@@ -170,7 +170,7 @@
170170
"yargs-promise": "^1.1.0"
171171
},
172172
"devDependencies": {
173-
"aegir": "^22.1.0",
173+
"aegir": "^23.0.0",
174174
"base64url": "^3.0.1",
175175
"clear-module": "^4.0.0",
176176
"cross-env": "^7.0.0",
@@ -191,7 +191,7 @@
191191
"qs": "^6.9.3",
192192
"rimraf": "^3.0.0",
193193
"sinon": "^9.0.1",
194-
"stream-to-promise": "^2.2.0",
194+
"stream-to-promise": "^3.0.0",
195195
"string-argv": "^0.3.1",
196196
"temp-write": "^4.0.0",
197197
"wrtc": "^0.4.4"

‎packages/ipfs/src/cli/commands/add.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ const {
1313
} = require('../utils')
1414
const { cidToString } = require('../../utils/cid')
1515
const globSource = require('ipfs-utils/src/files/glob-source')
16-
const parseDuration = require('parse-duration')
16+
const parseDuration = require('parse-duration').default
1717

1818
async function getTotalBytes (paths) {
1919
const sizes = await Promise.all(paths.map(p => getFolderSize(p)))
@@ -90,7 +90,7 @@ module.exports = {
9090
'cid-base': {
9191
describe: 'Number base to display CIDs in.',
9292
type: 'string',
93-
choices: multibase.names
93+
choices: Object.keys(multibase.names)
9494
},
9595
hash: {
9696
type: 'string',

‎packages/ipfs/src/cli/commands/bitswap/stat.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
const multibase = require('multibase')
44
const { cidToString } = require('../../../utils/cid')
55
const prettyBytes = require('pretty-bytes')
6-
const parseDuration = require('parse-duration')
6+
const parseDuration = require('parse-duration').default
77

88
module.exports = {
99
command: 'stat',
@@ -14,7 +14,7 @@ module.exports = {
1414
'cid-base': {
1515
describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
1616
type: 'string',
17-
choices: multibase.names
17+
choices: Object.keys(multibase.names)
1818
},
1919
human: {
2020
type: 'boolean',

‎packages/ipfs/src/cli/commands/bitswap/unwant.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
const multibase = require('multibase')
44
const { cidToString } = require('../../../utils/cid')
5-
const parseDuration = require('parse-duration')
5+
const parseDuration = require('parse-duration').default
66

77
module.exports = {
88
command: 'unwant <key>',
@@ -18,7 +18,7 @@ module.exports = {
1818
'cid-base': {
1919
describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
2020
type: 'string',
21-
choices: multibase.names
21+
choices: Object.keys(multibase.names)
2222
},
2323
timeout: {
2424
type: 'string',

0 commit comments

Comments
 (0)