Skip to content
This repository was archived by the owner on Oct 1, 2021. It is now read-only.

Commit cad4938

Browse files
AuHauAuHau
committed
Adding to migration blocks to multihash
1 parent 40b8ef0 commit cad4938

File tree

4 files changed

+218
-94
lines changed

4 files changed

+218
-94
lines changed

‎migrations/migration-8/blocks-to-multihash.js

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
const path = require('path')
2+
const CID = require('cids')
3+
const Key = require('interface-datastore').Key
4+
const core = require('datastore-core')
5+
const ShardingStore = core.ShardingDatastore
6+
const base32 = require('base32.js')
7+
const utils = require('../../src/utils')
8+
const log = require('debug')('ipfs-repo-migrations:migration-8')
9+
10+
// This function in js-ipfs-repo defaults to not using sharding
11+
// but the default value of the options.sharding is True hence this
12+
// function defaults to use sharding.
13+
async function maybeWithSharding (filestore, options) {
14+
if (options.sharding === false) {
15+
return filestore
16+
}
17+
18+
const shard = new core.shard.NextToLast(2)
19+
return await ShardingStore.createOrOpen(filestore, shard)
20+
}
21+
22+
function keyToMultihash(key){
23+
// Key to CID
24+
const decoder = new base32.Decoder()
25+
const buff = decoder.write(key.toString().slice(1)).finalize()
26+
const cid = new CID(Buffer.from(buff))
27+
28+
// CID to multihash
29+
const enc = new base32.Encoder()
30+
return new Key('/' + enc.finalize(cid.multihash), false)
31+
}
32+
33+
function keyToCid(key){
34+
// Key to CID
35+
const decoder = new base32.Decoder()
36+
const buff = decoder.write(key.toString().slice(1)).finalize()
37+
const cid = new CID(1, 'raw', Buffer.from(buff))
38+
39+
// CID to Key
40+
const enc = new base32.Encoder()
41+
return new Key('/' + enc.finalize(cid.buffer), false)
42+
}
43+
44+
async function process(repoPath, options, keyFunction){
45+
const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'blocks')
46+
47+
const baseStore = new StorageBackend(path.join(repoPath, 'blocks'), storageOptions)
48+
const store = await maybeWithSharding(baseStore, storageOptions)
49+
50+
try {
51+
const batch = store.batch()
52+
let counter = 0
53+
for await (const block of store.query({})) {
54+
batch.delete(block.key)
55+
56+
counter += 1
57+
const newKey = keyFunction(block.key)
58+
log(`Migrating Block from ${block.key.toString()} to ${newKey.toString()}`)
59+
batch.put(newKey, block.value)
60+
}
61+
62+
log(`Changing ${ counter } blocks`)
63+
await batch.commit()
64+
} finally {
65+
await store.close()
66+
}
67+
}
68+
69+
exports.migrate = async function blocksMigrate (repoPath, options) {
70+
return process(repoPath, options, keyToMultihash)
71+
}
72+
73+
exports.revert = async function blocksRevert (repoPath, options) {
74+
return process(repoPath, options, keyToCid)
75+
}

‎migrations/migration-8/index.js

Lines changed: 18 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -1,103 +1,39 @@
11
'use strict'
22

3-
const Datastore = require('datastore-fs')
4-
const path = require('path')
5-
const base32 = require('base32.js')
6-
const Key = require('interface-datastore').Key
3+
const keysEncoding = require('./keys-encoding')
4+
const blocksToMultihash = require('./blocks-to-multihash')
75
const log = require('debug')('ipfs-repo-migrations:migration-8')
86

9-
const KEY_PREFIX = 'key_'
10-
11-
function encode (name) {
12-
name = Buffer.from(name)
13-
const encoder = new base32.Encoder({ type: "rfc4648" })
14-
return (KEY_PREFIX + encoder.finalize(name)).toLowerCase()
15-
}
16-
17-
function decode (name) {
18-
if (!name.startsWith(KEY_PREFIX)) {
19-
throw Error("Unknown format of key's name!")
20-
}
21-
22-
const decoder = new base32.Decoder({ type: "rfc4648" })
23-
const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase())
24-
return Buffer.from(decodedNameBuff).toString()
25-
}
26-
27-
async function processFolder (store, prefix, fileNameProcessor) {
28-
const query = {
29-
prefix: `/${prefix}`
30-
}
31-
32-
const files = store.query(query)
33-
for await (let file of files) {
34-
const name = String(file.key._buf).replace(`/${prefix}/`, '')
35-
const encodedFileName = fileNameProcessor(name)
36-
const newKey = new Key(`${prefix}/${encodedFileName}`)
37-
38-
await store.delete(file.key)
39-
log(`Translating key's name '${file.key}' into '${newKey}'`)
40-
await store.put(newKey, file.value)
41-
}
42-
}
43-
447
async function migrate (repoPath, options, isBrowser) {
45-
let storageBackend, storageBackendOptions
46-
if (options !== undefined
47-
&& options['storageBackends'] !== undefined
48-
&& options['storageBackends']['keys'] !== undefined
49-
) {
50-
storageBackend = options['storageBackends']['keys']
51-
} else {
52-
storageBackend = Datastore
53-
}
54-
55-
if (options !== undefined
56-
&& options['storageBackendOptions'] !== undefined
57-
&& options['storageBackendOptions']['keys'] !== undefined
58-
) {
59-
storageBackendOptions = options['storageBackendOptions']['keys']
60-
} else {
61-
storageBackendOptions = {}
62-
}
8+
await keysEncoding.migrate(repoPath, options)
639

64-
const store = new storageBackend(path.join(repoPath, 'keys'), storageBackendOptions)
65-
try {
66-
const info = processFolder(store, 'info', encode)
67-
const data = processFolder(store, 'pkcs8', encode)
10+
try{
11+
await blocksToMultihash.migrate(repoPath, options)
12+
}catch (e) {
13+
log('During migration of Blockstore to multihash exception was raised! Reverting keys part of migration!')
14+
await keysEncoding.revert(repoPath, options)
6815

69-
await Promise.all([info, data])
70-
} finally {
71-
await store.close()
16+
throw e
7217
}
7318
}
7419

7520
async function revert (repoPath, options, isBrowser) {
76-
let storageBackend
77-
if (options !== undefined
78-
&& options['storageBackends'] !== undefined
79-
&& options['storageBackends']['keys'] !== undefined
80-
) {
81-
storageBackend = options['storageBackends']['keys']
82-
} else {
83-
storageBackend = Datastore
84-
}
21+
await keysEncoding.revert(repoPath, options)
8522

86-
const store = new storageBackend(path.join(repoPath, 'keys'), { extension: '.data' })
87-
try {
88-
const info = processFolder(store, 'info', decode)
89-
const data = processFolder(store, 'pkcs8', decode)
23+
try{
24+
await blocksToMultihash.revert(repoPath, options)
25+
}catch (e) {
26+
log('During reversion of Blockstore to CID exception was raised! Migrating keys part of migration!')
27+
await keysEncoding.migrate(repoPath, options)
9028

91-
await Promise.all([info, data])
92-
} finally {
93-
await store.close()
29+
throw e
9430
}
9531
}
9632

9733
module.exports = {
9834
version: 8,
99-
description: 'Transforms key\'s names into base32 encoding.',
100-
reversible: true,
35+
description: 'Transforms key\'s names into base32 encoding and converts Block store to use multihashes',
36+
reversible: false, // TODO: Currently Block's CIDs are lost
10137
migrate,
10238
revert
10339
}

‎migrations/migration-8/keys-encoding.js

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
const utils = require('../../src/utils')
2+
const path = require('path')
3+
const base32 = require('base32.js')
4+
const Key = require('interface-datastore').Key
5+
const log = require('debug')('ipfs-repo-migrations:migration-8')
6+
7+
const KEY_PREFIX = 'key_'
8+
9+
function encode (name) {
10+
name = Buffer.from(name)
11+
const encoder = new base32.Encoder({ type: 'rfc4648' })
12+
return (KEY_PREFIX + encoder.finalize(name)).toLowerCase()
13+
}
14+
15+
function decode (name) {
16+
if (!name.startsWith(KEY_PREFIX)) {
17+
throw Error('Unknown format of key\'s name!')
18+
}
19+
20+
const decoder = new base32.Decoder({ type: 'rfc4648' })
21+
const decodedNameBuff = decoder.finalize(name.replace(KEY_PREFIX, '').toUpperCase())
22+
return Buffer.from(decodedNameBuff).toString()
23+
}
24+
25+
async function processFolder (store, prefix, fileNameProcessor) {
26+
const query = {
27+
prefix: `/${ prefix }`
28+
}
29+
30+
const files = store.query(query)
31+
for await (let file of files) {
32+
const name = String(file.key._buf).replace(`/${ prefix }/`, '')
33+
const encodedFileName = fileNameProcessor(name)
34+
const newKey = new Key(`${ prefix }/${ encodedFileName }`)
35+
36+
await store.delete(file.key)
37+
log(`Translating key's name '${ file.key }' into '${ newKey }'`)
38+
await store.put(newKey, file.value)
39+
}
40+
}
41+
42+
async function process (repoPath, options, processor) {
43+
const { StorageBackend, storageOptions } = utils.getDatastoreAndOptions(options, 'keys')
44+
45+
const store = new StorageBackend(path.join(repoPath, 'keys'), storageOptions)
46+
try {
47+
const info = processFolder(store, 'info', processor)
48+
const data = processFolder(store, 'pkcs8', processor)
49+
50+
return await Promise.all([info, data])
51+
} finally {
52+
await store.close()
53+
}
54+
}
55+
56+
exports.migrate = async function keyEncode (repoPath, options) {
57+
return process(repoPath, options, encode)
58+
}
59+
60+
exports.revert = async function keyDecode (repoPath, options) {
61+
return process(repoPath, options, decode)
62+
}

‎test/migrations/migration-8-test.js

Lines changed: 63 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,32 @@ chai.use(chaiAsPromised)
88
const expect = chai.expect
99

1010
const path = require('path')
11-
const migration = require('../../migrations/migration-8')
11+
const keysMigration = require('../../migrations/migration-8/keys-encoding')
12+
const blocksMigration = require('../../migrations/migration-8/blocks-to-multihash')
1213
const Key = require('interface-datastore').Key
1314
const Datastore = require('datastore-fs')
15+
const core = require('datastore-core')
16+
const ShardingStore = core.ShardingDatastore
1417

15-
const log = require('debug')('js-ipfs-repo-migrations:migration-8')
16-
17-
const fixtures = [
18+
const keysFixtures = [
1819
['aAa', 'key_mfawc'],
1920
['bbb', 'key_mjrge'],
2021
['self', 'key_onswyzq']
2122
]
2223

24+
const blocksFixtures = [
25+
['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y', ['CIQMUSJFXRZX7ZRBICXJQPHVD7YSPD5KS75DRO7Q55ADVNORRBXV75Y']],
26+
['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA', ['CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA']],
27+
['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ', ['CIQJF2E6OPOEYYEVHYML4UD5A3R4QRAVBI7NVH3PL64D3IJCWR2SPUQ']],
28+
['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y', ['CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y']],
29+
]
30+
2331
async function bootstrapKeys (dir, encoded) {
2432
const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true })
2533
await store.open()
2634

2735
let name
28-
for (let keyNames of fixtures) {
36+
for (let keyNames of keysFixtures) {
2937
name = encoded ? keyNames[1] : keyNames[0]
3038
await store.put(new Key(`/pkcs8/${ name }`), '')
3139
await store.put(new Key(`/info/${ name }`), '')
@@ -39,7 +47,7 @@ async function validateKeys (dir, shouldBeEncoded) {
3947
await store.open()
4048

4149
let name
42-
for (let keyNames of fixtures) {
50+
for (let keyNames of keysFixtures) {
4351
name = shouldBeEncoded ? keyNames[1] : keyNames[0]
4452
expect(await store.has(new Key(`/pkcs8/${ name }`))).to.be.true(name)
4553
expect(await store.has(new Key(`/info/${ name }`))).to.be.true(name)
@@ -48,6 +56,36 @@ async function validateKeys (dir, shouldBeEncoded) {
4856
await store.close()
4957
}
5058

59+
async function bootstrapBlocks (dir, encoded) {
60+
const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: true })
61+
const shard = new core.shard.NextToLast(2)
62+
const store = await ShardingStore.createOrOpen(baseStore, shard)
63+
64+
let name
65+
for (let blocksNames of blocksFixtures) {
66+
name = encoded ? blocksNames[1] : blocksNames[0]
67+
await store.put(new Key(name), '')
68+
}
69+
70+
await store.close()
71+
}
72+
73+
async function validateBlocks (dir, shouldBeEncoded) {
74+
const baseStore = new Datastore(path.join(dir, 'blocks'), { extension: '.data', createIfMissing: false })
75+
const shard = new core.shard.NextToLast(2)
76+
const store = await ShardingStore.createOrOpen(baseStore, shard)
77+
78+
let newName, oldName
79+
for (let blockNames of blocksFixtures) {
80+
newName = shouldBeEncoded ? blockNames[1] : blockNames[0]
81+
oldName = shouldBeEncoded ? blockNames[0] : blockNames[1]
82+
expect(await store.has(new Key(oldName))).to.be.false(oldName)
83+
expect(await store.has(new Key(newName))).to.be.true(newName)
84+
}
85+
86+
await store.close()
87+
}
88+
5189
module.exports = (setup, cleanup) => {
5290
describe('migration 8', () => {
5391
let dir
@@ -57,19 +95,19 @@ module.exports = (setup, cleanup) => {
5795
})
5896
afterEach(() => cleanup(dir))
5997

60-
it('should migrate forward', async () => {
98+
it('should migrate keys forward', async () => {
6199
await bootstrapKeys(dir, false)
62-
await migration.migrate(dir)
100+
await keysMigration.migrate(dir)
63101
await validateKeys(dir, true)
64102
})
65103

66-
it('should migrate backward', async () => {
104+
it('should migrate keys backward', async () => {
67105
await bootstrapKeys(dir, true)
68-
await migration.revert(dir)
106+
await keysMigration.revert(dir)
69107
await validateKeys(dir, false)
70108
})
71109

72-
it('should fail to migrate backward with invalid key name', async () => {
110+
it('should fail to migrate keys backward with invalid key name', async () => {
73111
const store = new Datastore(path.join(dir, 'keys'), { extension: '.data', createIfMissing: true })
74112
await store.open()
75113

@@ -78,7 +116,20 @@ module.exports = (setup, cleanup) => {
78116

79117
await store.close()
80118

81-
expect(migration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!')
119+
expect(keysMigration.revert(dir)).to.eventually.rejectedWith('Unknown format of key\'s name!')
120+
})
121+
122+
it('should migrate blocks forward', async () => {
123+
await bootstrapBlocks(dir, false)
124+
await blocksMigration.migrate(dir)
125+
await validateBlocks(dir, true)
82126
})
127+
//
128+
// it('should migrate blocks backward', async () => {
129+
// await bootstrapKeys(dir, true)
130+
// await blocksMigration.revert(dir)
131+
// await validateKeys(dir, false)
132+
// })
133+
83134
})
84135
}

0 commit comments

Comments
 (0)