diff --git a/.gitignore b/.gitignore index 88b65f8..b2dc31c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ node_modules .node_repl_history dist +docs \ No newline at end of file diff --git a/README.md b/README.md index e77de2b..e6edc09 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,6 @@ backed by an [IPFS Repo][repo] as its datastore for blocks, and uses [Bitswap][b ┌────────────────────┐ │ BlockService │ └────────────────────┘ - │ ┌─────┴─────┐ ▼ ▼ ┌─────────┐ ┌───────┐ @@ -42,6 +41,7 @@ backed by an [IPFS Repo][repo] as its datastore for blocks, and uses [Bitswap][b - [Example](#example) - [Browser: Browserify, Webpack, other bundlers](#browser-browserify-webpack-other-bundlers) - [Browser: ` ``` -# API - -```js -const BlockService = require('ipfs-block-service') -``` - -### `new BlockService(repo)` - -- `repo: Repo` - -Creates a new block service backed by [IPFS Repo][repo] `repo` for storage. - -### `goOnline(bitswap)` - -- `bitswap: Bitswap` - -Add a bitswap instance that communicates with the network to retreive blocks -that are not in the local store. - -If the node is online all requests for blocks first check locally and -afterwards ask the network for the blocks. +## API -### `goOffline()` - -Remove the bitswap instance and fall back to offline mode. - -### `isOnline()` - -Returns a `Boolean` indicating if the block service is online or not. - -### `put(blockAndCID, callback)` - -- `blockAndCID: { block: block, cid: cid }` -- `callback: Function` - -Asynchronously adds a block instance to the underlying repo. - -### `putStream()` - -Returns a through pull-stream, which `blockAndCID`s can be written to, and -that emits the meta data about the written block. - -### `get(cid [, extension], callback)` - -- `cid: CID` -- `extension: String`, defaults to 'data' -- `callback: Function` - -Asynchronously returns the block whose content multihash matches `multihash`. - -### `getStream(cid [, extension])` - -- `cid: CID` -- `extension: String`, defaults to 'data' - -Returns a source pull-stream, which emits the requested block. - -### `delete(cids, [, extension], callback)` - -- `cids: CID | []CID` -- `extension: String`, defaults to 'data' - `extension: String`, defaults to 'data' -- `callback: Function` - -Deletes all blocks referenced by multihashes. +See https://ipfs.github.io/js-ipfs-block-service ## Contribute @@ -205,5 +137,3 @@ This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/c [ipfs]: https://ipfs.io [bitswap]: https://github.com/ipfs/specs/tree/master/bitswap [repo]: https://github.com/ipfs/specs/tree/master/repo - - diff --git a/package.json b/package.json index e21f989..455c07a 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,12 @@ "test": "aegir-test", "test:node": "aegir-test node", "test:browser": "aegir-test browser", - "release": "aegir-release", - "release-minor": "aegir-release --type minor", - "release-major": "aegir-release --type major", + "release": "aegir-release --docs", + "release-minor": "aegir-release --type minor --docs", + "release-major": "aegir-release --type major --docs", "coverage": "aegir-coverage", - "coverage-publish": "aegir-coverage publish" + "coverage-publish": "aegir-coverage publish", + "docs": "aegir-docs" }, "pre-commit": [ "lint", @@ -36,25 +37,24 @@ }, "homepage": "https://github.com/ipfs/js-ipfs-block-service#readme", "devDependencies": { - "aegir": "^10.0.0", - "buffer-loader": "0.0.1", + "aegir": "^11.0.0", + "async": "^2.1.5", "chai": "^3.5.0", - "fs-pull-blob-store": "~0.4.1", - "idb-pull-blob-store": "~0.5.1", - "ipfs-block": "~0.5.5", - "ipfs-repo": "~0.11.3", + "cids": "^0.4.2", + "dirty-chai": "^1.2.2", + "ipfs-block": "~0.6.0", + "ipfs-repo": "~0.12.0", "lodash": "^4.17.4", + "multihashing-async": "^0.4.4", "ncp": "^2.0.0", "pre-commit": "^1.2.2", - "rimraf": "^2.5.4" + "rimraf": "^2.6.1" }, "engines": { - "node": ">=4.0.0" - }, - "dependencies": { - "async": "^2.1.4", - "cids": "~0.4.1" + "node": ">=4.0.0", + "npm": ">=3.0.0" }, + "dependencies": {}, "contributors": [ "David Dias ", "Friedel Ziegelmayer ", @@ -64,4 +64,4 @@ "npmcdn-to-unpkg-bot ", "wanderer " ] -} \ No newline at end of file +} diff --git a/src/index.js b/src/index.js index 3ed7366..c761a9a 100644 --- a/src/index.js +++ b/src/index.js @@ -1,91 +1,109 @@ 'use strict' -const parallelLimit = require('async/parallelLimit') -const pull = require('pull-stream') - -// BlockService is a hybrid block datastore. It stores data in a local -// datastore and may retrieve data from a remote Exchange. -// It uses an internal `datastore.Datastore` instance to store values. -module.exports = class BlockService { +/** + * BlockService is a hybrid block datastore. It stores data in a local + * datastore and may retrieve data from a remote Exchange. + * It uses an internal `datastore.Datastore` instance to store values. + */ +class BlockService { + /** + * Create a new BlockService + * + * @param {IPFSRepo} ipfsRepo + * @returns {BlockService} + */ constructor (ipfsRepo) { this._repo = ipfsRepo this._bitswap = null } + /** + * Add a bitswap instance that communicates with the + * network to retreive blocks that are not in the local store. + * + * If the node is online all requests for blocks first + * check locally and afterwards ask the network for the blocks. + * + * @param {Bitswap} bitswap + * @returns {void} + */ goOnline (bitswap) { this._bitswap = bitswap } + /** + * Go offline, i.e. drop the reference to bitswap. + * + * @returns {void} + */ goOffline () { this._bitswap = null } + /** + * Is the blockservice online, i.e. is bitswap present. + * + * @returns {bool} + */ isOnline () { return this._bitswap != null } - // Note: we have to pass the CID, so that bitswap can then use it for - // the smart selectors. For now, passing the CID is used so that we know - // the right multihash, this means that now we have multiple hashes official - // support \o/ - put (blockAndCID, callback) { - callback = callback || (() => {}) - if (!blockAndCID) { - return callback(new Error('Missing block and CID')) + /** + * Put a block to the underlying datastore. + * + * @param {Block} block + * @param {function(Error)} callback + * @returns {void} + */ + put (block, callback) { + if (this.isOnline()) { + return this._bitswap.put(block, callback) } - pull( - pull.values([ - blockAndCID - ]), - this.putStream(), - pull.onEnd(callback) - ) + this._repo.blockstore.put(block, callback) } - putStream () { + /** + * Put a multiple blocks to the underlying datastore. + * + * @param {Array} blocks + * @param {function(Error)} callback + * @returns {void} + */ + putMany (blocks, callback) { if (this.isOnline()) { - return this._bitswap.putStream() - } else { - return pull( - pull.map((blockAndCID) => { - return { - data: blockAndCID.block.data, - key: blockAndCID.cid.multihash - } - }), - this._repo.blockstore.putStream() - ) + return this._bitswap.putMany(blocks, callback) } - } - get (cid, callback) { - pull( - this.getStream(cid), - pull.collect((err, result) => { - if (err) { - return callback(err) - } - callback(null, result[0]) - }) - ) + this._repo.blockstore.putMany(blocks, callback) } - getStream (cid) { + /** + * Get a block by cid. + * + * @param {CID} cid + * @param {function(Error, Block)} callback + * @returns {void} + */ + get (cid, callback) { if (this.isOnline()) { - return this._bitswap.getStream(cid) + return this._bitswap.get(cid, callback) } - return this._repo.blockstore.getStream(cid.multihash) + return this._repo.blockstore.get(cid, callback) } - delete (cids, callback) { - if (!Array.isArray(cids)) { - cids = [cids] - } - - parallelLimit(cids.map((cid) => (next) => { - this._repo.blockstore.delete(cid.multihash, next) - }), 100, callback) + /** + * Delete a block from the blockstore. + * + * @param {CID} cid + * @param {function(Error)} callback + * @return {void} + */ + delete (cid, callback) { + this._repo.blockstore.delete(cid, callback) } } + +module.exports = BlockService diff --git a/test/block-service-test.js b/test/block-service-test.js index b19f01c..60ad2de 100644 --- a/test/block-service-test.js +++ b/test/block-service-test.js @@ -1,13 +1,16 @@ /* eslint-env mocha */ 'use strict' -const expect = require('chai').expect +const chai = require('chai') +chai.use(require('dirty-chai')) +const expect = chai.expect + const Block = require('ipfs-block') -const pull = require('pull-stream') const _ = require('lodash') const map = require('async/map') const waterfall = require('async/waterfall') const CID = require('cids') +const multihashing = require('multihashing-async') const BlockService = require('../src') @@ -21,21 +24,16 @@ module.exports = (repo) => { describe('offline', () => { it('store and get a block', (done) => { - const b = new Block('A random data block') - b.key((err, key) => { - expect(err).to.not.exist - const cid = new CID(key) + const data = new Buffer('A random data block') + multihashing(data, 'sha2-256', (err, hash) => { + expect(err).to.not.exist() + const b = new Block(data, new CID(hash)) waterfall([ - (cb) => bs.put({ block: b, cid: cid }, cb), - (cb) => bs.get(cid, (err, block) => { - if (err) { - return cb(err) - } - block.key(cb) - }), - (blockKey, cb) => { - expect(key).to.be.eql(blockKey) + (cb) => bs.put(b, cb), + (cb) => bs.get(b.cid, cb), + (res, cb) => { + expect(res).to.be.eql(b) cb() } ], done) @@ -43,174 +41,92 @@ module.exports = (repo) => { }) it('get a non existent block', (done) => { - const b = new Block('Not stored') - - b.key((err, key) => { - expect(err).to.not.exist - const cid = new CID(key) + const data = new Buffer('Not stored') - bs.get(cid, (err, block) => { - expect(err).to.exist + multihashing(data, 'sha2-256', (err, hash) => { + expect(err).to.not.exist() + bs.get(new CID(hash), (err, block) => { + expect(err).to.exist() + expect(block).to.not.exist() done() }) }) }) it('store many blocks', (done) => { - const b1 = new Block('1') - const b2 = new Block('2') - const b3 = new Block('3') - - pull( - pull.values([b1, b2, b3]), - pull.asyncMap((b, cb) => { - b.key((err, key) => { - if (err) { - return cb(err) - } - cb(null, { - block: b, - cid: new CID(key) - }) - }) - }), - bs.putStream(), - pull.collect((err, meta) => { - expect(err).to.not.exist - expect(meta).to.have.length(3) - done() + const data = [new Buffer('1'), new Buffer('2'), new Buffer('3')] + map(data, (d, cb) => { + multihashing(d, 'sha2-256', (err, hash) => { + expect(err).to.not.exist() + cb(null, new Block(d, new CID(hash))) }) - ) + }, (err, blocks) => { + expect(err).to.not.exist() + bs.putMany(blocks, done) + }) }) it('get many blocks', (done) => { - const b1 = new Block('1') - const b2 = new Block('2') - const b3 = new Block('3') - - pull( - pull.values([b1, b2, b3]), - pull.asyncMap((b, cb) => { - b.key((err, key) => { - if (err) { - return cb(err) - } - cb(null, { - block: b, - cid: new CID(key) - }) - }) - }), - bs.putStream(), - pull.onEnd((err) => { - expect(err).to.not.exist - getAndAssert() - }) - ) - - function getAndAssert () { - pull( - pull.values([b1, b2, b3]), - pull.asyncMap((b, cb) => b.key(cb)), - pull.map((key) => { - const cid = new CID(key) - return bs.getStream(cid) - }), - pull.flatten(), - pull.collect((err, blocks) => { - expect(err).to.not.exist - map(blocks.concat([b1, b2, b3]), (b, cb) => { - b.key(cb) - }, (err, res) => { - expect(err).to.not.exist - expect(res.slice(0, blocks.length)).to.be.eql(res.slice(blocks.length)) - done() - }) + const data = [new Buffer('1'), new Buffer('2'), new Buffer('3')] + waterfall([ + (cb) => map(data, (d, cb) => { + multihashing(d, 'sha2-256', (err, hash) => { + expect(err).to.not.exist() + cb(null, new Block(d, new CID(hash))) }) + }, cb), + (blocks, cb) => map( + blocks, + (b, cb) => bs.get(b.cid, cb), + (err, res) => { + expect(err).to.not.exist() + expect(res).to.be.eql(blocks) + cb() + } ) - } - }) - - it('delete a block', (done) => { - const b = new Block('Will not live that much') - let cid - waterfall([ - (cb) => b.key(cb), - (key, cb) => { - cid = new CID(key) - cb() - }, - (cb) => bs.put({ block: b, cid: cid }, cb), - (cb) => bs.delete(cid, cb), - (res, cb) => bs.get(cid, (err, res) => { - expect(err).to.exist - expect(res).to.not.exist - cb() - }) ], done) }) - it('delete a non existent block', (done) => { - const b = new Block('I do not exist') - waterfall([ - (cb) => b.key(cb), - (key, cb) => bs.delete(new CID(key), cb) - ], done) - }) - - it('delete many blocks', (done) => { - const b1 = new Block('1') - const b2 = new Block('2') - const b3 = new Block('3') + it('delete a block', (done) => { + const data = new Buffer('Will not live that much') + multihashing(data, 'sha2-256', (err, hash) => { + expect(err).to.not.exist() + const b = new Block(data, new CID(hash)) - map([b1, b2, b3], (b, cb) => { - b.key(cb) - }, (err, keys) => { - expect(err).to.not.exist - bs.delete(keys.map((k) => new CID(k)), done) + waterfall([ + (cb) => bs.put(b, cb), + (cb) => bs.delete(b.cid, cb), + (cb) => bs._repo.blockstore.has(b.cid, cb), + (res, cb) => { + expect(res).to.be.eql(false) + cb() + } + ], done) }) }) - it('stores and gets lots of blocks', function (done) { - this.timeout(60 * 1000) - - const blocks = _.range(1000).map((i) => { - return new Block(`hello-${i}-${Math.random()}`) + it('stores and gets lots of blocks', (done) => { + const data = _.range(1000).map((i) => { + return new Buffer(`hello-${i}-${Math.random()}`) }) - pull( - pull.values(blocks), - pull.asyncMap((block, cb) => { - block.key((err, key) => { - if (err) { - return cb(err) - } - cb(null, { - block: block, - cid: new CID(key) - }) + map(data, (d, cb) => { + multihashing(d, 'sha2-256', (err, hash) => { + expect(err).to.not.exist() + cb(null, new Block(d, new CID(hash))) + }) + }, (err, blocks) => { + expect(err).to.not.exist() + bs.putMany(blocks, (err) => { + expect(err).to.not.exist() + + map(blocks, (b, cb) => bs.get(b.cid, cb), (err, res) => { + expect(err).to.not.exist() + expect(res).to.be.eql(blocks) + done() }) - }), - bs.putStream(), - pull.onEnd((err) => { - expect(err).to.not.exist - - pull( - pull.values(blocks), - pull.asyncMap((block, cb) => block.key(cb)), - pull.map((key) => { - const cid = new CID(key) - return bs.getStream(cid) - }), - pull.flatten(), - pull.collect((err, retrievedBlocks) => { - expect(err).to.not.exist - expect(retrievedBlocks.length).to.be.eql(blocks.length) - done() - }) - ) }) - ) + }) }) it('goes offline', () => { @@ -235,70 +151,44 @@ module.exports = (repo) => { it('retrieves a block through bitswap', (done) => { // returns a block with a value equal to its key const bitswap = { - getStream (key) { - return pull.values([ - new Block('secret') - ]) + get (cid, callback) { + callback(null, new Block(new Buffer('secret'), cid)) } } bs.goOnline(bitswap) - const block = new Block('secret') + const data = new Buffer('secret') waterfall([ - (cb) => block.key('sha2-256', cb), - (key, cb) => bs.get(new CID(key), cb), + (cb) => multihashing(data, 'sha2-256', cb), + (hash, cb) => bs.get(new CID(hash), cb), (block, cb) => { - expect(block.data).to.be.eql(new Block('secret').data) + expect(block.data).to.be.eql(data) cb() } ], done) }) it('puts the block through bitswap', (done) => { + const puts = [] const bitswap = { - putStream () { - return pull.through(() => {}) + put (block, callback) { + puts.push(block) + callback() } } bs.goOnline(bitswap) - const block = new Block('secret sauce') + const data = new Buffer('secret sauce') waterfall([ - (cb) => block.key('sha2-256', cb), - (key, cb) => bs.put({block: block, cid: new CID(key)}, cb) - ], done) - }) - - it('getStream through bitswap', (done) => { - const b = new Block('secret sauce 1') - - const bitswap = { - getStream () { - return pull.values([b]) - } - } - - bs.goOnline(bitswap) - - b.key((err, key) => { - expect(err).to.not.exist - const cid = new CID(key) - - pull( - bs.getStream(cid), - pull.collect((err, blocks) => { - expect(err).to.not.exist - expect(blocks[0].data).to.be.eql(b.data) - blocks[0].key('sha2-256', (err, blockKey) => { - expect(err).to.not.exist - expect(blockKey).to.be.eql(cid.multihash) - done() - }) - }) - ) + (cb) => multihashing(data, 'sha2-256', cb), + (hash, cb) => bs.put(new Block(data, new CID(hash)), cb) + ], (err) => { + expect(err).to.not.exist() + expect(puts).to.have.length(1) + done() }) }) }) diff --git a/test/browser.js b/test/browser.js index 28a034e..fd60cc4 100644 --- a/test/browser.js +++ b/test/browser.js @@ -1,53 +1,41 @@ /* eslint-env mocha */ +/* global self */ 'use strict' const series = require('async/series') -const Store = require('idb-pull-blob-store') -const _ = require('lodash') const IPFSRepo = require('ipfs-repo') -const pull = require('pull-stream') -const repoContext = require.context('buffer!./test-repo', true) const tests = require('./block-service-test') -const idb = window.indexedDB || - window.mozIndexedDB || - window.webkitIndexedDB || - window.msIndexedDB +const idb = self.indexedDB || + self.mozIndexedDB || + self.webkitIndexedDB || + self.msIndexedDB idb.deleteDatabase('ipfs') idb.deleteDatabase('ipfs/blocks') -describe('IPFS Repo Tests on the Browser', function () { - before((done) => { - const repoData = [] - repoContext.keys().forEach((key) => { - repoData.push({ - key: key.replace('./', ''), - value: repoContext(key) - }) - }) - - const mainBlob = new Store('ipfs') - const blocksBlob = new Store('ipfs/blocks') - - series(repoData.map((file) => (cb) => { - if (_.startsWith(file.key, 'datastore/')) { - return cb() - } +describe('IPFS Repo Tests on the Browser', () => { + const repo = new IPFSRepo('ipfs') - const blocks = _.startsWith(file.key, 'blocks/') - const blob = blocks ? blocksBlob : mainBlob + before((done) => { + series([ + (cb) => repo.init({}, cb), + (cb) => repo.open(cb) + ], done) + }) - const key = blocks ? file.key.replace(/^blocks\//, '') : file.key + after((done) => { + series([ + (cb) => repo.close(cb), + (cb) => { + idb.deleteDatabase('ipfs') + idb.deleteDatabase('ipfs/blocks') - pull( - pull.values([file.value]), - blob.write(key, cb) - ) - }), done) + cb() + } + ], done) }) - const repo = new IPFSRepo('ipfs', {stores: Store}) tests(repo) }) diff --git a/test/node.js b/test/node.js index 94eaf81..f03c709 100644 --- a/test/node.js +++ b/test/node.js @@ -2,10 +2,10 @@ 'use strict' const ncp = require('ncp').ncp +const series = require('async/series') const rimraf = require('rimraf') const path = require('path') const IPFSRepo = require('ipfs-repo') -const Store = require('fs-pull-blob-store') const tests = require('./block-service-test') @@ -13,15 +13,21 @@ describe('IPFS Block Tests on Node.js', () => { const testRepoPath = path.join(__dirname, 'test-repo') const date = Date.now().toString() const repoPath = testRepoPath + '-for-' + date + const repo = new IPFSRepo(repoPath) before((done) => { - ncp(testRepoPath, repoPath, done) + series([ + (cb) => ncp(testRepoPath, repoPath, cb), + (cb) => repo.open(cb) + ], done) }) after((done) => { - rimraf(repoPath, done) + series([ + (cb) => repo.close(cb), + (cb) => rimraf(repoPath, cb) + ], done) }) - const repo = new IPFSRepo(repoPath, {stores: Store}) tests(repo) }) diff --git a/test/test-repo/blocks/CIQEU/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data b/test/test-repo/blocks/2F/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data similarity index 100% rename from test/test-repo/blocks/CIQEU/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data rename to test/test-repo/blocks/2F/CIQEUWUVLBXVFYSYCHHSCRTXCYHGIOBXKWUMKFR3UPAFHQ5WK5362FQ.data diff --git a/test/test-repo/blocks/CIQFF/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data b/test/test-repo/blocks/5V/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data similarity index 100% rename from test/test-repo/blocks/CIQFF/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data rename to test/test-repo/blocks/5V/CIQFFRR4O52TS2Z7QLDDTF32OIR4FWLKT5YLL7MLDVIT7DC3NHOK5VA.data diff --git a/test/test-repo/blocks/CIQJ2/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data b/test/test-repo/blocks/5X/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data similarity index 100% rename from test/test-repo/blocks/CIQJ2/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data rename to test/test-repo/blocks/5X/CIQJ23BL4UHXA2KTI6NLTXZM4PW4VEFWQBJ4ACZQAS37BLGL4HUO5XY.data diff --git a/test/test-repo/blocks/CIQMB/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data b/test/test-repo/blocks/75/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data similarity index 100% rename from test/test-repo/blocks/CIQMB/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data rename to test/test-repo/blocks/75/CIQMB7DLJFKD267QJ2B5FJNHZPTSVA7IB6OHXSQ2XSVEEKMKK6RT75I.data diff --git a/test/test-repo/blocks/CIQKK/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data b/test/test-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data similarity index 100% rename from test/test-repo/blocks/CIQKK/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data rename to test/test-repo/blocks/7J/CIQKKLBWAIBQZOIS5X7E32LQAL6236OUKZTMHPQSFIXPWXNZHQOV7JQ.data diff --git a/test/test-repo/blocks/CIQON/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data b/test/test-repo/blocks/AE/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data similarity index 100% rename from test/test-repo/blocks/CIQON/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data rename to test/test-repo/blocks/AE/CIQONICFQZH7QVU6IPSIM3AK7AD554D3BWZPAGEAQYQOWMFZQDUUAEI.data diff --git a/test/test-repo/blocks/CIQHA/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data b/test/test-repo/blocks/AP/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data similarity index 100% rename from test/test-repo/blocks/CIQHA/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data rename to test/test-repo/blocks/AP/CIQHAKDLTL5GMIFGN5YVY4BA22FPHUIODJEXS4LCTQDWA275XAJDAPI.data diff --git a/test/test-repo/blocks/CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data b/test/test-repo/blocks/C4/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data similarity index 100% rename from test/test-repo/blocks/CIQDD/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data rename to test/test-repo/blocks/C4/CIQDDZ5EDQK5AP7LRTLZHQZUR2R3GECRFV3WPKNL7PL2SKFIL2LXC4Y.data diff --git a/test/test-repo/blocks/CIQDM/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data b/test/test-repo/blocks/CY/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data similarity index 100% rename from test/test-repo/blocks/CIQDM/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data rename to test/test-repo/blocks/CY/CIQDMKFEUGKSLXMEXO774EZOYCYNHPRVFD53ZSAU7237F67XDSQGCYQ.data diff --git a/test/test-repo/blocks/CIQOL/CIQOL6D3FNU4WAZCM654OKSZR22J4N3S7CKJ3LHMKXIJTY7BPLEFDQY.data b/test/test-repo/blocks/DQ/CIQOL6D3FNU4WAZCM654OKSZR22J4N3S7CKJ3LHMKXIJTY7BPLEFDQY.data similarity index 100% rename from test/test-repo/blocks/CIQOL/CIQOL6D3FNU4WAZCM654OKSZR22J4N3S7CKJ3LHMKXIJTY7BPLEFDQY.data rename to test/test-repo/blocks/DQ/CIQOL6D3FNU4WAZCM654OKSZR22J4N3S7CKJ3LHMKXIJTY7BPLEFDQY.data diff --git a/test/test-repo/blocks/CIQIX/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data b/test/test-repo/blocks/FN/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data similarity index 100% rename from test/test-repo/blocks/CIQIX/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data rename to test/test-repo/blocks/FN/CIQIXBZMUTXFC5QIGMLJNXLLHZOPGSL2PBC65D4UIVWM6TI5F5TAFNI.data diff --git a/test/test-repo/blocks/CIQFA/CIQFAOXMHO6VGOZZ7VLPKFEHKCVHQ7UCB7UY3P3ICMN7ODTZM2J5G4Y.data b/test/test-repo/blocks/G4/CIQFAOXMHO6VGOZZ7VLPKFEHKCVHQ7UCB7UY3P3ICMN7ODTZM2J5G4Y.data similarity index 100% rename from test/test-repo/blocks/CIQFA/CIQFAOXMHO6VGOZZ7VLPKFEHKCVHQ7UCB7UY3P3ICMN7ODTZM2J5G4Y.data rename to test/test-repo/blocks/G4/CIQFAOXMHO6VGOZZ7VLPKFEHKCVHQ7UCB7UY3P3ICMN7ODTZM2J5G4Y.data diff --git a/test/test-repo/blocks/CIQH7/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data b/test/test-repo/blocks/GQ/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data similarity index 100% rename from test/test-repo/blocks/CIQH7/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data rename to test/test-repo/blocks/GQ/CIQH7OEYWXL34RWYL7VXLWEU4FWPVGT24VJT7DUZPTNLF25N25IGGQA.data diff --git a/test/test-repo/blocks/CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data b/test/test-repo/blocks/HD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data similarity index 100% rename from test/test-repo/blocks/CIQDD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data rename to test/test-repo/blocks/HD/CIQDDVW2EZIJF4NQH7WJNESD7XHQSXA5EGJVNTPVHD7444C2KLKXHDI.data diff --git a/test/test-repo/blocks/CIQJF/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data b/test/test-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data similarity index 100% rename from test/test-repo/blocks/CIQJF/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data rename to test/test-repo/blocks/IL/CIQJFGRQHQ45VCQLM7AJNF2GF5UHUAGGHC6LLAH6VYDEKLQMD4QLILY.data diff --git a/test/test-repo/blocks/CIQGA/CIQGABRH3ADEENSQCB4R3YOWESA6RDJMOEAJS44MZPLJMQDVOMAZJOQ.data b/test/test-repo/blocks/JO/CIQGABRH3ADEENSQCB4R3YOWESA6RDJMOEAJS44MZPLJMQDVOMAZJOQ.data similarity index 100% rename from test/test-repo/blocks/CIQGA/CIQGABRH3ADEENSQCB4R3YOWESA6RDJMOEAJS44MZPLJMQDVOMAZJOQ.data rename to test/test-repo/blocks/JO/CIQGABRH3ADEENSQCB4R3YOWESA6RDJMOEAJS44MZPLJMQDVOMAZJOQ.data diff --git a/test/test-repo/blocks/CIQH2/CIQH2XGYCX2BVQCWNXMVNSCUDXHNOYLDHLFGOBF2AGU34VMTJWK6L4I.data b/test/test-repo/blocks/L4/CIQH2XGYCX2BVQCWNXMVNSCUDXHNOYLDHLFGOBF2AGU34VMTJWK6L4I.data similarity index 100% rename from test/test-repo/blocks/CIQH2/CIQH2XGYCX2BVQCWNXMVNSCUDXHNOYLDHLFGOBF2AGU34VMTJWK6L4I.data rename to test/test-repo/blocks/L4/CIQH2XGYCX2BVQCWNXMVNSCUDXHNOYLDHLFGOBF2AGU34VMTJWK6L4I.data diff --git a/test/test-repo/blocks/CIQJB/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data b/test/test-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data similarity index 100% rename from test/test-repo/blocks/CIQJB/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data rename to test/test-repo/blocks/LG/CIQJBQD2O6K4CGJVCCTJNUP57QHR4SKHZ74OIITBBGLOMCO3ZOLWLGA.data diff --git a/test/test-repo/blocks/CIQOL/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data b/test/test-repo/blocks/M4/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data similarity index 100% rename from test/test-repo/blocks/CIQOL/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data rename to test/test-repo/blocks/M4/CIQOLBQZSZAODJGGH6RYYVBUXHTS3SM5EORZDU63LYPEFUAFE4SBM4I.data diff --git a/test/test-repo/blocks/CIQOY/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data b/test/test-repo/blocks/O6/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data similarity index 100% rename from test/test-repo/blocks/CIQOY/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data rename to test/test-repo/blocks/O6/CIQOYW2THIZBRGI7IN33ROGCKOFZLXJJ2MPKYZBTV4H3N7GYHXMAO6A.data diff --git a/test/test-repo/blocks/CIQNC/CIQNCWZM67WMN3GL67TIYLSCSDUHCC6ZP2X4ZZJYBIJH3TC6UMVIQ7Y.data b/test/test-repo/blocks/Q7/CIQNCWZM67WMN3GL67TIYLSCSDUHCC6ZP2X4ZZJYBIJH3TC6UMVIQ7Y.data similarity index 100% rename from test/test-repo/blocks/CIQNC/CIQNCWZM67WMN3GL67TIYLSCSDUHCC6ZP2X4ZZJYBIJH3TC6UMVIQ7Y.data rename to test/test-repo/blocks/Q7/CIQNCWZM67WMN3GL67TIYLSCSDUHCC6ZP2X4ZZJYBIJH3TC6UMVIQ7Y.data diff --git a/test/test-repo/blocks/CIQGP/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data b/test/test-repo/blocks/QF/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data similarity index 100% rename from test/test-repo/blocks/CIQGP/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data rename to test/test-repo/blocks/QF/CIQGPALRQ24P6NS4OWHTQ7R247ZI7KJWP3QWPQYS43LFULQC5ANLQFI.data diff --git a/test/test-repo/blocks/CIQOH/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data b/test/test-repo/blocks/QV/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data similarity index 100% rename from test/test-repo/blocks/CIQOH/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data rename to test/test-repo/blocks/QV/CIQOHMGEIKMPYHAUTL57JSEZN64SIJ5OIHSGJG4TJSSJLGI3PBJLQVI.data diff --git a/test/test-repo/blocks/CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data b/test/test-repo/blocks/R3/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data similarity index 100% rename from test/test-repo/blocks/CIQBE/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data rename to test/test-repo/blocks/R3/CIQBED3K6YA5I3QQWLJOCHWXDRK5EXZQILBCKAPEDUJENZ5B5HJ5R3A.data diff --git a/test/test-repo/blocks/CIQHB/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data b/test/test-repo/blocks/S5/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data similarity index 100% rename from test/test-repo/blocks/CIQHB/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data rename to test/test-repo/blocks/S5/CIQHBGZNZRPWVEFNMTLP4OS5EAVHFMCX2HD7FZUC2B3WUU3D4LGKS5A.data diff --git a/test/test-repo/blocks/SHARDING b/test/test-repo/blocks/SHARDING new file mode 100644 index 0000000..a153331 --- /dev/null +++ b/test/test-repo/blocks/SHARDING @@ -0,0 +1 @@ +/repo/flatfs/shard/v1/next-to-last/2 diff --git a/test/test-repo/blocks/CIQFE/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data b/test/test-repo/blocks/TW/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data similarity index 100% rename from test/test-repo/blocks/CIQFE/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data rename to test/test-repo/blocks/TW/CIQFEAGMNNXXTYKYQSANT6IBNTFN7WR5RPD5F6GN6MBKUUO25DNOTWQ.data diff --git a/test/test-repo/blocks/CIQF3/CIQF375TZCHT3ZYDNXAYIDYAQ347ZS7V7IAQNXZZKVVTWQI6UQJBU3Y.data b/test/test-repo/blocks/U3/CIQF375TZCHT3ZYDNXAYIDYAQ347ZS7V7IAQNXZZKVVTWQI6UQJBU3Y.data similarity index 100% rename from test/test-repo/blocks/CIQF3/CIQF375TZCHT3ZYDNXAYIDYAQ347ZS7V7IAQNXZZKVVTWQI6UQJBU3Y.data rename to test/test-repo/blocks/U3/CIQF375TZCHT3ZYDNXAYIDYAQ347ZS7V7IAQNXZZKVVTWQI6UQJBU3Y.data diff --git a/test/test-repo/blocks/CIQOM/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data b/test/test-repo/blocks/UN/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data similarity index 100% rename from test/test-repo/blocks/CIQOM/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data rename to test/test-repo/blocks/UN/CIQOMBKARLB7PAITVSNH7VEGIQJRPL6J7FT2XYVKAXT4MQPXXPUYUNY.data diff --git a/test/test-repo/blocks/CIQGF/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data b/test/test-repo/blocks/VO/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data similarity index 100% rename from test/test-repo/blocks/CIQGF/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data rename to test/test-repo/blocks/VO/CIQGFTQ7FSI2COUXWWLOQ45VUM2GUZCGAXLWCTOKKPGTUWPXHBNIVOY.data diff --git a/test/test-repo/blocks/CIQFT/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data b/test/test-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data similarity index 100% rename from test/test-repo/blocks/CIQFT/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data rename to test/test-repo/blocks/X3/CIQFTFEEHEDF6KLBT32BFAGLXEZL4UWFNWM4LFTLMXQBCERZ6CMLX3Y.data diff --git a/test/test-repo/blocks/CIQJG/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data b/test/test-repo/blocks/XO/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data similarity index 100% rename from test/test-repo/blocks/CIQJG/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data rename to test/test-repo/blocks/XO/CIQJGO2B2N75IUEM372FSMG76VV256I4PXBULZZ5ASNLK4FL4EG7XOI.data diff --git a/test/test-repo/blocks/_README b/test/test-repo/blocks/_README new file mode 100644 index 0000000..23cb090 --- /dev/null +++ b/test/test-repo/blocks/_README @@ -0,0 +1,30 @@ +This is a repository of IPLD objects. Each IPLD object is in a single file, +named .data. Where is the +"base32" encoding of the CID (as specified in +https://github.com/multiformats/multibase) without the 'B' prefix. +All the object files are placed in a tree of directories, based on a +function of the CID. This is a form of sharding similar to +the objects directory in git repositories. Previously, we used +prefixes, we now use the next-to-last two charters. + + func NextToLast(base32cid string) { + nextToLastLen := 2 + offset := len(base32cid) - nextToLastLen - 1 + return str[offset : offset+nextToLastLen] + } + +For example, an object with a base58 CIDv1 of + + zb2rhYSxw4ZjuzgCnWSt19Q94ERaeFhu9uSqRgjSdx9bsgM6f + +has a base32 CIDv1 of + + BAFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA + +and will be placed at + + SC/AFKREIA22FLID5AJ2KU7URG47MDLROZIH6YF2KALU2PWEFPVI37YLKRSCA.data + +with 'SC' being the last-to-next two characters and the 'B' at the +beginning of the CIDv1 string is the multibase prefix that is not +stored in the filename. diff --git a/test/test-repo/version b/test/test-repo/version index b8626c4..7ed6ff8 100644 --- a/test/test-repo/version +++ b/test/test-repo/version @@ -1 +1 @@ -4 +5