-
Notifications
You must be signed in to change notification settings - Fork 3
feat: convert to async/await #21
Changes from 6 commits
579311b
eea8423
e1334bb
1c7456b
468cdfb
220de8a
3679e1d
ef5499b
747a664
a193651
6a73635
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -19,13 +19,21 @@ | |||||
|
||||||
## Table of Contents | ||||||
|
||||||
- [Install](#install) | ||||||
- [Usage](#usage) | ||||||
- [Example](#example) | ||||||
- [API](#api) | ||||||
- [exporter(cid, ipld)](#exportercid-ipld-options) | ||||||
- [Contribute](#contribute) | ||||||
- [License](#license) | ||||||
- [ipfs-unixfs-exporter](#ipfs-unixfs-exporter) | ||||||
- [Lead Maintainer](#lead-maintainer) | ||||||
- [Table of Contents](#table-of-contents) | ||||||
- [Install](#install) | ||||||
- [Usage](#usage) | ||||||
- [Example](#example) | ||||||
- [API](#api) | ||||||
- [`exporter(cid, ipld)`](#exportercid-ipld) | ||||||
- [UnixFS V1 entries](#unixfs-v1-entries) | ||||||
- [Raw entries](#raw-entries) | ||||||
- [CBOR entries](#cbor-entries) | ||||||
- [`entry.content({ offset, length })`](#entrycontent-offset-length) | ||||||
- [`exporter.path(cid, ipld)`](#exporterpathcid-ipld) | ||||||
- [Contribute](#contribute) | ||||||
- [License](#license) | ||||||
|
||||||
## Install | ||||||
|
||||||
|
@@ -38,29 +46,41 @@ | |||||
### Example | ||||||
|
||||||
```js | ||||||
// Create an export source pull-stream cid or ipfs path you want to export and a | ||||||
// <dag or ipld-resolver instance> to fetch the file from | ||||||
// import a file and export it again | ||||||
const importer = require('ipfs-unixfs-importer') | ||||||
const exporter = require('ipfs-unixfs-exporter') | ||||||
const pull = require('pull-stream/pull') | ||||||
const { stdout } = require('pull-stdio') | ||||||
|
||||||
const options = {} | ||||||
|
||||||
pull( | ||||||
exporter(cid, ipld, options), | ||||||
collect((error, files) => { | ||||||
if (error) { | ||||||
// ...handle error | ||||||
} | ||||||
|
||||||
// Set up a pull stream that sends the file content to process.stdout | ||||||
pull( | ||||||
// files[0].content is a pull-stream that contains the bytes of the file | ||||||
files[0].content, | ||||||
stdout() | ||||||
) | ||||||
}) | ||||||
) | ||||||
|
||||||
const files = [] | ||||||
|
||||||
for await (const file of importer([{ | ||||||
path: '/foo/bar.txt', | ||||||
content: Buffer.from(0, 1, 2, 3) | ||||||
}], ipld)) { | ||||||
files.push(file) | ||||||
} | ||||||
|
||||||
console.info(files[0].cid) // Qmbaz | ||||||
|
||||||
const entry = await exporter(files[0].cid, ipld) | ||||||
|
||||||
console.info(entry.cid) // Qmqux | ||||||
console.info(entry.path) // Qmbaz/foo/bar.txt | ||||||
console.info(entry.name) // bar.txt | ||||||
console.info(entry.unixfs.fileSize()) // 4 | ||||||
|
||||||
// stream content from unixfs node | ||||||
const bytes = [] | ||||||
|
||||||
for await (const buf of entry.content({ | ||||||
offset: 0, // optional offset | ||||||
length: 4 // optional length | ||||||
})) { | ||||||
bytes.push(buf) | ||||||
} | ||||||
|
||||||
const content = Buffer.concat(bytes) | ||||||
|
||||||
console.info(content) // 0, 1, 2, 3 | ||||||
``` | ||||||
|
||||||
#### API | ||||||
|
@@ -69,124 +89,126 @@ pull( | |||||
const exporter = require('ipfs-unixfs-exporter') | ||||||
``` | ||||||
|
||||||
### exporter(cid, ipld, options) | ||||||
### `exporter(cid, ipld)` | ||||||
|
||||||
Uses the given [dag API][] or an [ipld-resolver instance][] to fetch an IPFS [UnixFS][] object(s) by their CID. | ||||||
Uses the given [js-ipld instance][] to fetch an IPFS node by it's CID. | ||||||
|
||||||
Creates a new pull stream that outputs objects of the form | ||||||
Returns a Promise which resolves to an `entry`. | ||||||
|
||||||
```js | ||||||
#### UnixFS V1 entries | ||||||
|
||||||
Entries with a `dag-pb` codec `CID` return UnixFS V1 entries: | ||||||
|
||||||
```javascript | ||||||
{ | ||||||
path: 'a name', | ||||||
content: <pull stream> | ||||||
name: 'foo.txt', | ||||||
path: 'Qmbar/foo.txt', | ||||||
cid: CID, // see https://github.com/multiformats/js-cid | ||||||
node: DAGNode, // see https://github.com/ipld/js-ipld-dag-pb | ||||||
content: function, // returns an async iterator | ||||||
unixfs: UnixFS // see https://github.com/ipfs/js-ipfs-unixfs | ||||||
} | ||||||
``` | ||||||
|
||||||
#### `offset` and `length` | ||||||
If the entry is a file, `entry.content()` returns an async iterator that emits buffers containing the file content: | ||||||
|
||||||
`offset` and `length` arguments can optionally be passed to the exporter function. These will cause the returned stream to only emit bytes starting at `offset` and with length of `length`. | ||||||
```javascript | ||||||
for await (const chunk of entry.content()) { | ||||||
// chunk is a Buffer | ||||||
} | ||||||
``` | ||||||
|
||||||
See [the tests](test/exporter.js) for examples of using these arguments. | ||||||
If the entry is a directory or hamt shard, `entry.content()` returns further `entry` objects: | ||||||
achingbrain marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
|
||||||
```js | ||||||
const exporter = require('ipfs-unixfs-exporter') | ||||||
const pull = require('pull-stream') | ||||||
const drain = require('pull-stream/sinks/drain') | ||||||
|
||||||
pull( | ||||||
exporter(cid, ipld, { | ||||||
offset: 0, | ||||||
length: 10 | ||||||
}) | ||||||
drain((file) => { | ||||||
// file.content is a pull stream containing only the first 10 bytes of the file | ||||||
}) | ||||||
) | ||||||
```javascript | ||||||
for await (const entry of dir.content()) { | ||||||
console.info(entry.name) | ||||||
} | ||||||
``` | ||||||
|
||||||
### `fullPath` | ||||||
#### Raw entries | ||||||
|
||||||
If specified the exporter will emit an entry for every path component encountered. | ||||||
Entries with a `raw` codec `CID` return raw entries: | ||||||
|
||||||
```javascript | ||||||
const exporter = require('ipfs-unixfs-exporter') | ||||||
const pull = require('pull-stream') | ||||||
const collect = require('pull-stream/sinks/collect') | ||||||
|
||||||
pull( | ||||||
exporter('QmFoo.../bar/baz.txt', ipld, { | ||||||
fullPath: true | ||||||
}) | ||||||
collect((err, files) => { | ||||||
console.info(files) | ||||||
|
||||||
// [{ | ||||||
// depth: 0, | ||||||
// name: 'QmFoo...', | ||||||
// path: 'QmFoo...', | ||||||
// size: ... | ||||||
// cid: CID | ||||||
// content: undefined | ||||||
// type: 'dir' | ||||||
// }, { | ||||||
// depth: 1, | ||||||
// name: 'bar', | ||||||
// path: 'QmFoo.../bar', | ||||||
// size: ... | ||||||
// cid: CID | ||||||
// content: undefined | ||||||
// type: 'dir' | ||||||
// }, { | ||||||
// depth: 2, | ||||||
// name: 'baz.txt', | ||||||
// path: 'QmFoo.../bar/baz.txt', | ||||||
// size: ... | ||||||
// cid: CID | ||||||
// content: <Pull stream> | ||||||
// type: 'file' | ||||||
// }] | ||||||
// | ||||||
}) | ||||||
) | ||||||
{ | ||||||
name: 'foo.txt', | ||||||
path: 'Qmbar/foo.txt', | ||||||
cid: CID, // see https://github.com/multiformats/js-cid | ||||||
node: Buffer, // see https://nodejs.org/api/buffer.html | ||||||
content: function, // returns an async iterator | ||||||
} | ||||||
``` | ||||||
|
||||||
`entry.content()` returns an async iterator that emits buffers containing the node content: | ||||||
achingbrain marked this conversation as resolved.
Show resolved
Hide resolved
achingbrain marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||
|
||||||
```javascript | ||||||
for await (const chunk of entry.content()) { | ||||||
// chunk is a Buffer | ||||||
} | ||||||
``` | ||||||
|
||||||
### `maxDepth` | ||||||
#### CBOR entries | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. hmm, interesting...
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, through the I'm not sure what go does. The name starts to become misleading, true - might be worth turning it into a |
||||||
|
||||||
If specified the exporter will only emit entries up to the specified depth. | ||||||
Entries with a `dag-cbor` codec `CID` return JavaScript object entries: | ||||||
|
||||||
```javascript | ||||||
const exporter = require('ipfs-unixfs-exporter') | ||||||
const pull = require('pull-stream') | ||||||
const collect = require('pull-stream/sinks/collect') | ||||||
|
||||||
pull( | ||||||
exporter('QmFoo.../bar/baz.txt', ipld, { | ||||||
fullPath: true, | ||||||
maxDepth: 1 | ||||||
}) | ||||||
collect((err, files) => { | ||||||
console.info(files) | ||||||
|
||||||
// [{ | ||||||
// depth: 0, | ||||||
// name: 'QmFoo...', | ||||||
// path: 'QmFoo...', | ||||||
// size: ... | ||||||
// cid: CID | ||||||
// content: undefined | ||||||
// type: 'dir' | ||||||
// }, { | ||||||
// depth: 1, | ||||||
// name: 'bar', | ||||||
// path: 'QmFoo.../bar', | ||||||
// size: ... | ||||||
// cid: CID | ||||||
// content: undefined | ||||||
// type: 'dir' | ||||||
// }] | ||||||
// | ||||||
}) | ||||||
) | ||||||
{ | ||||||
name: 'foo.txt', | ||||||
path: 'Qmbar/foo.txt', | ||||||
cid: CID, // see https://github.com/multiformats/js-cid | ||||||
node: Object, // see https://github.com/ipld/js-ipld-dag-cbor | ||||||
} | ||||||
``` | ||||||
|
||||||
There is no `content` function for a `CBOR` node. | ||||||
|
||||||
|
||||||
#### `entry.content({ offset, length })` | ||||||
|
||||||
When `entry` is a file or a `raw` node, `offset` and/or `length` arguments can be passed to `entry.content()` to return slices of data: | ||||||
|
||||||
```javascript | ||||||
const bufs = [] | ||||||
|
||||||
for await (const chunk of entry.content({ | ||||||
offset: 0, | ||||||
length: 5 | ||||||
})) { | ||||||
bufs.push(chunk) | ||||||
} | ||||||
|
||||||
// `data` contains the first 5 bytes of the file | ||||||
const data = Buffer.concat(bufs) | ||||||
``` | ||||||
|
||||||
If `entry` is a directory or hamt shard, passing `offset` and/or `length` to `entry.content()` will limit the number of files return from the directory. | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
|
||||||
```javascript | ||||||
const entries = [] | ||||||
|
||||||
for await (const entry of dir.content({ | ||||||
offset: 0, | ||||||
length: 5 | ||||||
})) { | ||||||
entries.push(entry) | ||||||
} | ||||||
|
||||||
// `entries` contains the first 5 files/directories in the directory | ||||||
``` | ||||||
|
||||||
### `exporter.path(cid, ipld)` | ||||||
|
||||||
`exporter.path` will return an async iterator that yields entries for all segments in a path: | ||||||
|
||||||
```javascript | ||||||
const entries = [] | ||||||
|
||||||
for await (const entry of exporter('Qmfoo/foo/bar/baz.txt', ipld)) { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
entries.push(entry) | ||||||
} | ||||||
|
||||||
// entries contains 4x `entry` objects | ||||||
``` | ||||||
|
||||||
[dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,7 +17,8 @@ | |
"release": "aegir release", | ||
"release-minor": "aegir release --type minor", | ||
"release-major": "aegir release --type major", | ||
"coverage": "aegir coverage" | ||
"coverage": "nyc -s npm run test:node && nyc report --reporter=html", | ||
"dep-check": "aegir dep-check" | ||
}, | ||
"repository": { | ||
"type": "git", | ||
|
@@ -37,29 +38,28 @@ | |
"homepage": "https://github.com/ipfs/js-ipfs-unixfs-exporter#readme", | ||
"devDependencies": { | ||
"aegir": "^18.0.2", | ||
"async-iterator-all": "0.0.2", | ||
"async-iterator-buffer-stream": "0.0.1", | ||
"async-iterator-first": "0.0.2", | ||
"chai": "^4.2.0", | ||
"detect-node": "^2.0.4", | ||
"dirty-chai": "^2.0.1", | ||
"ipld": "~0.21.1", | ||
"ipld": "~0.22.0", | ||
"ipld-dag-pb": "~0.15.2", | ||
"ipld-in-memory": "^2.0.0", | ||
"pull-pushable": "^2.2.0", | ||
"pull-stream-to-stream": "^1.3.4", | ||
"pull-zip": "^2.0.1", | ||
"sinon": "^7.1.0", | ||
"stream-to-pull-stream": "^1.7.2" | ||
"multicodec": "~0.5.1", | ||
"multihashes": "~0.4.14", | ||
"nyc": "^14.0.0", | ||
"sinon": "^7.1.0" | ||
}, | ||
"dependencies": { | ||
"async": "^2.6.1", | ||
"async-iterator-last": "0.0.2", | ||
"cids": "~0.5.5", | ||
"err-code": "^1.1.2", | ||
"hamt-sharding": "0.0.2", | ||
"ipfs-unixfs": "~0.1.16", | ||
"ipfs-unixfs-importer": "~0.38.0", | ||
"pull-cat": "^1.1.11", | ||
"pull-defer": "~0.2.3", | ||
"pull-paramap": "^1.2.2", | ||
"pull-stream": "^3.6.9", | ||
"pull-traverse": "^1.0.3" | ||
"ipfs-unixfs-importer": "ipfs/js-ipfs-unixfs-importer#async-await", | ||
"promisify-es6": "^1.0.3" | ||
}, | ||
"contributors": [ | ||
"Alan Shaw <[email protected]>", | ||
|
Uh oh!
There was an error while loading. Please reload this page.