Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit bf56200

Browse files
dignifiedquiredignifiedquire
committed
Merge pull request #306 from ipfs/update/files-add-interface
update/files add interface
2 parents bddefb1 + 20d880c commit bf56200

File tree

13 files changed

+119
-94
lines changed

13 files changed

+119
-94
lines changed

‎gulpfile.js

+4-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ let nodes = []
1010

1111
function startNode (num, done) {
1212
createTempNode(num, (err, node) => {
13-
if (err) throw err
13+
if (err) {
14+
throw err
15+
}
1416

1517
const api = new API(node.repo.path())
1618
nodes.push(api)
@@ -23,7 +25,7 @@ gulp.task('libnode:start', (done) => {
2325
parallel([
2426
(cb) => startNode(7, cb),
2527
(cb) => startNode(8, cb),
26-
(cb) => startNode(9, cb)
28+
(cb) => startNode(13, cb)
2729
], done)
2830
})
2931

‎package.json

+6-4
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
"form-data": "^1.0.0-rc3",
4646
"gulp": "^3.9.1",
4747
"idb-plus-blob-store": "^1.1.2",
48-
"interface-ipfs-core": "^0.1.5",
48+
"interface-ipfs-core": "^0.2.2",
4949
"left-pad": "^1.1.0",
5050
"lodash": "^4.11.2",
5151
"mocha": "^2.5.1",
@@ -66,14 +66,15 @@
6666
"fs-blob-store": "^5.2.1",
6767
"glob": "^7.0.3",
6868
"hapi": "^13.4.1",
69-
"ipfs-api": "^4.1.0",
7069
"ipfs-bitswap": "^0.4.1",
70+
"ipfs-api": "^5.0.1",
7171
"ipfs-block": "^0.3.0",
7272
"ipfs-block-service": "^0.4.0",
7373
"ipfs-merkle-dag": "^0.6.0",
7474
"ipfs-multipart": "^0.1.0",
7575
"ipfs-repo": "^0.8.0",
76-
"ipfs-unixfs-engine": "^0.8.0",
76+
"ipfs-unixfs-engine": "^0.9.0",
77+
"isstream": "^0.1.2",
7778
"joi": "^8.0.5",
7879
"libp2p-ipfs": "^0.11.0",
7980
"libp2p-ipfs-browser": "^0.10.0",
@@ -92,7 +93,8 @@
9293
"run-parallel-limit": "^1.0.3",
9394
"run-series": "^1.1.4",
9495
"run-waterfall": "^1.1.3",
95-
"temp": "^0.8.3"
96+
"temp": "^0.8.3",
97+
"through2": "^2.0.1"
9698
},
9799
"contributors": [
98100
"Andrew de Andrade <[email protected]>",

‎src/cli/commands/files/add.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ module.exports = Command.extend({
7373
if (!fs.statSync(element).isDirectory()) {
7474
i.write({
7575
path: element.substring(index + 1, element.length),
76-
stream: fs.createReadStream(element)
76+
content: fs.createReadStream(element)
7777
})
7878
}
7979
callback()
@@ -86,7 +86,7 @@ module.exports = Command.extend({
8686
} else {
8787
rs = fs.createReadStream(inPath)
8888
inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length)
89-
filePair = {path: inPath, stream: rs}
89+
filePair = {path: inPath, content: rs}
9090
i.write(filePair)
9191
i.end()
9292
}

‎src/cli/commands/files/cat.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ module.exports = Command.extend({
3636
throw (err)
3737
}
3838
res.on('data', (data) => {
39-
data.stream.pipe(process.stdout)
39+
data.content.pipe(process.stdout)
4040
})
4141
})
4242
})

‎src/cli/commands/files/get.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ function fileHandler (result, dir) {
4848
const dirPath = path.join(dir, file.path)
4949
// Check to see if the result is a directory
5050
if (file.dir === false) {
51-
file.stream.pipe(fs.createWriteStream(dirPath))
51+
file.content.pipe(fs.createWriteStream(dirPath))
5252
} else {
5353
ensureDir(dirPath, (err) => {
5454
if (err) {
@@ -64,7 +64,7 @@ function fileHandler (result, dir) {
6464
throw err
6565
}
6666

67-
file.stream.pipe(fs.createWriteStream(dirPath))
67+
file.content.pipe(fs.createWriteStream(dirPath))
6868
})
6969
}
7070
}

‎src/core/ipfs/files.js

+82-17
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,105 @@
11
'use strict'
22

3-
const Importer = require('ipfs-unixfs-engine').importer
4-
const Exporter = require('ipfs-unixfs-engine').exporter
3+
const unixfsEngine = require('ipfs-unixfs-engine')
4+
const Importer = unixfsEngine.Importer
5+
const Exporter = unixfsEngine.Exporter
56
const UnixFS = require('ipfs-unixfs')
7+
const through = require('through2')
8+
const isStream = require('isstream')
9+
const promisify = require('promisify-es6')
10+
const Duplex = require('stream').Duplex
11+
const multihashes = require('multihashes')
612

713
module.exports = function files (self) {
814
return {
9-
add: (arr, callback) => {
10-
if (typeof arr === 'function') {
11-
callback = arr
12-
arr = undefined
15+
createAddStream: (callback) => {
16+
const i = new Importer(self._dagS)
17+
const ds = new Duplex({ objectMode: true })
18+
19+
ds._read = (n) => {}
20+
ds._write = (file, enc, next) => {
21+
i.write(file)
22+
next()
1323
}
14-
if (callback === undefined) {
24+
25+
ds.end = () => {
26+
i.end()
27+
}
28+
29+
let counter = 0
30+
31+
i.on('data', (file) => {
32+
counter++
33+
self.object.get(file.multihash, (err, node) => {
34+
if (err) {
35+
return ds.emit('error', err)
36+
}
37+
ds.push({path: file.path, node: node})
38+
counter--
39+
})
40+
})
41+
42+
i.on('end', () => {
43+
function canFinish () {
44+
if (counter === 0) {
45+
ds.push(null)
46+
} else {
47+
setTimeout(canFinish, 100)
48+
}
49+
}
50+
canFinish()
51+
})
52+
53+
callback(null, ds)
54+
},
55+
add: promisify((data, callback) => {
56+
// Buffer input
57+
if (Buffer.isBuffer(data)) {
58+
data = [{
59+
path: '',
60+
content: data
61+
}]
62+
}
63+
// Readable stream input
64+
if (isStream.isReadable(data)) {
65+
data = [{
66+
path: '',
67+
content: data
68+
}]
69+
}
70+
if (!callback || typeof callback !== 'function') {
1571
callback = function noop () {}
1672
}
17-
if (arr === undefined) {
18-
return new Importer(self._dagS)
73+
if (!Array.isArray(data)) {
74+
return callback(new Error('"data" must be an array of { path: string, content: Buffer|Readable } or Buffer or Readable'))
1975
}
2076

2177
const i = new Importer(self._dagS)
2278
const res = []
2379

24-
i.on('data', (info) => {
25-
res.push(info)
26-
})
27-
28-
i.once('end', () => {
80+
// Transform file info tuples to DAGNodes
81+
i.pipe(through.obj((info, enc, next) => {
82+
const mh = multihashes.toB58String(info.multihash)
83+
self._dagS.get(mh, (err, node) => {
84+
if (err) return callback(err)
85+
var obj = {
86+
path: info.path || mh,
87+
node: node
88+
}
89+
res.push(obj)
90+
next()
91+
})
92+
}, (done) => {
2993
callback(null, res)
30-
})
94+
}))
3195

32-
arr.forEach((tuple) => {
96+
data.forEach((tuple) => {
3397
i.write(tuple)
3498
})
3599

36100
i.end()
37-
},
101+
}),
102+
38103
cat: (hash, callback) => {
39104
self._dagS.get(hash, (err, fetchedNode) => {
40105
if (err) {

‎src/core/ipfs/init.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ module.exports = function init (self) {
9191
const rs = new Readable()
9292
rs.push(fs.readFileSync(element))
9393
rs.push(null)
94-
const filePair = {path: addPath, stream: rs}
94+
const filePair = {path: addPath, content: rs}
9595
i.write(filePair)
9696
}
9797
callback()

‎src/http-api/resources/files.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ exports.cat = {
4343
}).code(500)
4444
}
4545
stream.on('data', (data) => {
46-
return reply(data.stream)
46+
return reply(data.content)
4747
})
4848
})
4949
}

‎test/cli/test-bitswap.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ const createTempNode = require('../utils/temp-node')
1212
const repoPath = require('./index').repoPath
1313

1414
describe('bitswap', function () {
15-
this.timeout(20000)
15+
this.timeout(40000)
1616
const env = _.clone(process.env)
1717
env.IPFS_PATH = repoPath
1818

‎test/core/both/test-bitswap.js

+5-4
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ describe('bitswap', () => {
8181
// need timeout so we wait for identify to happen
8282
// in the browsers
8383
connectNodesSingle(node2, node1, cb)
84-
}, 100)
84+
}, 300)
8585
], done)
8686
}
8787

@@ -101,15 +101,16 @@ describe('bitswap', () => {
101101
})
102102

103103
afterEach((done) => {
104-
setTimeout(() => ipfs.goOffline(done), 500)
104+
// ipfs.goOffline(done)
105+
setTimeout(() => ipfs.goOffline(done), 1500)
105106
})
106107

107108
it('2 peers', (done) => {
108109
const block = makeBlock()
109110
let node
110111
series([
111112
// 0. Start node
112-
(cb) => addNode(9, (err, _ipfs) => {
113+
(cb) => addNode(13, (err, _ipfs) => {
113114
node = _ipfs
114115
cb(err)
115116
}),
@@ -196,7 +197,7 @@ describe('bitswap', () => {
196197
ipfs.files.cat(hash, (err, res) => {
197198
expect(err).to.not.exist
198199
res.on('file', (data) => {
199-
data.stream.pipe(bl((err, bldata) => {
200+
data.content.pipe(bl((err, bldata) => {
200201
expect(err).to.not.exist
201202
expect(bldata.toString()).to.equal('I love IPFS <3')
202203
cb()

‎test/core/both/test-files.js

+12-57
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,19 @@
11
/* eslint-env mocha */
22
'use strict'
33

4-
const bl = require('bl')
5-
const expect = require('chai').expect
6-
const Readable = require('stream').Readable
7-
const bs58 = require('bs58')
8-
4+
const test = require('interface-ipfs-core')
95
const IPFS = require('../../../src/core')
106

11-
describe('files', () => {
12-
let ipfs
13-
14-
before((done) => {
15-
ipfs = new IPFS(require('../../utils/repo-path'))
16-
ipfs.load(done)
17-
})
18-
19-
it('add', (done) => {
20-
const buffered = new Buffer('some data')
21-
const rs = new Readable()
22-
rs.push(buffered)
23-
rs.push(null)
24-
const arr = []
25-
const filePair = {path: 'data.txt', stream: rs}
26-
arr.push(filePair)
27-
ipfs.files.add(arr, (err, res) => {
28-
expect(err).to.not.exist
29-
expect(res[0].path).to.equal('data.txt')
30-
expect(res[0].size).to.equal(17)
31-
expect(bs58.encode(res[0].multihash).toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS')
32-
done()
7+
const common = {
8+
setup: function (cb) {
9+
const ipfs = new IPFS(require('../../utils/repo-path'))
10+
ipfs.load(() => {
11+
cb(null, ipfs)
3312
})
34-
})
13+
},
14+
teardown: function (cb) {
15+
cb()
16+
}
17+
}
3518

36-
it('cat', (done) => {
37-
const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o'
38-
ipfs.files.cat(hash, (err, res) => {
39-
expect(err).to.not.exist
40-
res.on('data', (data) => {
41-
data.stream.pipe(bl((err, bldata) => {
42-
expect(err).to.not.exist
43-
expect(bldata.toString()).to.equal('hello world\n')
44-
done()
45-
}))
46-
})
47-
})
48-
})
49-
50-
it('get', (done) => {
51-
// TODO create non-trival get test
52-
const hash = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o'
53-
ipfs.files.get(hash, (err, res) => {
54-
expect(err).to.not.exist
55-
res.on('data', (data) => {
56-
data.stream.pipe(bl((err, bldata) => {
57-
expect(err).to.not.exist
58-
expect(bldata.toString()).to.equal('hello world\n')
59-
done()
60-
}))
61-
})
62-
})
63-
})
64-
})
19+
test.files(common)

‎test/core/node-only/test-swarm.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ const parallel = require('run-parallel')
77
const createTempNode = require('../../utils/temp-node')
88

99
describe('swarm', function () {
10-
this.timeout(20 * 1000)
10+
this.timeout(40 * 1000)
1111

1212
let nodeA
1313
let nodeB

‎test/http-api/test-files.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ module.exports = (httpAPI) => {
99
describe('api', () => {
1010
let api
1111

12-
it('api', () => {
12+
before(() => {
1313
api = httpAPI.server.select('API')
1414
})
1515

0 commit comments

Comments
 (0)