data exporting

Author: nginnever

.npmignore

@@ -0,0 +1,33 @@
+
+.DS_Store
+tests/repo-tests*
+
+# Logs
+logs
+*.log
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
+node_modules
+
+test

.travis.yml

@@ -1,11 +1,17 @@
+sudo: false
 language: node_js
 node_js:
-  - '4'
-  - '5'
+  - 4
+  - 5
 
+# Make sure we have new NPM.
 before_install:
-  - npm i -g npm
-  # Workaround for a permissions issue with Travis virtual machine images
+  - npm install -g npm
+
+script:
+  - npm run lint
+  - npm test
+  - npm run coverage
 
 addons:
   firefox: 'latest'
@@ -14,6 +20,5 @@ before_script:
   - export DISPLAY=:99.0
   - sh -e /etc/init.d/xvfb start
 
-script:
-  - npm run lint
-  - npm test
+after_success:
+  - npm run coverage-publish

README.md

@@ -6,7 +6,7 @@ IPFS Data Importing
 [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
 [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs)
 [![Build Status](https://travis-ci.org/ipfs/js-ipfs-data-importing.svg?style=flat-square)](https://travis-ci.org/ipfs/js-ipfs-data-importing)
-![](https://img.shields.io/badge/coverage-%3F-yellow.svg?style=flat-square)
+[![Coverage Status](https://coveralls.io/repos/github/ipfs/js-ipfs-data-importing/badge.svg?branch=master)](https://coveralls.io/github/ipfs/js-ipfs-data-importing?branch=master)
 [![Dependency Status](https://david-dm.org/ipfs/js-ipfs-data-importing.svg?style=flat-square)](https://david-dm.org/ipfs/js-ipfs-data-importing)
 [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/feross/standard)
 

circle.yml

@@ -0,0 +1,12 @@
+machine:
+  node:
+    version: stable
+
+dependencies:
+  pre:
+    - google-chrome --version
+    - wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add -
+    - sudo sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
+    - sudo apt-get update
+    - sudo apt-get --only-upgrade install google-chrome-stable
+    - google-chrome --version

package.json

@@ -2,14 +2,17 @@
   "name": "ipfs-data-importing",
   "version": "0.3.3",
   "description": "JavaScript implementation of the layout and chunking mechanisms used by IPFS",
-  "main": "src/index.js",
+  "main": "lib/index.js",
+  "jsnext:main": "src/index.js",
   "scripts": {
-    "lint": "dignified-lint",
-    "build": "dignified-build",
-    "test": "dignified-test",
-    "test:node": "dignified-test node",
-    "test:browser": "dignified-test browser",
-    "release": "dignified-release"
+    "lint": "aegir-lint",
+    "build": "aegir-build",
+    "test": "aegir-test",
+    "test:node": "aegir-test node",
+    "test:browser": "aegir-test browser",
+    "release": "aegir-release",
+    "coverage": "aegir-coverage",
+    "coverage-publish": "aegir-coverage publish"
   },
   "pre-commit": [
     "lint",
@@ -29,26 +32,16 @@
   },
   "homepage": "https://github.com/diasdavid/js-ipfs-data-importing#readme",
   "devDependencies": {
-    "brfs": "^1.4.3",
+    "aegir": "^2.1.1",
     "block-stream2": "^1.1.0",
+    "brfs": "^1.4.3",
     "bs58": "^3.0.0",
     "buffer-loader": "0.0.1",
-    "chai": "^3.4.1",
-    "dignified.js": "^1.0.0",
+    "chai": "^3.5.0",
     "fs-blob-store": "^5.2.1",
     "highland": "^2.7.4",
     "idb-plus-blob-store": "^1.0.0",
     "ipfs-repo": "^0.6.1",
-    "istanbul": "^0.4.1",
-    "json-loader": "^0.5.4",
-    "karma": "^0.13.19",
-    "karma-chrome-launcher": "^0.2.2",
-    "karma-cli": "^0.1.2",
-    "karma-firefox-launcher": "^0.1.7",
-    "karma-mocha": "^0.2.1",
-    "karma-sourcemap-loader": "^0.3.7",
-    "karma-spec-reporter": "0.0.26",
-    "karma-webpack": "^1.7.0",
     "mocha": "^2.3.4",
     "ncp": "^2.0.0",
     "pre-commit": "^1.1.2",
@@ -61,7 +54,7 @@
     "block-stream2": "^1.1.0",
     "debug": "^2.2.0",
     "ipfs-blocks": "^0.1.2",
-    "ipfs-merkle-dag": "^0.3.0",
+    "ipfs-merkle-dag": "^0.4.0",
     "ipfs-unixfs": "^0.1.0",
     "through2": "^2.0.0"
   }

src/index.js

@@ -9,6 +9,9 @@ const FixedSizeChunker = require('./chunker-fixed-size')
 const through2 = require('through2')
 const UnixFS = require('ipfs-unixfs')
 const async = require('async')
+const events = require('events')
+const Readable = require('stream').Readable
+const pathj = require('path')
 
 exports = module.exports
 
@@ -206,7 +209,6 @@ exports.import = (target, dagService, options, callback) => {
             leafSize: raw.fileSize(),
             Name: ''
           })
-
           cb()
         })
       }, (cb) => {
@@ -249,6 +251,114 @@ exports.import = (target, dagService, options, callback) => {
   // function streamImporter (stream, callback) {}
 }
 
-exports.export = function () {
-  // export into files by hash
+exports.export = function (hash, dagService, options, callback) {
+  if (typeof options === 'function') {
+    callback = options
+    options = {}
+  }
+
+  const ee = new events.EventEmitter()
+  dagService.get(hash, (err, fetchedNode) => {
+    if (err) {
+      if (callback) {
+        return callback(err)
+      }
+      return
+    }
+    const data = UnixFS.unmarshal(fetchedNode.data)
+    const type = data.type
+    if (type === 'directory') {
+      dirExporter(fetchedNode, hash, callback)
+    }
+    if (type === 'file') {
+      fileExporter(fetchedNode, hash, false, callback)
+    }
+  })
+  return ee
+
+  function fileExporter (node, name, dir, callback) {
+    if (typeof dir === 'function') { callback = dir; dir = {} }
+    var rs = new Readable()
+    if (node.links.length === 0) {
+      const unmarshaledData = UnixFS.unmarshal(node.data)
+      ee.emit('file', { stream: rs, path: name, dir: dir })
+      rs.push(unmarshaledData.data)
+      rs.push(null)
+      if (callback) {
+        callback()
+      }
+      return
+    } else {
+      ee.emit('file', { stream: rs, path: name, dir: dir })
+      var init = false
+      rs._read = () => {
+        if (init) {
+          return
+        }
+        init = true
+        async.forEachSeries(node.links, (link, callback) => {
+          dagService.get(link.hash, (err, res) => {
+            if (err) {
+              callback(err)
+            }
+            var unmarshaledData = UnixFS.unmarshal(res.data)
+            rs.push(unmarshaledData.data)
+            callback()
+          })
+        }, (err) => {
+          if (err) {
+            if (callback) {
+              return callback(err)
+            }
+            return
+          }
+          rs.push(null)
+          if (callback) {
+            callback()
+          }
+          return
+        })
+      }
+    }
+  }
+
+  function dirExporter (node, name, callback) {
+    var rs = new Readable()
+    if (node.links.length === 0) {
+      rs.push(node.data)
+      rs.push(null)
+      ee.emit('file', {stream: rs, path: name})
+      if (callback) {
+        callback()
+      }
+      return
+    } else {
+      async.forEachSeries(node.links, (link, callback) => {
+        dagService.get(link.hash, (err, res) => {
+          if (err) {
+            callback(err)
+          }
+          var unmarshaledData = UnixFS.unmarshal(res.data)
+          if (unmarshaledData.type === 'file') {
+            return (fileExporter(res, pathj.join(name, link.name), callback))
+          }
+          if (unmarshaledData.type === 'directory') {
+            return (dirExporter(res, pathj.join(name, link.name), callback))
+          }
+          callback()
+        })
+      }, (err) => {
+        if (err) {
+          if (callback) {
+            return callback(err)
+          }
+          return
+        }
+        if (callback) {
+          callback()
+        }
+        return
+      })
+    }
+  }
 }

test/buffer-test.js

@@ -82,5 +82,60 @@ module.exports = function (repo) {
         })
       })
     })
+
+    it('export a file with no links', (done) => {
+      const hash = 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      testExport.on('file', (data) => {
+        ds.get(hash, (err, fetchedNode) => {
+          expect(err).to.not.exist
+          const unmarsh = UnixFS.unmarshal(fetchedNode.data)
+          expect(unmarsh.data).to.deep.equal(data.stream._readableState.buffer[0])
+          done()
+        })
+      })
+    })
+
+    it('export a small file with links', (done) => {
+      const hash = 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      testExport.on('file', (data) => {
+        expect(data.stream).to.exist
+        done()
+      })
+    })
+
+    it('export a large file > 5mb', (done) => {
+      const hash = 'QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      testExport.on('file', (data) => {
+        expect(data.stream).to.exist
+        done()
+      })
+    })
+
+    it('export a directory', (done) => {
+      const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      var fs = []
+      testExport.on('file', (data) => {
+        fs.push(data)
+      })
+      setTimeout(() => {
+        expect(fs[0].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/200Bytes.txt')
+        expect(fs[1].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/dir-another')
+        expect(fs[2].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/200Bytes.txt')
+        expect(fs[3].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/level-2')
+        done()
+      }, 1000)
+    })
   })
 }

test/repo-example/blocks/12200046/12200046ea932273bc69bf22ccd57a2837eee93896de64e70099724ae333c67eebae.data

@@ -0,0 +1,4 @@
+5
+" $��G�,�A�4{���x�Z/.����D`�200Bytes.txt�
/
+" Y��9_)a���˹2�R�m�Ŗke�9��level-2
+

test/repo-example/blocks/1220016d/1220016d22c790b4e50495df66eff6361276442eff53b5457cf8de5c2c6726325ca7.data

@@ -0,0 +1,5 @@
+5
+" $��G�,�A�4{���x�Z/.����D`�200Bytes.txt�
3
+" Y��9_)a���˹2�R�m�Ŗke�9��dir-another0
+" Ty�5;_9Yf�q��F�Lhyl���/��level-1�
+