fix(cli): make ipfs files add work online and offline

Fixes #480

Author: dignifiedquire

src/cli/commands/files/add.js

@@ -9,9 +9,9 @@ const path = require('path')
 const glob = require('glob')
 const sortBy = require('lodash.sortby')
 const pull = require('pull-stream')
-const pullFile = require('pull-file')
 const paramap = require('pull-paramap')
 const zip = require('pull-zip')
+const toPull = require('stream-to-pull-stream')
 
 function checkPath (inPath, recursive) {
   // This function is to check for the following possible inputs
@@ -59,46 +59,71 @@ module.exports = {
         throw err
       }
 
-      glob(path.join(inPath, '/**/*'), (err, list) => {
+      // TODO: revist when interface-ipfs-core exposes pull-streams
+      let createAddStream = (cb) => {
+        ipfs.files.createAddStream((err, stream) => {
+          cb(err, err ? null : toPull.transform(stream))
+        })
+      }
+
+      if (typeof ipfs.files.createAddPullStream === 'function') {
+        createAddStream = (cb) => {
+          cb(null, ipfs.files.createAddPullStream())
+        }
+      }
+
+      createAddStream((err, addStream) => {
         if (err) {
           throw err
         }
-        if (list.length === 0) {
-          list = [inPath]
-        }
 
-        pull(
-          zip(
-            pull.values(list),
-            pull(
-              pull.values(list),
-              paramap(fs.stat.bind(fs), 50)
-            )
-          ),
-          pull.map((pair) => ({
-            path: pair[0],
-            isDirectory: pair[1].isDirectory()
-          })),
-          pull.filter((file) => !file.isDirectory),
-          pull.map((file) => ({
-            path: file.path.substring(index, file.path.length),
-            content: pullFile(file.path)
-          })),
-          ipfs.files.createAddPullStream(),
-          pull.map((file) => ({
-            hash: file.hash,
-            path: file.path
-          })),
-          pull.collect((err, added) => {
-            if (err) throw err
-
-            sortBy(added, 'path')
-              .reverse()
-              .map((file) => `added ${file.hash} ${file.path}`)
-              .forEach((msg) => console.log(msg))
-          })
-        )
+        glob(path.join(inPath, '/**/*'), (err, list) => {
+          if (err) {
+            throw err
+          }
+          if (list.length === 0) {
+            list = [inPath]
+          }
+
+          addPipeline(index, addStream, list)
+        })
       })
     })
   }
 }
+
+function addPipeline (index, addStream, list) {
+  pull(
+    zip(
+      pull.values(list),
+      pull(
+        pull.values(list),
+        paramap(fs.stat.bind(fs), 50)
+      )
+    ),
+    pull.map((pair) => ({
+      path: pair[0],
+      isDirectory: pair[1].isDirectory()
+    })),
+    pull.filter((file) => !file.isDirectory),
+    pull.map((file) => ({
+      path: file.path.substring(index, file.path.length),
+      content: fs.createReadStream(file.path)
+    })),
+    addStream,
+    pull.map((file) => ({
+      hash: file.hash,
+      path: file.path
+    })),
+    pull.collect((err, added) => {
+      if (err) {
+        throw err
+      }
+
+      sortBy(added, 'path')
+        .reverse()
+        .map((file) => `added ${file.hash} ${file.path}`)
+        .forEach((msg) => console.log(msg))
+    })
+  )
+}

src/core/ipfs/files.js

@@ -13,12 +13,14 @@ const toStream = require('pull-stream-to-stream')
 const toPull = require('stream-to-pull-stream')
 
 module.exports = function files (self) {
-  const createAddPullStream = () => pull(
-    pull.map(normalizeContent),
-    pull.flatten(),
-    importer(self._dagS),
-    pull.asyncMap(prepareFile.bind(null, self))
-  )
+  const createAddPullStream = () => {
+    return pull(
+      pull.map(normalizeContent),
+      pull.flatten(),
+      importer(self._dagS),
+      pull.asyncMap(prepareFile.bind(null, self))
+    )
+  }
 
   return {
     createAddStream: (callback) => {

test/cli/test-files.js

@@ -84,5 +84,39 @@ describe('files', () => {
           done()
         })
     })
+
+    it('add', (done) => {
+      nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'add', 'src/init-files/init-docs/readme'], {env})
+        .run((err, stdout, exitcode) => {
+          expect(err).to.not.exist
+          expect(exitcode).to.equal(0)
+          expect(stdout[0]).to.equal('added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB readme')
+          done()
+        })
+    })
+
+    it('add recursively', (done) => {
+      nexpect.spawn('node', [process.cwd() + '/src/cli/bin.js', 'files', 'add', '-r', 'src/init-files/init-docs'], {env})
+        .run((err, stdout, exitcode) => {
+          expect(err).to.not.exist
+          expect(exitcode).to.equal(0)
+
+          const expected = [
+            'added QmYE7xo6NxbHEVEHej1yzxijYaNY51BaeKxjXxn6Ssa6Bs init-docs/tour/0.0-intro',
+            'added QmciSU8hfpAXKjvK5YLUSwApomGSWN5gFbP4EpDAEzu2Te init-docs/tour',
+            'added QmTumTjvcYCAvRRwQ8sDRxh8ezmrcr88YFU7iYNroGGTBZ init-docs/security-notes',
+            'added QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB init-docs/readme',
+            'added QmdncfsVm2h5Kqq9hPmU7oAVX2zTSVP3L869tgTbPYnsha init-docs/quick-start',
+            'added QmY5heUM5qgRubMDD1og9fhCPA6QdkMp3QCwd4s7gJsyE7 init-docs/help',
+            'added QmQN88TEidd3RY2u3dpib49fERTDfKtDpvxnvczATNsfKT init-docs/docs/index',
+            'added QmegvLXxpVKiZ4b57Xs1syfBVRd8CbucVHAp7KpLQdGieC init-docs/docs',
+            'added QmYCvbfNbCwFR45HiNP45rwJgvatpiW38D961L5qAhUM5Y init-docs/contact',
+            'added QmZTR5bcpQD7cFgTorqxZDYaew1Wqgfbd2ud9QqGPAkK2V init-docs/about',
+            'added QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU init-docs'
+          ]
+          expect(stdout).to.deep.equal(expected)
+          done()
+        })
+    })
   })
 })