feat: streaming http requests

No more 4GB file limits.

Still need to get http tests passing properly.

Author: achingbrain

packages/interface-ipfs-core/package.json

@@ -52,7 +52,8 @@
     "multihashes": "^0.4.14",
     "multihashing-async": "^0.8.0",
     "peer-id": "^0.13.5",
-    "readable-stream": "^3.4.0"
+    "readable-stream": "^3.4.0",
+    "temp-write": "^4.0.0"
   },
   "devDependencies": {
     "aegir": "^21.3.0",

packages/interface-ipfs-core/src/files/stat.js

@@ -6,7 +6,6 @@ const all = require('it-all')
 const { fixtures } = require('../utils')
 const { getDescribe, getIt, expect } = require('../utils/mocha')
 const createShardedDirectory = require('../utils/create-sharded-directory')
-const mc = require('multicodec')
 const CID = require('cids')
 const mh = require('multihashes')
 const Block = require('ipfs-block')

packages/interface-ipfs-core/src/files/write.js

@@ -61,7 +61,7 @@ module.exports = (common, options) => {
     })
   }
 
-  describe.only('.files.write', function () {
+  describe('.files.write', function () {
     this.timeout(40 * 1000)
 
     let ipfs
@@ -93,19 +93,15 @@ module.exports = (common, options) => {
     }
 
     before(async () => {
-      console.info('spawning')
       ipfs = (await common.spawn()).api
-      console.info('spawned')
     })
 
     after(() => common.clean())
 
     it('explodes if it cannot convert content to a source', async () => {
-      console.info('writing')
       await expect(ipfs.files.write('/foo', -1, {
         create: true
       })).to.eventually.be.rejected()
-      console.info('wrote')
     })
 
     it('explodes if given an invalid path', async () => {

packages/interface-ipfs-core/src/utils/create-shard.js

@@ -5,7 +5,7 @@ const last = require('it-last')
 const createShard = async (ipfs, files, shardSplitThreshold = 10) => {
   const result = await last(ipfs.add(files, {
     enableShardingExperiment: true,
-    shardSplitThreshold,
+    shardSplitThreshold
     // reduceSingleLeafToSelf: false, // same as go-ipfs-mfs implementation, differs from `ipfs add`(!)
     // rawLeaves: 'raw' // same as go-ipfs-mfs implementation, differs from `ipfs add`(!)
   }))

packages/ipfs-http-client/package.json

@@ -47,6 +47,7 @@
     "cids": "^0.7.3",
     "debug": "^4.1.0",
     "form-data": "^3.0.0",
+    "hat": "0.0.3",
     "ipfs-block": "^0.8.1",
     "ipfs-utils": "^0.7.2",
     "ipld-dag-cbor": "^0.15.1",

packages/ipfs-http-client/src/add/index.js renamed to packages/ipfs-http-client/src/add.js

@@ -2,9 +2,9 @@
 
 const CID = require('cids')
 const merge = require('merge-options')
-const { toFormData } = require('./form-data')
-const toCamel = require('../lib/object-to-camel')
-const configure = require('../lib/configure')
+const toCamel = require('./lib/object-to-camel')
+const configure = require('./lib/configure')
+const multipartRequest = require('./lib/multipart-request')
 
 module.exports = configure((api) => {
   return async function * add (input, options = {}) {
@@ -18,10 +18,13 @@ module.exports = configure((api) => {
       }
     )
 
+    const { body, headers } = multipartRequest(input)
+
     const res = await api.ndjson('add', {
       method: 'POST',
       searchParams: options,
-      body: await toFormData(input),
+      headers,
+      body: body,
       timeout: options.timeout,
       signal: options.signal
     })

packages/ipfs-http-client/src/add/form-data.browser.js

@@ -1,34 +1,32 @@
 'use strict'
 
-const toFormData = require('../lib/buffer-to-form-data')
+const normaliseInput = require('ipfs-utils/src/files/normalise-input')
 const modeToString = require('../lib/mode-to-string')
 const mtimeToObject = require('../lib/mtime-to-object')
 const configure = require('../lib/configure')
+const multipartRequest = require('../lib/multipart-request')
 
 module.exports = configure(api => {
   return async (path, input, options = {}) => {
-    const mtime = mtimeToObject(options.mtime)
-
     const searchParams = new URLSearchParams(options)
     searchParams.set('arg', path)
     searchParams.set('stream-channels', 'true')
     searchParams.set('hash', options.hashAlg)
     searchParams.set('hashAlg', null)
-    if (mtime) {
-      searchParams.set('mtime', mtime.secs)
-      searchParams.set('mtimeNsecs', mtime.nsecs)
-    }
+
+    const { body, headers } = multipartRequest(normaliseInput({
+      content: input,
+      path: path,
+      mode: modeToString(options.mode),
+      mtime: mtimeToObject(options.mtime)
+    }))
 
     const res = await api.post('files/write', {
       timeout: options.timeout,
       signal: options.signal,
       searchParams,
-      body: multipartRequest([{
-        content: input,
-        path: path,
-        mode: options.mode,
-        mtime: options.mtime
-      }], boundary)
+      headers,
+      body
     })
 
     return res.text()