net/http: add js/wasm compatible DefaultTransport

Adds a new Transport type for the js/wasm target that uses the
JavaScript Fetch API for sending HTTP requests. Support for
streaming response bodies is used when available, falling back
to reading the entire response into memory at once.

Author: johanbrandhorst

src/go/build/deps_test.go

@@ -410,6 +410,7 @@ var pkgDeps = map[string][]string{
 		"net/http/httptrace",
 		"net/http/internal",
 		"runtime/debug",
+		"syscall/js",
 	},
 	"net/http/internal":  {"L4"},
 	"net/http/httptrace": {"context", "crypto/tls", "internal/nettrace", "net", "reflect", "time"},

src/net/http/roundtrip.go

@@ -0,0 +1,15 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !js !wasm
+
+package http
+
+// RoundTrip implements the RoundTripper interface.
+//
+// For higher-level HTTP client support (such as handling of cookies
+// and redirects), see Get, Post, and the Client type.
+func (t *Transport) RoundTrip(req *Request) (*Response, error) {
+	return t.roundTrip(req)
+}

src/net/http/roundtrip_js.go

@@ -0,0 +1,267 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build js,wasm
+
+package http
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"strconv"
+	"syscall/js"
+)
+
+// RoundTrip implements the RoundTripper interface using the WHATWG Fetch API.
+func (*Transport) RoundTrip(req *Request) (*Response, error) {
+	if useFakeNetwork(req) {
+		return t.roundTrip(req)
+	}
+	headers := js.Global.Get("Headers").New()
+	for key, values := range req.Header {
+		for _, value := range values {
+			headers.Call("append", key, value)
+		}
+	}
+
+	ac := js.Global.Get("AbortController")
+	if ac != js.Undefined {
+		// Some browsers that support WASM don't necessarily support
+		// the AbortController. See
+		// https://developer.mozilla.org/en-US/docs/Web/API/AbortController#Browser_compatibility.
+		ac = ac.New()
+	}
+
+	opt := js.Global.Get("Object").New()
+	// See https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
+	// for options available.
+	opt.Set("headers", headers)
+	opt.Set("method", req.Method)
+	opt.Set("credentials", "same-origin")
+	if ac != js.Undefined {
+		opt.Set("signal", ac.Get("signal"))
+	}
+
+	if req.Body != nil {
+		// TODO(johanbrandhorst): Stream request body when possible.
+		// See https://bugs.chromium.org/p/chromium/issues/detail?id=688906 for Blink issue.
+		// See https://bugzilla.mozilla.org/show_bug.cgi?id=1387483 for Firefox issue.
+		// See https://github.com/web-platform-tests/wpt/issues/7693 for WHATWG tests issue.
+		// See https://developer.mozilla.org/en-US/docs/Web/API/Streams_API for more details on the Streams API
+		// and browser support.
+		body, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			req.Body.Close() // RoundTrip must always close the body, including on errors.
+			return nil, err
+		}
+		req.Body.Close()
+		opt.Set("body", body)
+	}
+	respPromise := js.Global.Call("fetch", req.URL.String(), opt)
+	var (
+		respCh = make(chan *Response, 1)
+		errCh  = make(chan error, 1)
+	)
+	success := js.NewCallback(func(args []js.Value) {
+		result := args[0]
+		header := Header{}
+		// https://developer.mozilla.org/en-US/docs/Web/API/Headers/entries
+		headersIt := result.Get("headers").Call("entries")
+		for {
+			n := headersIt.Call("next")
+			if n.Get("done").Bool() {
+				break
+			}
+			pair := n.Get("value")
+			key, value := pair.Index(0).String(), pair.Index(1).String()
+			ck := CanonicalHeaderKey(key)
+			header[ck] = append(header[ck], value)
+		}
+
+		contentLength := int64(0)
+		if cl, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64); err == nil {
+			contentLength = cl
+		}
+
+		b := result.Get("body")
+		var body io.ReadCloser
+		if b != js.Undefined {
+			body = &streamReader{stream: b.Call("getReader")}
+		} else {
+			// Fall back to using ArrayBuffer
+			// https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer
+			body = &arrayReader{arrayPromise: result.Call("arrayBuffer")}
+		}
+
+		select {
+		case respCh <- &Response{
+			Status:        result.Get("status").String() + " " + StatusText(result.Get("status").Int()),
+			StatusCode:    result.Get("status").Int(),
+			Header:        header,
+			ContentLength: contentLength,
+			Body:          body,
+			Request:       req,
+		}:
+		case <-req.Context().Done():
+		}
+	})
+	defer success.Close()
+	failure := js.NewCallback(func(args []js.Value) {
+		err := fmt.Errorf("net/http: fetch() failed: %s", args[0].String())
+		select {
+		case errCh <- err:
+		case <-req.Context().Done():
+		}
+	})
+	defer failure.Close()
+	respPromise.Call("then", success, failure)
+	select {
+	case <-req.Context().Done():
+		if ac != js.Undefined {
+			// Abort the Fetch request
+			ac.Call("abort")
+		}
+		return nil, req.Context().Err()
+	case resp := <-respCh:
+		return resp, nil
+	case err := <-errCh:
+		return nil, err
+	}
+}
+
+// useFakeNetwork is used to determine whether the request is made
+// by a test and should be made to use the fake in-memory network.
+func useFakeNetwork(req *Request) bool {
+	host, _, err := net.SplitHostPort(req.Host)
+	if err != nil {
+		host = req.Host
+	}
+	if ip := net.ParseIP(host); ip != nil {
+		return ip.IsLoopback(ip)
+	}
+	return host == "localhost"
+}
+
+// streamReader implements an io.ReadCloser wrapper for ReadableStream.
+// See https://fetch.spec.whatwg.org/#readablestream for more information.
+type streamReader struct {
+	pending []byte
+	stream  js.Value
+	err     error // sticky read error
+}
+
+func (r *streamReader) Read(p []byte) (n int, err error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+	if len(r.pending) == 0 {
+		var (
+			bCh   = make(chan []byte, 1)
+			errCh = make(chan error, 1)
+		)
+		success := js.NewCallback(func(args []js.Value) {
+			result := args[0]
+			if result.Get("done").Bool() {
+				errCh <- io.EOF
+				return
+			}
+			value := make([]byte, result.Get("value").Get("byteLength").Int())
+			js.ValueOf(value).Call("set", result.Get("value"))
+			bCh <- value
+		})
+		defer success.Close()
+		failure := js.NewCallback(func(args []js.Value) {
+			// Assumes it's a TypeError. See
+			// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
+			// for more information on this type. See
+			// https://streams.spec.whatwg.org/#byob-reader-read for the spec on
+			// the read method.
+			errCh <- errors.New(args[0].Get("message").String())
+		})
+		defer failure.Close()
+		r.stream.Call("read").Call("then", success, failure)
+		select {
+		case b := <-bCh:
+			r.pending = b
+		case err := <-errCh:
+			r.err = err
+			return 0, err
+		}
+	}
+	n = copy(p, r.pending)
+	r.pending = r.pending[n:]
+	return n, nil
+}
+
+func (r *streamReader) Close() error {
+	// This ignores any error returned from cancel method. So far, I did not encounter any concrete
+	// situation where reporting the error is meaningful. Most users ignore error from resp.Body.Close().
+	// If there's a need to report error here, it can be implemented and tested when that need comes up.
+	r.stream.Call("cancel")
+	if r.err == nil {
+		r.err = errClosed
+	}
+	return nil
+}
+
+// arrayReader implements an io.ReadCloser wrapper for ArrayBuffer.
+// https://developer.mozilla.org/en-US/docs/Web/API/Body/arrayBuffer.
+type arrayReader struct {
+	arrayPromise js.Value
+	pending      []byte
+	read         bool
+	err          error // sticky read error
+}
+
+func (r *arrayReader) Read(p []byte) (n int, err error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+	if !r.read {
+		r.read = true
+		var (
+			bCh   = make(chan []byte, 1)
+			errCh = make(chan error, 1)
+		)
+		success := js.NewCallback(func(args []js.Value) {
+			// Wrap the input ArrayBuffer with a Uint8Array
+			uint8arrayWrapper := js.Global.Get("Uint8Array").New(args[0])
+			value := make([]byte, uint8arrayWrapper.Get("byteLength").Int())
+			js.ValueOf(value).Call("set", uint8arrayWrapper)
+			bCh <- value
+		})
+		defer success.Close()
+		failure := js.NewCallback(func(args []js.Value) {
+			// Assumes it's a TypeError. See
+			// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypeError
+			// for more information on this type.
+			// See https://fetch.spec.whatwg.org/#concept-body-consume-body for reasons this might error.
+			errCh <- errors.New(args[0].Get("message").String())
+		})
+		defer failure.Close()
+		r.arrayPromise.Call("then", success, failure)
+		select {
+		case b := <-bCh:
+			r.pending = b
+		case err := <-errCh:
+			return 0, err
+		}
+	}
+	if len(r.pending) == 0 {
+		return 0, io.EOF
+	}
+	n = copy(p, r.pending)
+	r.pending = r.pending[n:]
+	return n, nil
+}
+
+func (r *arrayReader) Close() error {
+	if r.err == nil {
+		r.err = errClosed
+	}
+	return nil
+}

src/net/http/transport.go

@@ -342,11 +342,8 @@ func (tr *transportRequest) setError(err error) {
 	tr.mu.Unlock()
 }
 
-// RoundTrip implements the RoundTripper interface.
-//
-// For higher-level HTTP client support (such as handling of cookies
-// and redirects), see Get, Post, and the Client type.
-func (t *Transport) RoundTrip(req *Request) (*Response, error) {
+// roundTrip implements a RoundTripper over HTTP.
+func (t *Transport) roundTrip(req *Request) (*Response, error) {
 	t.nextProtoOnce.Do(t.onceSetNextProtoDefaults)
 	ctx := req.Context()
 	trace := httptrace.ContextClientTrace(ctx)