crypto/rand: replace this package with a TinyGo version

This package provides access to an operating system resource
(cryptographic numbers) and so needs to be replaced with a TinyGo
version that does this in a different way.

I've made the following choices while adding this feature:

  - I'm using the getentropy call whenever possible (most POSIX like
    systems), because it is easier to use and more reliable. Linux is
    the exception: it only added getentropy relatively recently.
  - I've left bare-metal implementations to a future patch. This because
    it's hard to reliably get cryptographically secure random numbers on
    embedded devices: most devices do not have a hardware PRNG for this
    purpose.

Author: aykevl

loader/goroot.go

@@ -2,6 +2,14 @@ package loader
 
 // This file constructs a new temporary GOROOT directory by merging both the
 // standard Go GOROOT and the GOROOT from TinyGo using symlinks.
+//
+// The goal is to replace specific packages from Go with a TinyGo version. It's
+// never a partial replacement, either a package is fully replaced or it is not.
+// This is important because if we did allow to merge packages (e.g. by adding
+// files to a package), it would lead to a dependency on implementation details
+// with all the maintenance burden that results in. Only allowing to replace
+// packages as a whole avoids this as packages are already designed to have a
+// public (backwards-compatible) API.
 
 import (
 	"crypto/sha512"
@@ -139,6 +147,7 @@ func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides
 		if err != nil {
 			return err
 		}
+		hasTinyGoFiles := false
 		for _, e := range tinygoEntries {
 			if e.IsDir() {
 				// A directory, so merge this thing.
@@ -154,6 +163,7 @@ func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides
 				if err != nil {
 					return err
 				}
+				hasTinyGoFiles = true
 			}
 		}
 
@@ -164,21 +174,30 @@ func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides
 			return err
 		}
 		for _, e := range gorootEntries {
-			if !e.IsDir() {
-				// Don't merge in files from Go. Otherwise we'd end up with a
-				// weird syscall package with files from both roots.
-				continue
-			}
-			if _, ok := overrides[path.Join(importPath, e.Name())+"/"]; ok {
-				// Already included above, so don't bother trying to create this
-				// symlink.
-				continue
-			}
-			newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
-			oldname := filepath.Join(goroot, "src", importPath, e.Name())
-			err := symlink(oldname, newname)
-			if err != nil {
-				return err
+			if e.IsDir() {
+				if _, ok := overrides[path.Join(importPath, e.Name())+"/"]; ok {
+					// Already included above, so don't bother trying to create this
+					// symlink.
+					continue
+				}
+				newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+				oldname := filepath.Join(goroot, "src", importPath, e.Name())
+				err := symlink(oldname, newname)
+				if err != nil {
+					return err
+				}
+			} else {
+				// Only merge files from Go if TinyGo does not have any files.
+				// Otherwise we'd end up with a weird mix from both Go
+				// implementations.
+				if !hasTinyGoFiles {
+					newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+					oldname := filepath.Join(goroot, "src", importPath, e.Name())
+					err := symlink(oldname, newname)
+					if err != nil {
+						return err
+					}
+				}
 			}
 		}
 	}
@@ -201,6 +220,8 @@ func needsSyscallPackage(buildTags []string) bool {
 func pathsToOverride(needsSyscallPackage bool) map[string]bool {
 	paths := map[string]bool{
 		"/":                     true,
+		"crypto/":               true,
+		"crypto/rand/":          false,
 		"device/":               false,
 		"examples/":             false,
 		"internal/":             true,

src/crypto/rand/rand.go

@@ -0,0 +1,19 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package rand implements a cryptographically secure
+// random number generator.
+package rand
+
+import "io"
+
+// Reader is a global, shared instance of a cryptographically
+// secure random number generator.
+var Reader io.Reader
+
+// Read is a helper function that calls Reader.Read using io.ReadFull.
+// On return, n == len(b) if and only if err == nil.
+func Read(b []byte) (n int, err error) {
+	return io.ReadFull(Reader, b)
+}

src/crypto/rand/rand_getentropy.go

@@ -0,0 +1,38 @@
+// +build darwin freebsd wasi
+
+// This implementation of crypto/rand uses the getentropy system call (available
+// on both MacOS and WASI) to generate random numbers.
+
+package rand
+
+import (
+	"errors"
+	"unsafe"
+)
+
+var errReadFailed = errors.New("rand: could not read random bytes")
+
+func init() {
+	Reader = &reader{}
+}
+
+type reader struct {
+}
+
+func (r *reader) Read(b []byte) (n int, err error) {
+	if len(b) != 0 {
+		if len(b) > 256 {
+			b = b[:256]
+		}
+		result := libc_getentropy(unsafe.Pointer(&b[0]), len(b))
+		if result < 0 {
+			// Maybe we should return a syscall.Errno here?
+			return 0, errReadFailed
+		}
+	}
+	return len(b), nil
+}
+
+// int getentropy(void *buf, size_t buflen);
+//export getentropy
+func libc_getentropy(buf unsafe.Pointer, buflen int) int

src/crypto/rand/rand_urandom.go

@@ -0,0 +1,38 @@
+// +build linux,!baremetal,!wasi
+
+// This implementation of crypto/rand uses the /dev/urandom pseudo-file to
+// generate random numbers.
+// TODO: convert to the getentropy or getrandom libc function on Linux once it
+// is more widely supported.
+
+package rand
+
+import (
+	"syscall"
+)
+
+func init() {
+	Reader = &reader{}
+}
+
+type reader struct {
+	fd int
+}
+
+func (r *reader) Read(b []byte) (n int, err error) {
+	if len(b) == 0 {
+		return
+	}
+
+	// Open /dev/urandom first if needed.
+	if r.fd == 0 {
+		fd, err := syscall.Open("/dev/urandom", syscall.O_RDONLY, 0)
+		if err != nil {
+			return 0, err
+		}
+		r.fd = fd
+	}
+
+	// Read from the file.
+	return syscall.Read(r.fd, b)
+}

src/crypto/rand/util.go

@@ -0,0 +1,143 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package rand
+
+import (
+	"errors"
+	"io"
+	"math/big"
+)
+
+// smallPrimes is a list of small, prime numbers that allows us to rapidly
+// exclude some fraction of composite candidates when searching for a random
+// prime. This list is truncated at the point where smallPrimesProduct exceeds
+// a uint64. It does not include two because we ensure that the candidates are
+// odd by construction.
+var smallPrimes = []uint8{
+	3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53,
+}
+
+// smallPrimesProduct is the product of the values in smallPrimes and allows us
+// to reduce a candidate prime by this number and then determine whether it's
+// coprime to all the elements of smallPrimes without further big.Int
+// operations.
+var smallPrimesProduct = new(big.Int).SetUint64(16294579238595022365)
+
+// Prime returns a number, p, of the given size, such that p is prime
+// with high probability.
+// Prime will return error for any error returned by rand.Read or if bits < 2.
+func Prime(rand io.Reader, bits int) (p *big.Int, err error) {
+	if bits < 2 {
+		err = errors.New("crypto/rand: prime size must be at least 2-bit")
+		return
+	}
+
+	b := uint(bits % 8)
+	if b == 0 {
+		b = 8
+	}
+
+	bytes := make([]byte, (bits+7)/8)
+	p = new(big.Int)
+
+	bigMod := new(big.Int)
+
+	for {
+		_, err = io.ReadFull(rand, bytes)
+		if err != nil {
+			return nil, err
+		}
+
+		// Clear bits in the first byte to make sure the candidate has a size <= bits.
+		bytes[0] &= uint8(int(1<<b) - 1)
+		// Don't let the value be too small, i.e, set the most significant two bits.
+		// Setting the top two bits, rather than just the top bit,
+		// means that when two of these values are multiplied together,
+		// the result isn't ever one bit short.
+		if b >= 2 {
+			bytes[0] |= 3 << (b - 2)
+		} else {
+			// Here b==1, because b cannot be zero.
+			bytes[0] |= 1
+			if len(bytes) > 1 {
+				bytes[1] |= 0x80
+			}
+		}
+		// Make the value odd since an even number this large certainly isn't prime.
+		bytes[len(bytes)-1] |= 1
+
+		p.SetBytes(bytes)
+
+		// Calculate the value mod the product of smallPrimes. If it's
+		// a multiple of any of these primes we add two until it isn't.
+		// The probability of overflowing is minimal and can be ignored
+		// because we still perform Miller-Rabin tests on the result.
+		bigMod.Mod(p, smallPrimesProduct)
+		mod := bigMod.Uint64()
+
+	NextDelta:
+		for delta := uint64(0); delta < 1<<20; delta += 2 {
+			m := mod + delta
+			for _, prime := range smallPrimes {
+				if m%uint64(prime) == 0 && (bits > 6 || m != uint64(prime)) {
+					continue NextDelta
+				}
+			}
+
+			if delta > 0 {
+				bigMod.SetUint64(delta)
+				p.Add(p, bigMod)
+			}
+			break
+		}
+
+		// There is a tiny possibility that, by adding delta, we caused
+		// the number to be one bit too long. Thus we check BitLen
+		// here.
+		if p.ProbablyPrime(20) && p.BitLen() == bits {
+			return
+		}
+	}
+}
+
+// Int returns a uniform random value in [0, max). It panics if max <= 0.
+func Int(rand io.Reader, max *big.Int) (n *big.Int, err error) {
+	if max.Sign() <= 0 {
+		panic("crypto/rand: argument to Int is <= 0")
+	}
+	n = new(big.Int)
+	n.Sub(max, n.SetUint64(1))
+	// bitLen is the maximum bit length needed to encode a value < max.
+	bitLen := n.BitLen()
+	if bitLen == 0 {
+		// the only valid result is 0
+		return
+	}
+	// k is the maximum byte length needed to encode a value < max.
+	k := (bitLen + 7) / 8
+	// b is the number of bits in the most significant byte of max-1.
+	b := uint(bitLen % 8)
+	if b == 0 {
+		b = 8
+	}
+
+	bytes := make([]byte, k)
+
+	for {
+		_, err = io.ReadFull(rand, bytes)
+		if err != nil {
+			return nil, err
+		}
+
+		// Clear bits in the first byte to increase the probability
+		// that the candidate is < max.
+		bytes[0] &= uint8(int(1<<b) - 1)
+
+		n.SetBytes(bytes)
+		if n.Cmp(max) < 0 {
+			return
+		}
+	}
+}

testdata/env.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"crypto/rand"
 	"os"
 )
 
@@ -20,4 +21,26 @@ func main() {
 	for _, arg := range os.Args[1:] {
 		println("arg:", arg)
 	}
+
+	// Check for crypto/rand support.
+	checkRand()
+}
+
+func checkRand() {
+	buf := make([]byte, 500)
+	n, err := rand.Read(buf)
+	if n != len(buf) || err != nil {
+		println("could not read random numbers:", err)
+	}
+
+	// Very simple test that random numbers are at least somewhat random.
+	sum := 0
+	for _, b := range buf {
+		sum += int(b)
+	}
+	if sum < 95*len(buf) || sum > 159*len(buf) {
+		println("random numbers don't seem that random, the average byte is", sum/len(buf))
+	} else {
+		println("random number check was successful")
+	}
 }

testdata/env.txt

@@ -3,3 +3,4 @@ ENV2: VALUE2
 
 arg: first
 arg: second
+random number check was successful