MB-30924 improve util.uuid

uuid() (which is used to set requests ids) can be quite slow because of

golang/go#22614

essentially access to /dev/urandom has to be serialized, and when everyone is piling in,
the cost of assigning a request id is (measured) 3.5% of the whole execution time!
This change makes the access to /dev/urandom asynchronous, reading in 32k chunks (one read
every 4k requests), which eliminates the cost completely.

It has been used in anger, at a rate of 50k requests/second, and it has not misbehaved.

Change-Id: I47e524ef11344aadafff770444f214093fe6e008
Reviewed-on: http://review.couchbase.org/98891
Reviewed-by: Sitaram Vemulapalli <[email protected]>
Reviewed-by: Johan Larson <[email protected]>
Tested-by: Marco Greco <[email protected]>

Author: marcogrecopriolo

util/uuid.go

@@ -15,28 +15,136 @@ import (
 	"fmt"
 	"io"
 	"sync"
+
+	atomic "github.com/couchbase/go-couchbase/platform"
 )
 
 var uuidPool *sync.Pool
 
 const _UUID_SIZE = 16
 
+type randomBytesBuffer struct {
+	switchHandler sync.WaitGroup
+	switchWaiter  sync.WaitGroup
+	currentBuffer []byte
+	nextBuffer    []byte
+	currentIndex  uint32
+	err           error
+}
+
+var randomBytes randomBytesBuffer
+
+const _RANDOM_SIZE = 4096 * _UUID_SIZE
+const _FIRE_THRESHOLD = _RANDOM_SIZE / 2
+const _RANDOM_LOCKER = _RANDOM_SIZE + _UUID_SIZE
+
 func init() {
 	uuidPool = &sync.Pool{
 		New: func() interface{} {
 			b := make([]byte, _UUID_SIZE, _UUID_SIZE)
 			return b
 		},
 	}
+	randomBytes.currentBuffer = make([]byte, _RANDOM_SIZE)
+	randomBytes.nextBuffer = make([]byte, _RANDOM_SIZE)
+	randomBytes.switchHandler.Add(1)
+	randomBytes.switchWaiter.Add(1)
+	getNextBuffer()
+	tmp := randomBytes.currentBuffer
+	randomBytes.currentBuffer = randomBytes.nextBuffer
+	randomBytes.nextBuffer = tmp
+}
+
+func readFull(bytes []byte) error {
+	for {
+
+		// copy pointer so that the structure can be changed
+		buffer := randomBytes.currentBuffer
+
+		// get next position
+		index := atomic.AddUint32(&randomBytes.currentIndex, _UUID_SIZE)
+
+		// we are close to needing the next buffer
+		if index == _FIRE_THRESHOLD {
+			randomBytes.switchHandler.Add(1)
+			randomBytes.switchWaiter.Add(1)
+			go getNextBuffer()
+		}
+
+		// we are in luck
+		if index <= _RANDOM_SIZE {
+			copy(bytes, buffer[index-_UUID_SIZE:index])
+			return nil
+		}
+
+		// out of space - slow path
+		// first reader waiting does the dirty work
+		if index == _RANDOM_LOCKER {
+
+			// wait for the asynchronous read
+			randomBytes.switchHandler.Wait()
+
+			// it didn't work
+			if randomBytes.err != nil {
+
+				// wake the waiters (if extra readers come along they will
+				// get an error anyway)
+				randomBytes.switchWaiter.Done()
+
+				// try again: block everyone
+				randomBytes.switchHandler.Add(1)
+				randomBytes.switchWaiter.Add(1)
+
+				// set up another first reader
+				atomic.StoreUint32(&randomBytes.currentIndex, _RANDOM_SIZE)
+
+				go getNextBuffer()
+				return randomBytes.err
+			}
+
+			// switch buffer
+			tmp := randomBytes.currentBuffer
+			randomBytes.currentBuffer = randomBytes.nextBuffer
+			randomBytes.nextBuffer = tmp
+
+			// get our random data
+			copy(bytes, randomBytes.currentBuffer[0:_UUID_SIZE])
+
+			// reset the index and wake up the others
+			atomic.StoreUint32(&randomBytes.currentIndex, _UUID_SIZE)
+			randomBytes.switchWaiter.Done()
+			return nil
+		} else {
+
+			// everyone else takes advantage
+			randomBytes.switchHandler.Wait()
+			if randomBytes.err != nil {
+				return randomBytes.err
+			}
+
+			// try again
+			continue
+		}
+	}
+}
+
+func getNextBuffer() {
+	n, err := io.ReadFull(rand.Reader, randomBytes.nextBuffer)
+	if n != _RANDOM_SIZE {
+		randomBytes.err = fmt.Errorf("random reader only provide %v bytes", n)
+	} else {
+		randomBytes.err = err
+	}
+	randomBytes.switchHandler.Done()
 }
 
 // UUID generates a random UUID according to RFC 4122
 func UUID() (string, error) {
 	uuid := uuidPool.Get().([]byte)
 	defer uuidPool.Put(uuid)
 
-	n, err := io.ReadFull(rand.Reader, uuid)
-	if n != len(uuid) || err != nil {
+	err := readFull(uuid)
+	if err != nil {
 		return "", err
 	}
 	// variant bits; see section 4.1.1