fix issue 84

.circleci/config.yml

@@ -6,7 +6,7 @@ jobs:
     steps:
       - checkout
       - run: go mod download
-      - run: make test
+      - run: make test -j8
 
   benchmark:
     docker:

Makefile

@@ -10,14 +10,25 @@ go-fuzz-build := ${GOPATH}/bin/go-fuzz-build
 go-fuzz-corpus := ${GOPATH}/src/github.com/dvyukov/go-fuzz-corpus
 go-fuzz-dep := ${GOPATH}/src/github.com/dvyukov/go-fuzz/go-fuzz-dep
 
-test:
-	go test -v -cover ./ascii
-	go test -v -cover ./json
-	go test -v -cover ./proto
-	go test -v -cover -tags go1.15 ./json
-	go test -v -cover ./iso8601
-	go run ./json/bugs/issue11/main.go
-	go run ./json/bugs/issue18/main.go
+test: test-ascii test-json test-json-bugs test-json-1.17 test-proto test-iso8601
+
+test-ascii:
+	go test -cover -race ./ascii
+
+test-json:
+	go test -cover -race ./json
+
+test-json-bugs:
+	go test -cover -race ./json/bugs/...
+
+test-json-1.17:
+	go test -cover -race -tags go1.17 ./json
+
+test-proto:
+	go test -cover -race ./proto
+
+test-iso8601:
+	go test -cover -race ./iso8601
 
 $(benchstat):
 	GO111MODULE=off go get -u golang.org/x/perf/cmd/benchstat

ascii/ascii.go

@@ -35,22 +35,18 @@ const (
 	hasMoreConstR32 = hasMoreConstL32 * 128
 )
 
-//go:nosplit
 func hasLess64(x, n uint64) bool {
 	return ((x - (hasLessConstL64 * n)) & ^x & hasLessConstR64) != 0
 }
 
-//go:nosplit
 func hasLess32(x, n uint32) bool {
 	return ((x - (hasLessConstL32 * n)) & ^x & hasLessConstR32) != 0
 }
 
-//go:nosplit
 func hasMore64(x, n uint64) bool {
 	return (((x + (hasMoreConstL64 * (127 - n))) | x) & hasMoreConstR64) != 0
 }
 
-//go:nosplit
 func hasMore32(x, n uint32) bool {
 	return (((x + (hasMoreConstL32 * (127 - n))) | x) & hasMoreConstR32) != 0
 }

ascii/equal_fold.go

@@ -35,32 +35,41 @@ func EqualFoldString(a, b string) bool {
 	n := uintptr(len(a))
 	p := *(*unsafe.Pointer)(unsafe.Pointer(&a))
 	q := *(*unsafe.Pointer)(unsafe.Pointer(&b))
-	// If there is more than 32 bytes to copy, use the AVX optimized version,
-	// otherwise the overhead of the function call tends to be greater than
-	// looping 2 or 3 times over 8 bytes.
-	if n >= 32 && asm.equalFoldAVX2 != nil {
-		if asm.equalFoldAVX2((*byte)(p), (*byte)(q), n) == 0 {
-			return false
+	// Pre-check to avoid the other tests that would all evaluate to false.
+	// For very small strings, this helps reduce the processing overhead.
+	if n >= 8 {
+		// If there is more than 32 bytes to copy, use the AVX optimized version,
+		// otherwise the overhead of the function call tends to be greater than
+		// looping 2 or 3 times over 8 bytes.
+		if n > 32 && asm.equalFoldAVX2 != nil {
+			if asm.equalFoldAVX2((*byte)(p), (*byte)(q), n) == 0 {
+				return false
+			}
+			k := (n / 16) * 16
+			p = unsafe.Pointer(uintptr(p) + k)
+			q = unsafe.Pointer(uintptr(q) + k)
+			n -= k
 		}
-		k := (n / 16) * 16
-		p = unsafe.Pointer(uintptr(p) + k)
-		q = unsafe.Pointer(uintptr(q) + k)
-		n -= k
-	}
 
-	for n >= 8 {
-		const mask = 0xDFDFDFDFDFDFDFDF
+		for n > 8 {
+			const mask = 0xDFDFDFDFDFDFDFDF
 
-		if (*(*uint64)(p) & mask) != (*(*uint64)(q) & mask) {
-			return false
+			if (*(*uint64)(p) & mask) != (*(*uint64)(q) & mask) {
+				return false
+			}
+
+			p = unsafe.Pointer(uintptr(p) + 8)
+			q = unsafe.Pointer(uintptr(q) + 8)
+			n -= 8
 		}
 
-		p = unsafe.Pointer(uintptr(p) + 8)
-		q = unsafe.Pointer(uintptr(q) + 8)
-		n -= 8
+		if n == 8 {
+			const mask = 0xDFDFDFDFDFDFDFDF
+			return (*(*uint64)(p) & mask) == (*(*uint64)(q) & mask)
+		}
 	}
 
-	if n >= 4 {
+	if n > 4 {
 		const mask = 0xDFDFDFDF
 
 		if (*(*uint32)(p) & mask) != (*(*uint32)(q) & mask) {
@@ -73,6 +82,8 @@ func EqualFoldString(a, b string) bool {
 	}
 
 	switch n {
+	case 4:
+		return (*(*uint32)(p) & 0xDFDFDFDF) == (*(*uint32)(q) & 0xDFDFDFDF)
 	case 3:
 		x := uint32(*(*uint16)(p)) | uint32(*(*uint8)(unsafe.Pointer(uintptr(p) + 2)))<<16
 		y := uint32(*(*uint16)(q)) | uint32(*(*uint8)(unsafe.Pointer(uintptr(q) + 2)))<<16

ascii/valid.go

@@ -23,24 +23,30 @@ func ValidString(s string) bool {
 	p := *(*unsafe.Pointer)(unsafe.Pointer(&s))
 	n := uintptr(len(s))
 
-	if n >= 32 && asm.validAVX2 != nil {
-		if asm.validAVX2((*byte)(p), n) == 0 {
-			return false
+	if n >= 8 {
+		if n > 32 && asm.validAVX2 != nil {
+			if asm.validAVX2((*byte)(p), n) == 0 {
+				return false
+			}
+			k := (n / 16) * 16
+			p = unsafe.Pointer(uintptr(p) + k)
+			n -= k
 		}
-		k := (n / 16) * 16
-		p = unsafe.Pointer(uintptr(p) + k)
-		n -= k
-	}
 
-	for n >= 8 {
-		if (*(*uint64)(p) & 0x8080808080808080) != 0 {
-			return false
+		for n > 8 {
+			if (*(*uint64)(p) & 0x8080808080808080) != 0 {
+				return false
+			}
+			p = unsafe.Pointer(uintptr(p) + 8)
+			n -= 8
+		}
+
+		if n == 8 {
+			return (*(*uint64)(p) & 0x8080808080808080) == 0
 		}
-		p = unsafe.Pointer(uintptr(p) + 8)
-		n -= 8
 	}
 
-	if n >= 4 {
+	if n > 4 {
 		if (*(*uint32)(p) & 0x80808080) != 0 {
 			return false
 		}
@@ -50,6 +56,8 @@ func ValidString(s string) bool {
 
 	var x uint32
 	switch n {
+	case 4:
+		x = *(*uint32)(p)
 	case 3:
 		x = uint32(*(*uint16)(p)) | uint32(*(*uint8)(unsafe.Pointer(uintptr(p) + 2)))<<16
 	case 2:

ascii/valid_print.go

@@ -23,24 +23,33 @@ func ValidPrintString(s string) bool {
 	p := *(*unsafe.Pointer)(unsafe.Pointer(&s))
 	n := uintptr(len(s))
 
-	if n >= 32 && asm.validPrintAVX2 != nil {
-		if asm.validPrintAVX2((*byte)(p), n) == 0 {
-			return false
+	if n >= 8 {
+		if n > 32 && asm.validPrintAVX2 != nil {
+			if asm.validPrintAVX2((*byte)(p), n) == 0 {
+				return false
+			}
+			if (n % 16) == 0 {
+				return true
+			}
+			k := (n / 16) * 16
+			p = unsafe.Pointer(uintptr(p) + k)
+			n -= k
 		}
-		k := (n / 16) * 16
-		p = unsafe.Pointer(uintptr(p) + k)
-		n -= k
-	}
 
-	for n >= 8 {
-		if hasLess64(*(*uint64)(p), 0x20) || hasMore64(*(*uint64)(p), 0x7e) {
-			return false
+		for n > 8 {
+			if hasLess64(*(*uint64)(p), 0x20) || hasMore64(*(*uint64)(p), 0x7e) {
+				return false
+			}
+			p = unsafe.Pointer(uintptr(p) + 8)
+			n -= 8
+		}
+
+		if n == 8 {
+			return !(hasLess64(*(*uint64)(p), 0x20) || hasMore64(*(*uint64)(p), 0x7e))
 		}
-		p = unsafe.Pointer(uintptr(p) + 8)
-		n -= 8
 	}
 
-	if n >= 4 {
+	if n > 4 {
 		if hasLess32(*(*uint32)(p), 0x20) || hasMore32(*(*uint32)(p), 0x7e) {
 			return false
 		}
@@ -50,6 +59,8 @@ func ValidPrintString(s string) bool {
 
 	var x uint32
 	switch n {
+	case 4:
+		x = 0x20202020 | *(*uint32)(p)
 	case 3:
 		x = 0x20000000 | uint32(*(*uint16)(p)) | uint32(*(*uint8)(unsafe.Pointer(uintptr(p) + 2)))<<16
 	case 2:

internal/runtime_reflect/map.go

@@ -16,12 +16,12 @@ func Assign(typ, dst, src unsafe.Pointer) {
 	typedmemmove(typ, dst, src)
 }
 
-func MapAssign(t, m, k unsafe.Pointer) uintptr {
-	return uintptr(mapassign(t, m, k))
+func MapAssign(t, m, k unsafe.Pointer) unsafe.Pointer {
+	return mapassign(t, m, k)
 }
 
-func MakeMap(t unsafe.Pointer, cap int) uintptr {
-	return uintptr(makemap(t, cap))
+func MakeMap(t unsafe.Pointer, cap int) unsafe.Pointer {
+	return makemap(t, cap)
 }
 
 type MapIter struct{ hiter }
@@ -45,9 +45,9 @@ func (it *MapIter) HasNext() bool {
 	return it.key != nil
 }
 
-func (it *MapIter) Key() uintptr { return uintptr(it.key) }
+func (it *MapIter) Key() unsafe.Pointer { return it.key }
 
-func (it *MapIter) Value() uintptr { return uintptr(it.value) }
+func (it *MapIter) Value() unsafe.Pointer { return it.value }
 
 // copied from src/runtime/map.go, all pointer types replaced with
 // unsafe.Pointer.

internal/runtime_reflect/slice.go

@@ -20,8 +20,8 @@ func (s *Slice) SetLen(n int) {
 	s.len = n
 }
 
-func (s *Slice) Index(i int, elemSize uintptr) uintptr {
-	return uintptr(s.data) + (uintptr(i) * elemSize)
+func (s *Slice) Index(i int, elemSize uintptr) unsafe.Pointer {
+	return unsafe.Pointer(uintptr(s.data) + (uintptr(i) * elemSize))
 }
 
 func MakeSlice(elemType unsafe.Pointer, len, cap int) Slice {

json/bugs/issue11/main.go

@@ -1,22 +1,3 @@
 package main
 
-import (
-	"fmt"
-	"log"
-
-	"github.com/segmentio/encoding/json"
-)
-
-func main() {
-	m := map[string]map[string]interface{}{
-		"outerkey": {
-			"innerkey": "innervalue",
-		},
-	}
-
-	b, err := json.Marshal(m)
-	if err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println(string(b))
-}
+func main() {}

json/bugs/issue11/main_test.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"testing"
+
+	"github.com/segmentio/encoding/json"
+)
+
+func TestIssue11(t *testing.T) {
+	m := map[string]map[string]interface{}{
+		"outerkey": {
+			"innerkey": "innervalue",
+		},
+	}
+
+	b, err := json.Marshal(m)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println(string(b))
+}

json/bugs/issue18/main.go

@@ -1,22 +1,4 @@
 package main
 
-import (
-	"bytes"
-	"fmt"
-
-	"github.com/segmentio/encoding/json"
-)
-
 func main() {
-	b := []byte(`{
-	"userId": "blah",
-	}`)
-
-	d := json.NewDecoder(bytes.NewReader(b))
-
-	var a struct {
-		UserId string `json:"userId"`
-	}
-	fmt.Println(d.Decode(&a))
-	fmt.Println(a)
 }

json/bugs/issue18/main_test.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/segmentio/encoding/json"
+)
+
+func TestIssue18(t *testing.T) {
+	b := []byte(`{
+	"userId": "blah",
+	}`)
+
+	d := json.NewDecoder(bytes.NewReader(b))
+
+	var a struct {
+		UserId string `json:"userId"`
+	}
+	fmt.Println(d.Decode(&a))
+	fmt.Println(a)
+}

json/bugs/issue84/main.go

@@ -0,0 +1,3 @@
+package main
+
+func main() {}