feat:bufio add DiscardRunes and utf8 add RuneBytes

Author: cafra

src/bufio/bufio.go

@@ -179,6 +179,31 @@ func (b *Reader) Discard(n int) (discarded int, err error) {
 	}
 }
 
+// DiscardRune skips the next n UTF-8 encoded runes, returning the number of bytes discarded.
+//
+// If Discard skips fewer than n runes, it also returns an error.
+// If 0 <= n <= b.Buffered(), Discard is guaranteed to succeed without
+// reading from the underlying io.Reader.
+func (b *Reader) DiscardRunes(n int) (discardedBytes int, err error) {
+	if n < 0 {
+		return 0, ErrNegativeCount
+	}
+	if n == 0 {
+		return
+	}
+	for i := 0; i < n; i++ {
+		for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil && b.w-b.r < len(b.buf) {
+			b.fill() // b.w-b.r < len(buf) => buffer is not full
+		}
+
+		bytes := utf8.RuneBytes(b.buf[b.r:b.w])
+		discardedBytes += bytes
+		b.r += bytes
+	}
+
+	return discardedBytes, nil
+}
+
 // Read reads data into p.
 // It returns the number of bytes read into p.
 // The bytes are taken from at most one Read on the underlying Reader,

src/bufio/bufio_test.go

@@ -1591,3 +1591,81 @@ func BenchmarkWriterFlush(b *testing.B) {
 		bw.Flush()
 	}
 }
+
+func TestReader_DiscardRunes(t *testing.T) {
+	type args struct {
+		runeLen int
+	}
+	tests := []struct {
+		name        string
+		fields      string
+		args        args
+		wantByteLen int
+		wantErr     bool
+	}{
+		{
+			name:        "中",
+			fields:      "中",
+			args:        struct{ runeLen int }{runeLen: 1},
+			wantByteLen: 3,
+			wantErr:     false,
+		},
+		{
+			name:        "a中",
+			fields:      "a中",
+			args:        struct{ runeLen int }{runeLen: 2},
+			wantByteLen: 4,
+			wantErr:     false,
+		},
+		{
+			name:        "a中b",
+			fields:      "a中b",
+			args:        struct{ runeLen int }{runeLen: 3},
+			wantByteLen: 5,
+			wantErr:     false,
+		},
+		{
+			name:        "multi_chunk_4097",
+			fields:      strings.Repeat("中", 4097),
+			args:        struct{ runeLen int }{runeLen: 4097},
+			wantByteLen: 4097 * 3,
+			wantErr:     false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			buf := bytes.NewBufferString(tt.fields)
+			b := NewReader(buf)
+			gotByteLen, err := b.DiscardRunes(tt.args.runeLen)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ByteLenOfRune() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if gotByteLen != tt.wantByteLen {
+				t.Errorf("ByteLenOfRune() gotByteLen = %v, want %v", gotByteLen, tt.wantByteLen)
+			}
+		})
+	}
+}
+
+func BenchmarkDiscardVsRead(b *testing.B) {
+	b.Run("DiscardRunes", func(b *testing.B) {
+		data := strings.Repeat("中", 4097)
+		for i := 0; i < b.N; i++ {
+			buf := bytes.NewBufferString(data)
+			b := NewReader(buf)
+			b.DiscardRunes(4097)
+		}
+	})
+
+	b.Run("readRuneForDiscard", func(b *testing.B) {
+		data := strings.Repeat("中", 4097)
+		for i := 0; i < b.N; i++ {
+			buf := bytes.NewBufferString(data)
+			b := NewReader(buf)
+			for i := 0; i < 4097; i++ {
+				b.ReadRune()
+			}
+		}
+	})
+}

src/unicode/utf8/utf8.go

@@ -190,6 +190,63 @@ func DecodeRune(p []byte) (r rune, size int) {
 	return rune(p0&mask4)<<18 | rune(b1&maskx)<<12 | rune(b2&maskx)<<6 | rune(b3&maskx), 4
 }
 
+// RuneBytes unpacks the first UTF-8 encoding in p and returns its width in bytes.
+// If p is empty it returns 0. Otherwise, if
+// the encoding is invalid, it returns 1. Both are impossible
+// results for correct, non-empty UTF-8.
+//
+// An encoding is invalid if it is incorrect UTF-8, encodes a rune that is
+// out of range, or is not the shortest possible UTF-8 encoding for the
+// value. No other validation is performed.
+func RuneBytes(p []byte) (size int) {
+	if p == nil {
+		return 0
+	}
+	if len(p) == 0 {
+		return 0
+	}
+	r := p[0]
+	if r < RuneSelf {
+		return 1
+	}
+	n := len(p)
+	if n < 1 {
+		return 0
+	}
+	p0 := p[0]
+	x := first[p0]
+	if x >= as {
+		// The following code simulates an additional check for x == xx and
+		// handling the ASCII and invalid cases accordingly. This mask-and-or
+		// approach prevents an additional branch.
+		return 1
+	}
+	sz := int(x & 7)
+	accept := acceptRanges[x>>4]
+	if n < sz {
+		return 1
+	}
+	b1 := p[1]
+	if b1 < accept.lo || accept.hi < b1 {
+		return 1
+	}
+	if sz <= 2 { // <= instead of == to help the compiler eliminate some bounds checks
+		return 2
+	}
+	b2 := p[2]
+	if b2 < locb || hicb < b2 {
+		return 1
+	}
+	if sz <= 3 {
+		return 3
+	}
+	b3 := p[3]
+	if b3 < locb || hicb < b3 {
+		return 1
+	}
+	return 4
+}
+
 // DecodeRuneInString is like DecodeRune but its input is a string. If s is
 // empty it returns (RuneError, 0). Otherwise, if the encoding is invalid, it
 // returns (RuneError, 1). Both are impossible results for correct, non-empty