unicode: use new Scanner interface in table creation

Update norm and local/collate as well.

R=mpvl
CC=golang-dev
https://golang.org/cl/7395045

Author: robpike

src/pkg/exp/locale/collate/maketables.go

@@ -236,18 +236,10 @@ func parseUCA(builder *build.Builder) {
 	}
 	failOnError(err)
 	defer r.Close()
-	input := bufio.NewReader(r)
+	scanner := bufio.NewScanner(r)
 	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
-	for i := 1; err == nil; i++ {
-		l, prefix, e := input.ReadLine()
-		err = e
-		line := string(l)
-		if prefix {
-			log.Fatalf("%d: buffer overflow", i)
-		}
-		if err != nil && err != io.EOF {
-			log.Fatalf("%d: %v", i, err)
-		}
+	for i := 1; scanner.Scan(); i++ {
+		line := scanner.Text()
 		if len(line) == 0 || line[0] == '#' {
 			continue
 		}
@@ -300,6 +292,9 @@ func parseUCA(builder *build.Builder) {
 			failOnError(builder.Add(lhs, rhs, vars))
 		}
 	}
+	if scanner.Err() != nil {
+		log.Fatal(scanner.Err())
+	}
 }
 
 func convHex(line int, s string) int {

src/pkg/exp/locale/collate/regtest.go

@@ -166,16 +166,10 @@ func loadTestData() []Test {
 		ff, err := f.Open()
 		Error(err)
 		defer ff.Close()
-		input := bufio.NewReader(ff)
+		scanner := bufio.NewScanner(ff)
 		test := Test{name: path.Base(f.Name)}
-		for {
-			line, err := input.ReadString('\n')
-			if err != nil {
-				if err == io.EOF {
-					break
-				}
-				log.Fatal(err)
-			}
+		for scanner.Scan() {
+			line := scanner.Text()
 			if len(line) <= 1 || line[0] == '#' {
 				if m := versionRe.FindStringSubmatch(line); m != nil {
 					if m[1] != unicode.Version {
@@ -206,6 +200,9 @@ func loadTestData() []Test {
 				test.comment = append(test.comment, m[2])
 			}
 		}
+		if scanner.Err() != nil {
+			log.Fatal(scanner.Err())
+		}
 		tests = append(tests, test)
 	}
 	return tests

src/pkg/exp/norm/maketables.go

@@ -307,16 +307,12 @@ func parseCharacter(line string) {
 func loadUnicodeData() {
 	f := openReader("UnicodeData.txt")
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
-		parseCharacter(line[0 : len(line)-1])
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		parseCharacter(scanner.Text())
+	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
 	}
 }
 
@@ -347,16 +343,9 @@ func parseExclusion(line string) int {
 func loadCompositionExclusions() {
 	f := openReader("CompositionExclusions.txt")
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
-		point := parseExclusion(line[0 : len(line)-1])
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		point := parseExclusion(scanner.Text())
 		if point == 0 {
 			continue
 		}
@@ -366,6 +355,9 @@ func loadCompositionExclusions() {
 		}
 		c.excludeInComp = true
 	}
+	if scanner.Err() != nil {
+		log.Fatal(scanner.Err())
+	}
 }
 
 // hasCompatDecomp returns true if any of the recursive
@@ -853,15 +845,9 @@ func testDerived() {
 	}
 	f := openReader("DerivedNormalizationProps.txt")
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
 		qc := qcRe.FindStringSubmatch(line)
 		if qc == nil {
 			continue
@@ -920,6 +906,9 @@ func testDerived() {
 			}
 		}
 	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
+	}
 	// Any unspecified value must be QCYes. Verify this.
 	for i, c := range chars {
 		for j, fd := range c.forms {

src/pkg/exp/norm/normregtest.go

@@ -12,7 +12,6 @@ import (
 	"exp/norm"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"os"
@@ -118,8 +117,8 @@ func (t Test) Name() string {
 	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
 }
 
-var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n$`)
-var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n?$`)
+var partRe = regexp.MustCompile(`@Part(\d) # (.*)$`)
+var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)$`)
 
 var counter int
 
@@ -141,15 +140,9 @@ func loadTestData() {
 	}
 	f := resp.Body
 	defer f.Close()
-	input := bufio.NewReader(f)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
 		if len(line) == 0 || line[0] == '#' {
 			continue
 		}
@@ -190,6 +183,9 @@ func loadTestData() {
 		part := &part[len(part)-1]
 		part.tests = append(part.tests, test)
 	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
+	}
 }
 
 var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}

src/pkg/unicode/maketables.go

@@ -13,7 +13,6 @@ import (
 	"bufio"
 	"flag"
 	"fmt"
-	"io"
 	"log"
 	"net/http"
 	"os"
@@ -321,16 +320,11 @@ func loadChars() {
 		flag.Set("data", *url+"UnicodeData.txt")
 	}
 	input := open(*dataURL)
+	defer input.close()
+	scanner := bufio.NewScanner(input)
 	var first rune = 0
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
-		switch parseCategory(line[0 : len(line)-1]) {
+	for scanner.Scan() {
+		switch parseCategory(scanner.Text()) {
 		case SNormal:
 			if first != 0 {
 				logger.Fatalf("bad state normal at %U", lastChar)
@@ -351,23 +345,21 @@ func loadChars() {
 			first = 0
 		}
 	}
-	input.close()
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
+	}
 }
 
 func loadCasefold() {
 	if *casefoldingURL == "" {
 		flag.Set("casefolding", *url+"CaseFolding.txt")
 	}
 	input := open(*casefoldingURL)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
-		if line[0] == '#' || len(strings.TrimSpace(line)) == 0 {
+	defer input.close()
+	scanner := bufio.NewScanner(input)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if len(line) == 0 || line[0] == '#' || len(strings.TrimSpace(line)) == 0 {
 			continue
 		}
 		field := strings.Split(line, "; ")
@@ -389,7 +381,9 @@ func loadCasefold() {
 		}
 		chars[p1].foldCase = rune(p2)
 	}
-	input.close()
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
+	}
 }
 
 const progHeader = `// Generated by running
@@ -710,15 +704,12 @@ func printScriptOrProperty(doProps bool) {
 		return
 	}
 	input := open(*url + file)
-	for {
-		line, err := input.ReadString('\n')
-		if err != nil {
-			if err == io.EOF {
-				break
-			}
-			logger.Fatal(err)
-		}
-		parseScript(line[0:len(line)-1], table)
+	scanner := bufio.NewScanner(input)
+	for scanner.Scan() {
+		parseScript(scanner.Text(), table)
+	}
+	if scanner.Err() != nil {
+		logger.Fatal(scanner.Err())
 	}
 	input.close()