Factored all c++ source lines parsers

Author: cmaglie

arduino/builder/cpp.go renamed to arduino/builder/cpp/cpp.go

@@ -13,29 +13,63 @@
 // Arduino software without disclosing the source code of your own applications.
 // To purchase a commercial license, send an email to [email protected].
 
-package builder
+package cpp
 
 import (
+	"strconv"
 	"strings"
 	"unicode/utf8"
+
+	"github.com/arduino/go-paths-helper"
 )
 
-// QuoteCppString returns the given string as a quoted string for use with the C
+// QuoteString returns the given string as a quoted string for use with the C
 // preprocessor. This adds double quotes around it and escapes any
 // double quotes and backslashes in the string.
-func QuoteCppString(str string) string {
+func QuoteString(str string) string {
 	str = strings.Replace(str, "\\", "\\\\", -1)
 	str = strings.Replace(str, "\"", "\\\"", -1)
 	return "\"" + str + "\""
 }
 
-// Parse a C-preprocessor string as emitted by the preprocessor. This
+// ParseLineMarker parses the given line as a gcc line marker and returns the contained
+// filename.
+func ParseLineMarker(line string) *paths.Path {
+	// A line marker contains the line number and filename and looks like:
+	// # 123 /path/to/file.cpp
+	// It can be followed by zero or more flag number that indicate the
+	// preprocessor state and can be ignored.
+	// For exact details on this format, see:
+	// https://github.com/gcc-mirror/gcc/blob/edd716b6b1caa1a5cb320a8cd7f626f30198e098/gcc/c-family/c-ppoutput.c#L413-L415
+
+	split := strings.SplitN(line, " ", 3)
+	if len(split) < 3 || len(split[0]) == 0 || split[0][0] != '#' {
+		return nil
+	}
+
+	_, err := strconv.Atoi(split[1])
+	if err != nil {
+		return nil
+	}
+
+	// If we get here, we found a # followed by a line number, so
+	// assume this is a line marker and see if the rest of the line
+	// starts with a string containing the filename
+	str, rest, ok := ParseString(split[2])
+
+	if ok && (rest == "" || rest[0] == ' ') {
+		return paths.New(str)
+	}
+	return nil
+}
+
+// ParseString parse a string as emitted by the preprocessor. This
 // is a string contained in double quotes, with any backslashes or
 // quotes escaped with a backslash. If a valid string was present at the
 // start of the given line, returns the unquoted string contents, the
 // remainder of the line (everything after the closing "), and true.
 // Otherwise, returns the empty string, the entire line and false.
-func ParseCppString(line string) (string, string, bool) {
+func ParseString(line string) (string, string, bool) {
 	// For details about how these strings are output by gcc, see:
 	// https://github.com/gcc-mirror/gcc/blob/a588355ab948cf551bc9d2b89f18e5ae5140f52c/libcpp/macro.c#L491-L511
 	// Note that the documentation suggests all non-printable

arduino/builder/cpp/cpp_test.go

@@ -0,0 +1,74 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2023 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to [email protected].
+
+package cpp_test
+
+import (
+	"testing"
+
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseString(t *testing.T) {
+	_, _, ok := cpp.ParseString(`foo`)
+	require.Equal(t, false, ok)
+
+	_, _, ok = cpp.ParseString(`"foo`)
+	require.Equal(t, false, ok)
+
+	str, rest, ok := cpp.ParseString(`"foo"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"foo\\bar"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo\bar`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"foo \"is\" quoted and \\\\bar\"\" escaped\\" and "then" some`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `foo "is" quoted and \\bar"" escaped\`, str)
+	require.Equal(t, ` and "then" some`, rest)
+
+	str, rest, ok = cpp.ParseString(`" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, ` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"/home/ççç/"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/`, str)
+	require.Equal(t, ``, rest)
+
+	str, rest, ok = cpp.ParseString(`"/home/ççç/ /$sdsdd\\"`)
+	require.Equal(t, true, ok)
+	require.Equal(t, `/home/ççç/ /$sdsdd\`, str)
+	require.Equal(t, ``, rest)
+}
+
+func TestQuoteString(t *testing.T) {
+	cases := map[string]string{
+		`foo`:                                  `"foo"`,
+		`foo\bar`:                              `"foo\\bar"`,
+		`foo "is" quoted and \\bar"" escaped\`: `"foo \"is\" quoted and \\\\bar\"\" escaped\\"`,
+		// ASCII 0x20 - 0x7e, excluding `
+		` !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_abcdefghijklmnopqrstuvwxyz{|}~`: `" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_abcdefghijklmnopqrstuvwxyz{|}~"`,
+	}
+	for input, expected := range cases {
+		require.Equal(t, expected, cpp.QuoteString(input))
+	}
+}

arduino/builder/cpp_test.go

@@ -21,11 +21,11 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/arduino/builder/preprocessor/ctags"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/executils"
 	"github.com/arduino/arduino-cli/i18n"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
 	"github.com/arduino/go-paths-helper"
 	"github.com/arduino/go-properties-orderedmap"
 	"github.com/pkg/errors"
@@ -96,7 +96,7 @@ func composePrototypeSection(line int, prototypes []*ctags.Prototype) string {
 	str := joinPrototypes(prototypes)
 	str += "\n#line "
 	str += strconv.Itoa(line)
-	str += " " + utils.QuoteCppString(prototypes[0].File)
+	str += " " + cpp.QuoteString(prototypes[0].File)
 	str += "\n"
 
 	return str
@@ -108,7 +108,7 @@ func joinPrototypes(prototypes []*ctags.Prototype) string {
 		if signatureContainsaDefaultArg(proto) {
 			continue
 		}
-		prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+utils.QuoteCppString(proto.File))
+		prototypesSlice = append(prototypesSlice, "#line "+strconv.Itoa(proto.Line)+" "+cpp.QuoteString(proto.File))
 		prototypeParts := []string{}
 		if proto.Modifiers != "" {
 			prototypeParts = append(prototypeParts, proto.Modifiers)

arduino/builder/preprocessor/ctags.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"regexp"
 
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/i18n"
 	"github.com/arduino/go-paths-helper"
@@ -96,7 +97,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st
 		lineOffset++
 	}
 
-	mergedSource += "#line 1 " + QuoteCppString(sk.MainFile.String()) + "\n"
+	mergedSource += "#line 1 " + cpp.QuoteString(sk.MainFile.String()) + "\n"
 	mergedSource += mainSrc + "\n"
 	lineOffset++
 
@@ -105,7 +106,7 @@ func sketchMergeSources(sk *sketch.Sketch, overrides map[string]string) (int, st
 		if err != nil {
 			return 0, "", err
 		}
-		mergedSource += "#line 1 " + QuoteCppString(file.String()) + "\n"
+		mergedSource += "#line 1 " + cpp.QuoteString(file.String()) + "\n"
 		mergedSource += src + "\n"
 	}
 
@@ -145,7 +146,7 @@ func sketchCopyAdditionalFiles(sketch *sketch.Sketch, destPath *paths.Path, over
 		}
 
 		// tag each addtional file with the filename of the source it was copied from
-		sourceBytes = append([]byte("#line 1 "+QuoteCppString(file.String())+"\n"), sourceBytes...)
+		sourceBytes = append([]byte("#line 1 "+cpp.QuoteString(file.String())+"\n"), sourceBytes...)
 
 		err = writeIfDifferent(sourceBytes, targetPath)
 		if err != nil {

arduino/builder/sketch.go

@@ -25,7 +25,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/internal/integrationtest"
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
@@ -1191,7 +1191,7 @@ void loop() {
 }
 
 `
-	expected = strings.ReplaceAll(expected, "%SKETCH_PATH%", builder.QuoteCppString(sketchPath.Join("SketchSimple.ino").String()))
+	expected = strings.ReplaceAll(expected, "%SKETCH_PATH%", cpp.QuoteString(sketchPath.Join("SketchSimple.ino").String()))
 
 	jsonOut, _, err := cli.Run("compile", "-b", fqbn, "--preprocess", sketchPath.String(), "--format", "json")
 	require.NoError(t, err)

internal/integrationtest/compile_1/compile_test.go

@@ -23,7 +23,7 @@ import (
 	"strconv"
 	"strings"
 
-	bldr "github.com/arduino/arduino-cli/arduino/builder"
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/arduino/builder/preprocessor"
 	"github.com/arduino/arduino-cli/arduino/sketch"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
@@ -137,7 +137,7 @@ func parseLineMarker(line string) *paths.Path {
 	// If we get here, we found a # followed by a line number, so
 	// assume this is a line marker and see if the rest of the line
 	// starts with a string containing the filename
-	str, rest, ok := bldr.ParseCppString(split[2])
+	str, rest, ok := cpp.ParseString(split[2])
 
 	if ok && (rest == "" || rest[0] == ' ') {
 		return paths.New(str)

legacy/builder/container_add_prototypes.go

@@ -23,19 +23,19 @@ import (
 	"testing"
 	"text/template"
 
+	"github.com/arduino/arduino-cli/arduino/builder/cpp"
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"github.com/arduino/arduino-cli/arduino/libraries"
 	"github.com/arduino/arduino-cli/legacy/builder/constants"
 	"github.com/arduino/arduino-cli/legacy/builder/types"
-	"github.com/arduino/arduino-cli/legacy/builder/utils"
 	paths "github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func LoadAndInterpolate(t *testing.T, filename string, ctx *types.Context) string {
 	funcsMap := template.FuncMap{
-		"QuoteCppString": utils.QuoteCppPath,
+		"QuoteCppString": func(p *paths.Path) string { return cpp.QuoteString(p.String()) },
 	}
 
 	tpl, err := template.New(filepath.Base(filename)).Funcs(funcsMap).ParseFiles(filename)