diff --git a/ast/node.go b/ast/node.go index ba6545871..1adf95ac9 100644 --- a/ast/node.go +++ b/ast/node.go @@ -151,6 +151,13 @@ type ConditionalNode struct { Exp2 Node } +type VariableDeclaratorNode struct { + base + Name string + Value Node + Expr Node +} + type ArrayNode struct { base Nodes []Node diff --git a/ast/print.go b/ast/print.go index 2e8744730..dd9e0db0f 100644 --- a/ast/print.go +++ b/ast/print.go @@ -130,6 +130,10 @@ func (n *PointerNode) String() string { return "#" } +func (n *VariableDeclaratorNode) String() string { + return fmt.Sprintf("let %s = %s; %s", n.Name, n.Value.String(), n.Expr.String()) +} + func (n *ConditionalNode) String() string { var cond, exp1, exp2 string if _, ok := n.Cond.(*ConditionalNode); ok { diff --git a/ast/visitor.go b/ast/visitor.go index 351e5d72b..287a75589 100644 --- a/ast/visitor.go +++ b/ast/visitor.go @@ -45,6 +45,9 @@ func Walk(node *Node, v Visitor) { case *ClosureNode: Walk(&n.Node, v) case *PointerNode: + case *VariableDeclaratorNode: + Walk(&n.Value, v) + Walk(&n.Expr, v) case *ConditionalNode: Walk(&n.Cond, v) Walk(&n.Exp1, v) diff --git a/builtin/builtin.go b/builtin/builtin.go index f03162b2f..82922e9dd 100644 --- a/builtin/builtin.go +++ b/builtin/builtin.go @@ -12,11 +12,12 @@ import ( ) type Function struct { - Name string - Func func(args ...interface{}) (interface{}, error) - Types []reflect.Type - Builtin1 func(arg interface{}) interface{} - Validate func(args []reflect.Type) (reflect.Type, error) + Name string + Func func(args ...interface{}) (interface{}, error) + Types []reflect.Type + Builtin1 func(arg interface{}) interface{} + Validate func(args []reflect.Type) (reflect.Type, error) + Predicate bool } var ( @@ -34,6 +35,41 @@ func init() { } var Builtins = []*Function{ + { + Name: "all", + Predicate: true, + Types: types(new(func([]interface{}, func(interface{}) bool) bool)), + }, + { + Name: "none", + Predicate: true, + Types: types(new(func([]interface{}, func(interface{}) bool) bool)), + }, + { + Name: "any", + Predicate: true, + Types: types(new(func([]interface{}, func(interface{}) bool) bool)), + }, + { + Name: "one", + Predicate: true, + Types: types(new(func([]interface{}, func(interface{}) bool) bool)), + }, + { + Name: "filter", + Predicate: true, + Types: types(new(func([]interface{}, func(interface{}) bool) []interface{})), + }, + { + Name: "map", + Predicate: true, + Types: types(new(func([]interface{}, func(interface{}) interface{}) []interface{})), + }, + { + Name: "count", + Predicate: true, + Types: types(new(func([]interface{}, func(interface{}) bool) int)), + }, { Name: "len", Builtin1: Len, diff --git a/builtin/builtin_test.go b/builtin/builtin_test.go index ea273cce7..d1e4b88fc 100644 --- a/builtin/builtin_test.go +++ b/builtin/builtin_test.go @@ -113,6 +113,9 @@ func TestBuiltin_works_with_any(t *testing.T) { } for _, b := range builtin.Builtins { + if b.Predicate { + continue + } t.Run(b.Name, func(t *testing.T) { arity := 1 if c, ok := config[b.Name]; ok { @@ -249,12 +252,15 @@ func TestBuiltin_disallow_builtins_override(t *testing.T) { func TestBuiltin_DisableBuiltin(t *testing.T) { t.Run("via env", func(t *testing.T) { - for _, name := range builtin.Names { - t.Run(name, func(t *testing.T) { + for _, b := range builtin.Builtins { + if b.Predicate { + continue // TODO: allow to disable predicates + } + t.Run(b.Name, func(t *testing.T) { env := map[string]interface{}{ - name: func() int { return 42 }, + b.Name: func() int { return 42 }, } - program, err := expr.Compile(name+"()", expr.Env(env), expr.DisableBuiltin(name)) + program, err := expr.Compile(b.Name+"()", expr.Env(env), expr.DisableBuiltin(b.Name)) require.NoError(t, err) out, err := expr.Run(program, env) @@ -264,15 +270,18 @@ func TestBuiltin_DisableBuiltin(t *testing.T) { } }) t.Run("via expr.Function", func(t *testing.T) { - for _, name := range builtin.Names { - t.Run(name, func(t *testing.T) { - fn := expr.Function(name, + for _, b := range builtin.Builtins { + if b.Predicate { + continue // TODO: allow to disable predicates + } + t.Run(b.Name, func(t *testing.T) { + fn := expr.Function(b.Name, func(params ...interface{}) (interface{}, error) { return 42, nil }, new(func() int), ) - program, err := expr.Compile(name+"()", fn, expr.DisableBuiltin(name)) + program, err := expr.Compile(b.Name+"()", fn, expr.DisableBuiltin(b.Name)) require.NoError(t, err) out, err := expr.Run(program, nil) diff --git a/checker/checker.go b/checker/checker.go index fe31f8c92..4a001f4d2 100644 --- a/checker/checker.go +++ b/checker/checker.go @@ -58,10 +58,17 @@ func Check(tree *parser.Tree, config *conf.Config) (t reflect.Type, err error) { type visitor struct { config *conf.Config collections []reflect.Type + scopes []scope parents []ast.Node err *file.Error } +type scope struct { + name string + vtype reflect.Type + info info +} + type info struct { method bool fn *builtin.Function @@ -104,6 +111,8 @@ func (v *visitor) visit(node ast.Node) (reflect.Type, info) { t, i = v.ClosureNode(n) case *ast.PointerNode: t, i = v.PointerNode(n) + case *ast.VariableDeclaratorNode: + t, i = v.VariableDeclaratorNode(n) case *ast.ConditionalNode: t, i = v.ConditionalNode(n) case *ast.ArrayNode: @@ -135,6 +144,9 @@ func (v *visitor) NilNode(*ast.NilNode) (reflect.Type, info) { } func (v *visitor) IdentifierNode(node *ast.IdentifierNode) (reflect.Type, info) { + if s, ok := v.lookupVariable(node.Value); ok { + return s.vtype, s.info + } if node.Value == "$env" { return mapType, info{} } @@ -862,6 +874,35 @@ func (v *visitor) PointerNode(node *ast.PointerNode) (reflect.Type, info) { return v.error(node, "cannot use %v as array", collection) } +func (v *visitor) VariableDeclaratorNode(node *ast.VariableDeclaratorNode) (reflect.Type, info) { + if _, ok := v.config.Types[node.Name]; ok { + return v.error(node, "cannot redeclare %v", node.Name) + } + if _, ok := v.config.Functions[node.Name]; ok { + return v.error(node, "cannot redeclare function %v", node.Name) + } + if _, ok := v.config.Builtins[node.Name]; ok { + return v.error(node, "cannot redeclare builtin %v", node.Name) + } + if _, ok := v.lookupVariable(node.Name); ok { + return v.error(node, "cannot redeclare variable %v", node.Name) + } + vtype, vinfo := v.visit(node.Value) + v.scopes = append(v.scopes, scope{node.Name, vtype, vinfo}) + t, i := v.visit(node.Expr) + v.scopes = v.scopes[:len(v.scopes)-1] + return t, i +} + +func (v *visitor) lookupVariable(name string) (scope, bool) { + for i := len(v.scopes) - 1; i >= 0; i-- { + if v.scopes[i].name == name { + return v.scopes[i], true + } + } + return scope{}, false +} + func (v *visitor) ConditionalNode(node *ast.ConditionalNode) (reflect.Type, info) { c, _ := v.visit(node.Cond) if !isBool(c) && !isAny(c) { diff --git a/checker/checker_test.go b/checker/checker_test.go index 4b4f77c46..e1c7e50b2 100644 --- a/checker/checker_test.go +++ b/checker/checker_test.go @@ -125,6 +125,7 @@ var successTests = []string{ "Any.A?.B == nil", "(Any.Bool ?? Bool) > 0", "Bool ?? Bool", + "let foo = 1; foo == 1", } func TestCheck(t *testing.T) { @@ -506,6 +507,26 @@ repeat("0", 1/0) cannot use float64 as argument (type int) to call repeat (1:14) | repeat("0", 1/0) | .............^ + +let map = 42; map +cannot redeclare builtin map (1:5) + | let map = 42; map + | ....^ + +let len = 42; len +cannot redeclare builtin len (1:5) + | let len = 42; len + | ....^ + +let Float = 42; Float +cannot redeclare Float (1:5) + | let Float = 42; Float + | ....^ + +let foo = 1; let foo = 2; foo +cannot redeclare variable foo (1:18) + | let foo = 1; let foo = 2; foo + | .................^ ` func TestCheck_error(t *testing.T) { @@ -634,7 +655,7 @@ func TestCheck_AllowUndefinedVariables(t *testing.T) { A int } - tree, err := parser.Parse(`any + fn()`) + tree, err := parser.Parse(`Any + fn()`) require.NoError(t, err) config := conf.New(Env{}) @@ -647,7 +668,7 @@ func TestCheck_AllowUndefinedVariables(t *testing.T) { func TestCheck_AllowUndefinedVariables_DefaultType(t *testing.T) { env := map[string]bool{} - tree, err := parser.Parse(`any`) + tree, err := parser.Parse(`Any`) require.NoError(t, err) config := conf.New(env) diff --git a/compiler/compiler.go b/compiler/compiler.go index 89a44039e..b40e12756 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -28,6 +28,7 @@ func Compile(tree *parser.Tree, config *conf.Config) (program *Program, err erro locations: make([]file.Location, 0), constantsIndex: make(map[interface{}]int), functionsIndex: make(map[string]int), + debugInfo: make(map[string]string), } if config != nil { @@ -50,11 +51,12 @@ func Compile(tree *parser.Tree, config *conf.Config) (program *Program, err erro Node: tree.Node, Source: tree.Source, Locations: c.locations, + Variables: c.variables, Constants: c.constants, Bytecode: c.bytecode, Arguments: c.arguments, Functions: c.functions, - FuncNames: c.functionNames, + DebugInfo: c.debugInfo, } return } @@ -62,11 +64,13 @@ func Compile(tree *parser.Tree, config *conf.Config) (program *Program, err erro type compiler struct { locations []file.Location bytecode []Opcode + variables []interface{} + scopes []scope constants []interface{} constantsIndex map[interface{}]int functions []Function - functionNames []string functionsIndex map[string]int + debugInfo map[string]string mapEnv bool cast reflect.Kind nodes []ast.Node @@ -74,6 +78,11 @@ type compiler struct { arguments []int } +type scope struct { + variableName string + index int +} + func (c *compiler) emitLocation(loc file.Location, op Opcode, arg int) int { c.bytecode = append(c.bytecode, op) current := len(c.bytecode) @@ -129,6 +138,13 @@ func (c *compiler) addConstant(constant interface{}) int { return p } +func (c *compiler) addVariable(name string) int { + c.variables = append(c.variables, nil) + p := len(c.variables) - 1 + c.debugInfo[fmt.Sprintf("var_%d", p)] = name + return p +} + // emitFunction adds builtin.Function.Func to the program.Functions and emits call opcode. func (c *compiler) emitFunction(fn *builtin.Function, argsLen int) { switch argsLen { @@ -156,8 +172,8 @@ func (c *compiler) addFunction(fn *builtin.Function) int { } p := len(c.functions) c.functions = append(c.functions, fn.Func) - c.functionNames = append(c.functionNames, fn.Name) c.functionsIndex[fn.Name] = p + c.debugInfo[fmt.Sprintf("func_%d", p)] = fn.Name return p } @@ -209,6 +225,8 @@ func (c *compiler) compile(node ast.Node) { c.ClosureNode(n) case *ast.PointerNode: c.PointerNode(n) + case *ast.VariableDeclaratorNode: + c.VariableDeclaratorNode(n) case *ast.ConditionalNode: c.ConditionalNode(n) case *ast.ArrayNode: @@ -227,6 +245,10 @@ func (c *compiler) NilNode(_ *ast.NilNode) { } func (c *compiler) IdentifierNode(node *ast.IdentifierNode) { + if index, ok := c.lookupVariable(node.Value); ok { + c.emit(OpLoadVar, index) + return + } if node.Value == "$env" { c.emit(OpLoadEnv) return @@ -742,10 +764,36 @@ func (c *compiler) ClosureNode(node *ast.ClosureNode) { c.compile(node.Node) } -func (c *compiler) PointerNode(node *ast.PointerNode) { +func (c *compiler) PointerNode(_ *ast.PointerNode) { c.emit(OpPointer) } +func (c *compiler) VariableDeclaratorNode(node *ast.VariableDeclaratorNode) { + c.compile(node.Value) + index := c.addVariable(node.Name) + c.emit(OpStore, index) + c.beginScope(node.Name, index) + c.compile(node.Expr) + c.endScope() +} + +func (c *compiler) beginScope(name string, index int) { + c.scopes = append(c.scopes, scope{name, index}) +} + +func (c *compiler) endScope() { + c.scopes = c.scopes[:len(c.scopes)-1] +} + +func (c *compiler) lookupVariable(name string) (int, bool) { + for i := len(c.scopes) - 1; i >= 0; i-- { + if c.scopes[i].variableName == name { + return c.scopes[i].index, true + } + } + return 0, false +} + func (c *compiler) ConditionalNode(node *ast.ConditionalNode) { c.compile(node.Cond) otherwise := c.emit(OpJumpIfFalse, placeholder) diff --git a/expr_test.go b/expr_test.go index 3e959bb19..792586278 100644 --- a/expr_test.go +++ b/expr_test.go @@ -948,6 +948,18 @@ func TestExpr(t *testing.T) { `1 /* one */ + 2 // two`, 3, }, + { + `let x = 1; x + 2`, + 3, + }, + { + `map(1..3, let x = #; let y = x * x; y * y)`, + []interface{}{1, 16, 81}, + }, + { + `map(1..2, let x = #; map(2..3, let y = #; x + y))`, + []interface{}{[]interface{}{3, 4}, []interface{}{4, 5}}, + }, } for _, tt := range tests { diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index d69ff7750..1a9418068 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -188,6 +188,17 @@ func TestLex(t *testing.T) { {Kind: EOF}, }, }, + { + `let foo = bar;`, + []Token{ + {Kind: Operator, Value: "let"}, + {Kind: Identifier, Value: "foo"}, + {Kind: Operator, Value: "="}, + {Kind: Identifier, Value: "bar"}, + {Kind: Operator, Value: ";"}, + {Kind: EOF}, + }, + }, } for _, test := range tests { diff --git a/parser/lexer/state.go b/parser/lexer/state.go index c8f0d1725..8e1001b26 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -37,7 +37,7 @@ func root(l *lexer) stateFn { l.emit(Bracket) case strings.ContainsRune(")]}", r): l.emit(Bracket) - case strings.ContainsRune("#,:%+-^", r): // single rune operator + case strings.ContainsRune("#,:;%+-^", r): // single rune operator l.emit(Operator) case strings.ContainsRune("&!=*<>", r): // possible double rune operator l.accept("&=*") @@ -124,6 +124,8 @@ loop: return not case "in", "or", "and", "matches", "contains", "startsWith", "endsWith": l.emit(Operator) + case "let": + l.emit(Operator) default: l.emit(Identifier) } diff --git a/parser/parser.go b/parser/parser.go index 102091748..f2afc2eca 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -110,6 +110,12 @@ func (p *parser) expect(kind Kind, values ...string) { // parse functions func (p *parser) parseExpression(precedence int) Node { + if precedence == 0 { + if p.current.Is(Operator, "let") { + return p.parseVariableDeclaration() + } + } + nodeLeft := p.parsePrimary() prevOperator := "" @@ -179,6 +185,23 @@ func (p *parser) parseExpression(precedence int) Node { return nodeLeft } +func (p *parser) parseVariableDeclaration() Node { + p.expect(Operator, "let") + variableName := p.current + p.expect(Identifier) + p.expect(Operator, "=") + value := p.parseExpression(0) + p.expect(Operator, ";") + node := p.parseExpression(0) + let := &VariableDeclaratorNode{ + Name: variableName.Value, + Value: value, + Expr: node, + } + let.SetLocation(variableName.Location) + return let +} + func (p *parser) parseConditional(node Node) Node { var expr1, expr2 Node for p.current.Is(Operator, "?") && p.err == nil { diff --git a/parser/parser_test.go b/parser/parser_test.go index bbd38476d..002f1f34b 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -452,6 +452,17 @@ func TestParse(t *testing.T) { Callee: &IdentifierNode{Value: "ok"}, Arguments: []Node{ &BoolNode{Value: true}}}}, + { + `let foo = a + b; foo + c`, + &VariableDeclaratorNode{ + Name: "foo", + Value: &BinaryNode{Operator: "+", + Left: &IdentifierNode{Value: "a"}, + Right: &IdentifierNode{Value: "b"}}, + Expr: &BinaryNode{Operator: "+", + Left: &IdentifierNode{Value: "foo"}, + Right: &IdentifierNode{Value: "c"}}}, + }, } for _, test := range parseTests { actual, err := parser.ParseWithConfig(test.input, &conf.Config{Pipes: true}) diff --git a/test/deref/deref_test.go b/test/deref/deref_test.go index 761e779f2..1e1bb83a1 100644 --- a/test/deref/deref_test.go +++ b/test/deref/deref_test.go @@ -12,12 +12,12 @@ func TestDeref_binary(t *testing.T) { i := 1 env := map[string]interface{}{ "i": &i, - "map": map[string]interface{}{ + "obj": map[string]interface{}{ "i": &i, }, } t.Run("==", func(t *testing.T) { - program, err := expr.Compile(`i == 1 && map.i == 1`, expr.Env(env)) + program, err := expr.Compile(`i == 1 && obj.i == 1`, expr.Env(env)) require.NoError(t, err) out, err := expr.Run(program, env) @@ -25,7 +25,7 @@ func TestDeref_binary(t *testing.T) { require.Equal(t, true, out) }) t.Run("><", func(t *testing.T) { - program, err := expr.Compile(`i > 0 && map.i < 99`, expr.Env(env)) + program, err := expr.Compile(`i > 0 && obj.i < 99`, expr.Env(env)) require.NoError(t, err) out, err := expr.Run(program, env) @@ -33,7 +33,7 @@ func TestDeref_binary(t *testing.T) { require.Equal(t, true, out) }) t.Run("??+", func(t *testing.T) { - program, err := expr.Compile(`(i ?? map.i) + 1`, expr.Env(env)) + program, err := expr.Compile(`(i ?? obj.i) + 1`, expr.Env(env)) require.NoError(t, err) out, err := expr.Run(program, env) @@ -47,12 +47,12 @@ func TestDeref_unary(t *testing.T) { ok := true env := map[string]interface{}{ "i": &i, - "map": map[string]interface{}{ + "obj": map[string]interface{}{ "ok": &ok, }, } - program, err := expr.Compile(`-i < 0 && !!map.ok`, expr.Env(env)) + program, err := expr.Compile(`-i < 0 && !!obj.ok`, expr.Env(env)) require.NoError(t, err) out, err := expr.Run(program, env) @@ -64,11 +64,11 @@ func TestDeref_eval(t *testing.T) { i := 1 env := map[string]interface{}{ "i": &i, - "map": map[string]interface{}{ + "obj": map[string]interface{}{ "i": &i, }, } - out, err := expr.Eval(`i == 1 && map.i == 1`, env) + out, err := expr.Eval(`i == 1 && obj.i == 1`, env) require.NoError(t, err) require.Equal(t, true, out) } diff --git a/vm/opcodes.go b/vm/opcodes.go index 1720fd4c2..079d1c76f 100644 --- a/vm/opcodes.go +++ b/vm/opcodes.go @@ -7,6 +7,8 @@ const ( OpPush OpPushInt OpPop + OpStore + OpLoadVar OpLoadConst OpLoadField OpLoadFast diff --git a/vm/program.go b/vm/program.go index ccc7cfb41..1481dbf93 100644 --- a/vm/program.go +++ b/vm/program.go @@ -19,11 +19,12 @@ type Program struct { Node ast.Node Source *file.Source Locations []file.Location + Variables []interface{} Constants []interface{} Bytecode []Opcode Arguments []int Functions []Function - FuncNames []string + DebugInfo map[string]string } func (program *Program) Disassemble() string { @@ -54,6 +55,9 @@ func (program *Program) Opcodes(w io.Writer) { argument := func(label string) { _, _ = fmt.Fprintf(w, "%v\t%v\t<%v>\n", pp, label, arg) } + argumentWithInfo := func(label string, prefix string) { + _, _ = fmt.Fprintf(w, "%v\t%v\t<%v>\t%v\n", pp, label, arg, program.DebugInfo[fmt.Sprintf("%s_%d", prefix, arg)]) + } constant := func(label string) { var c interface{} if arg < len(program.Constants) { @@ -72,12 +76,9 @@ func (program *Program) Opcodes(w io.Writer) { } _, _ = fmt.Fprintf(w, "%v\t%v\t<%v>\t%v\n", pp, label, arg, c) } - builtin := func(label string) { + builtinArg := func(label string) { _, _ = fmt.Fprintf(w, "%v\t%v\t<%v>\t%v\n", pp, label, arg, builtin.Builtins[arg].Name) } - funcName := func(label string) { - _, _ = fmt.Fprintf(w, "%v\t%v\t<%v>\t%v()\n", pp, label, arg, program.FuncNames[arg]) - } switch op { case OpInvalid: @@ -92,6 +93,12 @@ func (program *Program) Opcodes(w io.Writer) { case OpPop: code("OpPop") + case OpStore: + argumentWithInfo("OpStore", "var") + + case OpLoadVar: + argumentWithInfo("OpLoadVar", "var") + case OpLoadConst: constant("OpLoadConst") @@ -222,16 +229,16 @@ func (program *Program) Opcodes(w io.Writer) { argument("OpCall") case OpCall0: - funcName("OpCall0") + argumentWithInfo("OpCall0", "func") case OpCall1: - funcName("OpCall1") + argumentWithInfo("OpCall1", "func") case OpCall2: - funcName("OpCall2") + argumentWithInfo("OpCall2", "func") case OpCall3: - funcName("OpCall3") + argumentWithInfo("OpCall3", "func") case OpCallN: argument("OpCallN") @@ -244,7 +251,7 @@ func (program *Program) Opcodes(w io.Writer) { _, _ = fmt.Fprintf(w, "%v\t%v\t<%v>\t%v\n", pp, "OpCallTyped", arg, signature) case OpCallBuiltin1: - builtin("OpCallBuiltin1") + builtinArg("OpCallBuiltin1") case OpArray: code("OpArray") diff --git a/vm/program_test.go b/vm/program_test.go index c1cdf4f4c..06e83b18c 100644 --- a/vm/program_test.go +++ b/vm/program_test.go @@ -13,7 +13,6 @@ func TestProgram_Disassemble(t *testing.T) { Constants: []interface{}{1, 2}, Bytecode: []vm.Opcode{op}, Arguments: []int{1}, - FuncNames: []string{"foo", "bar"}, } d := program.Disassemble() if strings.Contains(d, "(unknown)") { diff --git a/vm/vm.go b/vm/vm.go index 343a02064..49bee401f 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -102,6 +102,12 @@ func (vm *VM) Run(program *Program, env interface{}) (_ interface{}, err error) case OpPop: vm.pop() + case OpStore: + program.Variables[arg] = vm.pop() + + case OpLoadVar: + vm.push(program.Variables[arg]) + case OpLoadConst: vm.push(runtime.Fetch(env, program.Constants[arg]))