Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions builtin/builtin_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ func TestBuiltin(t *testing.T) {
"NestedAnyArrays": []any{[]any{1, 2}, []any{3, 4}},
"MixedNestedArray": []any{1, []int{2, 3}, []float64{4.0, 5.0}},
"NestedInt32Array": []any{[]int32{1, 2}, []int32{3, 4}},
"MapOfInt": map[string]int{"a": 1, "b": 2, "c": 3},
}

var tests = []struct {
Expand Down Expand Up @@ -197,6 +198,21 @@ func TestBuiltin(t *testing.T) {
{`flatten([["a", "b"], [1, 2, [3, [[[["c", "d"], "e"]]], 4]]])`, []any{"a", "b", 1, 2, 3, "c", "d", "e", 4}},
{`uniq([1, 15, "a", 2, 3, 5, 2, "a", 2, "b"])`, []any{1, 15, "a", 2, 3, 5, "b"}},
{`uniq([[1, 2], "a", 2, 3, [1, 2], [1, 3]])`, []any{[]any{1, 2}, "a", 2, 3, []any{1, 3}}},
{`filter(MapOfInt, # > 1)`, map[string]any{"b": 2, "c": 3}},
{`map(MapOfInt, # * 10)`, map[string]any{"a": 10, "b": 20, "c": 30}},
{`all(MapOfInt, # > 0)`, true},
{`any(MapOfInt, # > 2)`, true},
{`none(MapOfInt, # > 3)`, true},
{`one(MapOfInt, # == 2)`, true},
{`count(MapOfInt, # > 1)`, 2},
{`sum(MapOfInt)`, 6},
{`find(MapOfInt, # > 1)`, 2},
{`findLast(MapOfInt, # > 1)`, 3},
{`reduce(MapOfInt, #acc + #, 0)`, 6},
{`min(MapOfInt)`, 1},
{`max(MapOfInt)`, 3},
{`mean(MapOfInt)`, 2.0},
{`median(MapOfInt)`, 2.0},
}

for _, test := range tests {
Expand Down
45 changes: 45 additions & 0 deletions builtin/lib.go
Original file line number Diff line number Diff line change
Expand Up @@ -233,12 +233,47 @@ func String(arg any) any {
return fmt.Sprintf("%v", arg)
}

// aggregateMapValues unwraps a map argument of an aggregation builtin into a
// slice of its values. It reports whether the argument was a map. Only maps
// with string keys can be aggregated.
func aggregateMapValues(arg any) (any, bool, error) {
if m, ok := arg.(map[string]any); ok {
values := make([]any, 0, len(m))
for _, v := range m {
values = append(values, v)
}
return values, true, nil
}
rv := reflect.ValueOf(arg)
if rv.Kind() != reflect.Map {
return nil, false, nil
}
values := make([]any, 0, rv.Len())
iter := rv.MapRange()
for iter.Next() {
k := iter.Key()
if k.Kind() == reflect.Interface {
k = k.Elem()
}
if k.Kind() != reflect.String {
return nil, true, fmt.Errorf("cannot aggregate over map with non-string keys (%T)", arg)
}
values = append(values, iter.Value().Interface())
}
return values, true, nil
}

func minMax(name string, fn func(any, any) bool, depth int, args ...any) (any, error) {
if depth > MaxDepth {
return nil, ErrorMaxDepth
}
var val any
for _, arg := range args {
if values, ok, err := aggregateMapValues(arg); err != nil {
return nil, err
} else if ok {
arg = values
}
// Fast paths for common typed slices - avoid reflection and allocations
switch arr := arg.(type) {
case []int:
Expand Down Expand Up @@ -353,6 +388,11 @@ func mean(depth int, args ...any) (int, float64, error) {
var count int

for _, arg := range args {
if values, ok, err := aggregateMapValues(arg); err != nil {
return 0, 0, err
} else if ok {
arg = values
}
// Fast paths for common typed slices - avoid reflection and allocations
switch arr := arg.(type) {
case []int:
Expand Down Expand Up @@ -449,6 +489,11 @@ func median(depth int, args ...any) ([]float64, error) {
var values []float64

for _, arg := range args {
if mapValues, ok, err := aggregateMapValues(arg); err != nil {
return nil, err
} else if ok {
arg = mapValues
}
// Fast paths for common typed slices - avoid reflection and allocations
switch arr := arg.(type) {
case []int:
Expand Down
9 changes: 8 additions & 1 deletion builtin/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,16 @@ func validateAggregateFunc(name string, args []reflect.Type) (reflect.Type, erro
return anyType, fmt.Errorf("not enough arguments to call %s", name)
default:
for _, arg := range args {
switch kind(deref.Type(arg)) {
t := deref.Type(arg)
switch kind(t) {
case reflect.Interface, reflect.Array, reflect.Slice:
return anyType, nil
case reflect.Map:
keyKind := t.Key().Kind()
if keyKind != reflect.String && !(keyKind == reflect.Interface && t.Key().NumMethod() == 0) {
return anyType, fmt.Errorf("invalid argument for %s (type %s)", name, arg)
}
return anyType, nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64:
default:
return anyType, fmt.Errorf("invalid argument for %s (type %s)", name, arg)
Expand Down
88 changes: 63 additions & 25 deletions checker/checker.go
Original file line number Diff line number Diff line change
Expand Up @@ -701,16 +701,44 @@ func (v *Checker) callNode(node *ast.CallNode) Nature {
return v.error(node, "%s is not callable", nt.String())
}

// isIterable reports whether a collection nature can be iterated by the
// predicate builtins: an array, or a map with string keys.
func (v *Checker) isIterable(collection Nature) bool {
if collection.IsArray() {
return true
}
if collection.IsMap() {
key := collection.Key(&v.config.NtCache)
if key.Kind == reflect.String {
return true
}
return key.Kind == reflect.Interface && key.Type.NumMethod() == 0
}
return false
}

// indexNature returns the nature of the #index pointer for a collection:
// the key nature for maps, int for arrays, unknown for unknown collections.
func (v *Checker) indexNature(collection Nature) Nature {
if collection.IsUnknown(&v.config.NtCache) {
return Nature{}
}
if collection.IsMap() {
return collection.Key(&v.config.NtCache)
}
return v.config.NtCache.FromType(intType)
}

func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
switch node.Name {
case "all", "none", "any", "one":
collection := v.visit(node.Arguments[0])
collection = collection.Deref(&v.config.NtCache)
if !collection.IsArray() && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
if !v.isIterable(collection) && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array or map with string keys (got %v)", node.Name, collection.String())
}

v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -729,11 +757,11 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
case "filter":
collection := v.visit(node.Arguments[0])
collection = collection.Deref(&v.config.NtCache)
if !collection.IsArray() && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
if !v.isIterable(collection) && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array or map with string keys (got %v)", node.Name, collection.String())
}

v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -748,6 +776,9 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
if collection.IsUnknown(&v.config.NtCache) {
return v.config.NtCache.FromType(arrayType)
}
if collection.IsMap() {
return collection
}
collection = collection.Elem(&v.config.NtCache)
return collection.MakeArrayOf(&v.config.NtCache)
}
Expand All @@ -756,34 +787,41 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
case "map":
collection := v.visit(node.Arguments[0])
collection = collection.Deref(&v.config.NtCache)
if !collection.IsArray() && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
if !v.isIterable(collection) && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array or map with string keys (got %v)", node.Name, collection.String())
}

v.begin(collection, varScope{"index", v.config.NtCache.FromType(intType)})
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

if predicate.IsFunc() &&
predicate.NumOut() == 1 &&
predicate.NumIn() == 1 && predicate.IsFirstArgUnknown(&v.config.NtCache) {

if collection.IsMap() {
elemType := anyType
if predicate.Ref != nil && predicate.Ref.Type != nil {
elemType = predicate.Ref.Type
}
return v.config.NtCache.FromType(reflect.MapOf(collection.Type.Key(), elemType))
}
return predicate.Ref.MakeArrayOf(&v.config.NtCache)
}
return v.error(node.Arguments[1], "predicate should has one input and one output param")

case "count":
collection := v.visit(node.Arguments[0])
collection = collection.Deref(&v.config.NtCache)
if !collection.IsArray() && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
if !v.isIterable(collection) && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array or map with string keys (got %v)", node.Name, collection.String())
}

if len(node.Arguments) == 1 {
return v.config.NtCache.FromType(intType)
}

v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -802,12 +840,12 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
case "sum":
collection := v.visit(node.Arguments[0])
collection = collection.Deref(&v.config.NtCache)
if !collection.IsArray() && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
if !v.isIterable(collection) && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array or map with string keys (got %v)", node.Name, collection.String())
}

if len(node.Arguments) == 2 {
v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -826,11 +864,11 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
case "find", "findLast":
collection := v.visit(node.Arguments[0])
collection = collection.Deref(&v.config.NtCache)
if !collection.IsArray() && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
if !v.isIterable(collection) && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array or map with string keys (got %v)", node.Name, collection.String())
}

v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -856,7 +894,7 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
}

v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -879,7 +917,7 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
}

v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -902,7 +940,7 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
}

v.begin(collection)
v.begin(collection, varScope{"index", v.indexNature(collection)})
predicate := v.visit(node.Arguments[1])
v.end()

Expand All @@ -924,11 +962,11 @@ func (v *Checker) builtinNode(node *ast.BuiltinNode) Nature {
case "reduce":
collection := v.visit(node.Arguments[0])
collection = collection.Deref(&v.config.NtCache)
if !collection.IsArray() && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array (got %v)", node.Name, collection.String())
if !v.isIterable(collection) && !collection.IsUnknown(&v.config.NtCache) {
return v.error(node.Arguments[0], "builtin %v takes only array or map with string keys (got %v)", node.Name, collection.String())
}

v.begin(collection, varScope{"index", v.config.NtCache.FromType(intType)}, varScope{"acc", Nature{}})
v.begin(collection, varScope{"index", v.indexNature(collection)}, varScope{"acc", Nature{}})
predicate := v.visit(node.Arguments[1])
v.end()

Expand Down Expand Up @@ -1241,7 +1279,7 @@ func (v *Checker) pointerNode(node *ast.PointerNode) Nature {
return Nature{}
}
switch scope.collection.Kind {
case reflect.Array, reflect.Slice:
case reflect.Array, reflect.Slice, reflect.Map:
return scope.collection.Elem(&v.config.NtCache)
}
return v.error(node, "cannot use %v as array", scope)
Expand Down
16 changes: 12 additions & 4 deletions checker/checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ cannot use string as argument (type int) to call Variadic (1:13)
`,
},
{`count(1, {#})`, `
builtin count takes only array (got int) (1:7)
builtin count takes only array or map with string keys (got int) (1:7)
| count(1, {#})
| ......^
`,
Expand Down Expand Up @@ -507,7 +507,7 @@ predicate should return boolean (got string) (1:18)
`,
},
{`map(1, {2})`, `
builtin map takes only array (got int) (1:5)
builtin map takes only array or map with string keys (got int) (1:5)
| map(1, {2})
| ....^
`,
Expand Down Expand Up @@ -549,13 +549,13 @@ invalid argument for len (type int) (1:1)
`,
},
{`any(42, {#})`, `
builtin any takes only array (got int) (1:5)
builtin any takes only array or map with string keys (got int) (1:5)
| any(42, {#})
| ....^
`,
},
{`filter(42, {#})`, `
builtin filter takes only array (got int) (1:8)
builtin filter takes only array or map with string keys (got int) (1:8)
| filter(42, {#})
| .......^
`,
Expand Down Expand Up @@ -1146,6 +1146,9 @@ func TestCheck_types(t *testing.T) {
"arr": types.Array(types.Map{
"value": types.String,
}),
"prices": types.Map{
types.Extra: types.Int,
},
types.Extra: types.Any,
}

Expand All @@ -1164,6 +1167,11 @@ func TestCheck_types(t *testing.T) {
{`[foo] | map(.bar) | filter(.baz)`, `predicate should return boolean (got string)`},
{`arr | filter(.value > 0)`, `invalid operation: > (mismatched types string and int)`},
{`arr | filter(.value contains "a") | filter(.value == 0)`, `invalid operation: == (mismatched types string and int)`},
{`prices | filter(# > 0)`, noerr},
{`prices | filter(# > 0) | sum()`, noerr},
{`prices | map(# * 2)`, noerr},
{`prices | filter(#index startsWith "a")`, noerr},
{`prices | filter(# + "x" > 0)`, `invalid operation: + (mismatched types int and string)`},
}

c := new(checker.Checker)
Expand Down
Loading