-
Notifications
You must be signed in to change notification settings - Fork 78
/
filterqlvm_test.go
362 lines (324 loc) · 11.9 KB
/
filterqlvm_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
package vm_test
import (
"encoding/json"
"fmt"
"strings"
"testing"
"time"
"github.com/araddon/dateparse"
u "github.com/araddon/gou"
"github.com/stretchr/testify/assert"
"github.com/araddon/qlbridge/datasource"
"github.com/araddon/qlbridge/expr"
"github.com/araddon/qlbridge/rel"
"github.com/araddon/qlbridge/value"
"github.com/araddon/qlbridge/vm"
)
var _ = u.EMPTY
// Our test struct, try as many different field types as possible
type User struct {
Name string
Created time.Time
Updated *time.Time
Authenticated bool
HasSession *bool
Roles []string
BankAmount float64
Address Address
Data json.RawMessage
Context u.JsonHelper
Hits map[string]int64
FirstEvent map[string]time.Time
}
type Address struct {
City string
Zip int
}
func (m *User) FullName() string {
return m.Name + ", Jedi"
}
type includectx struct {
expr.ContextReader
filters map[string]*rel.FilterStatement
}
func newIncluderCtx(cr expr.ContextReader, statements string) *includectx {
stmts := rel.MustParseFilters(statements)
filters := make(map[string]*rel.FilterStatement, len(stmts))
for _, stmt := range stmts {
filters[strings.ToLower(stmt.Alias)] = stmt
}
return &includectx{ContextReader: cr, filters: filters}
}
func (m *includectx) Include(name string) (expr.Node, error) {
if filter, ok := m.filters[strings.ToLower(name)]; ok {
return filter.Filter, nil
}
return nil, expr.ErrNoIncluder
}
func TestFilterQlVm(t *testing.T) {
t.Parallel()
t1 := dateparse.MustParse("12/18/2015")
nminus1 := time.Now().Add(time.Hour * -1)
tr := true
user := &User{
Name: "Yoda",
Created: t1,
Updated: &nminus1,
Authenticated: true,
HasSession: &tr,
Address: Address{"Detroit", 55},
Roles: []string{"admin", "api"},
BankAmount: 55.5,
Hits: map[string]int64{"foo": 5},
FirstEvent: map[string]time.Time{"signedup": t1},
}
readers := []expr.ContextReader{
datasource.NewContextWrapper(user),
datasource.NewContextMap(map[string]interface{}{
"city": "Peoria, IL",
"zip": 5,
"lastevent": map[string]time.Time{"signedup": t1},
"last.event": map[string]time.Time{"has.period": t1},
"transactions": []interface{}{t1.Add(-1 * time.Hour * 24), t1.Add(1 * time.Hour * 24)},
"transactionsnil": []interface{}{},
}, true),
}
nc := datasource.NewNestedContextReader(readers, time.Now())
incctx := newIncluderCtx(nc, `
-- Filter All
FILTER * ALIAS match_all_include;
FILTER name == "Yoda" ALIAS is_yoda_true;
FILTER name == "not gonna happen ALIAS name_false"
`)
hits := []string{
`FILTER name == "Yoda"`, // upper case sensitive name
`FILTER name != "yoda"`, // we should be case-sensitive by default
`FILTER name = "Yoda"`, // is equivalent to ==
`FILTER "Yoda" == name`, // reverse order of identity/value
`FILTER name != "Anakin"`, // negation on missing fields == true
`FILTER first_name != "Anakin"`, // key doesn't exist
`FILTER tolower(name) == "yoda"`, // use functions in evaluation
`FILTER FullName == "Yoda, Jedi"`, // use functions on structs in evaluation
`FILTER Address.City == "Detroit"`, // traverse struct with path.field
`FILTER name LIKE "*da"`, // LIKE
`FILTER name NOT LIKE "*kin"`, // LIKE Negation
`FILTER name CONTAINS "od"`, // Contains
`FILTER name NOT CONTAINS "kin"`, // Contains
`FILTER roles INTERSECTS ("user", "api")`, // Intersects
`FILTER roles IN ("user", "api")`, // #14564 IN is now a synonym of INTERSECTS when used with slices
`FILTER roles NOT INTERSECTS ("user", "guest")`, // Intersects
`FILTER Created BETWEEN "12/01/2015" AND "01/01/2016"`, // Between Operator
`FILTER Created < "now-1d"`, // Date Math
`FILTER NOT ( Created > "now-1d") `, // Date Math (negated)
`FILTER NOT ( FakeDate > "now-1d") `, // Date Math (negated, missing field)
`FILTER Updated > "now-2h"`, // Date Math
`FILTER transactions < "now-1h"`, // Date Compare with []time.Time
`FILTER FirstEvent.signedup < "now-2h"`, // Date Math on map[string]time
`FILTER FirstEvent.signedup == "12/18/2015"`, // Date equality on map[string]time
`FILTER lastevent.signedup < "now-2h"`, // Date Math on map[string]time
`FILTER lastevent.signedup == "12/18/2015"`, // Date equality on map[string]time
"FILTER `lastevent`.`signedup` == \"12/18/2015\"", // escaping of field names using backticks
"FILTER `last.event`.`has.period` == \"12/18/2015\"", // escaping of field names using backticks
`FILTER hits INTERSECTS ("bar", "foo")`,
`FILTER hits IN ("bar", "foo")`, // IN means the same as INTERSECTS with respect to map keys
`FILTER hits NOT IN ("not-gonna-happen")`,
`FILTER lastevent IN ("signedup")`,
`FILTER lastevent NOT IN ("not-gonna-happen")`,
`FILTER *`, // match all
`FILTER OR (
name == "Rey" -- false
INCLUDE match_all_include
)`,
`FILTER OR (
name == "Rey" -- false
INCLUDE is_yoda_true
)`,
`FILTER OR (
EXISTS name, -- inline comments
EXISTS not_a_key, -- more inline comments
)`,
`FILTER EXISTS transactions`, // exists on slice of []time
// show that line-breaks serve as expression separators
`FILTER OR (
EXISTS name
EXISTS not_a_key -- even if they have inline comments
)`,
//`FILTER a == "Yoda" AND b == "Peoria, IL" AND c == 5`,
`FILTER AND (name == "Yoda", city == "Peoria, IL", zip == 5, BankAmount > 50)`,
// Coerce strings to numbers when appropriate
`FILTER AND (zip == "5", BankAmount > "50")`,
`FILTER bankamount > "9.4"`,
`FILTER AND (zip == 5, "Yoda" == name, OR ( city IN ( "Portland, OR", "New York, NY", "Peoria, IL" ) ) )`,
`FILTER OR (
EXISTS q,
AND (
zip > 0,
OR ( zip > 10000, zip < 100 )
),
NOT ( name == "Yoda" ) )`,
`FILTER hits.foo > 1.5`,
`FILTER hits.foo > "1.5"`,
`FILTER NOT ( hits.foo > 5.5 )`,
`FILTER not_a_field NOT IN ("Yoda")`,
}
// hits = []string{
// `FILTER transactions < "now-1h"`, // Date Compare with []time.Time
// }
//u.Debugf("len hits: %v", len(hitsx))
//expr.Trace = true
for _, q := range hits {
fs, err := rel.ParseFilterQL(q)
assert.Equal(t, nil, err)
match, ok := vm.Matches(incctx, fs)
assert.True(t, ok, "should be ok matching on query %q: %v", q, ok)
assert.True(t, match, q)
match, ok = vm.MatchesExpr(incctx, fs.Filter)
assert.True(t, ok, "should be ok matching on query %q: %v", q, ok)
assert.True(t, match, q)
// now resolve includes
err = vm.ResolveIncludes(incctx, fs.Filter)
assert.Equal(t, nil, err)
match, ok = vm.Matches(incctx, fs)
assert.True(t, ok, "should be ok matching on query %q: %v", q, ok)
assert.True(t, match, q)
}
misses := []string{
`FILTER name == "yoda"`, // casing
`FILTER not_a_field + "yoda"`, // invalid statement
"FILTER OR (false, false, AND (true, false))",
`FILTER AND (name == "Yoda", city == "xxx", zip == 5)`,
`FILTER lastevent.signedup > "now-2h"`, // Date Math on map[string]time
`FILTER lastevent.signedup != "12/18/2015"`, // Date equality on map[string]time
`FILTER transactionsnil < "now-1h"`, // Date Compare with empty slice
`FILTER ["hello","apple"] < "now-1h"`, // Date Compare with left hand strings
`FILTER zip * 5 * 2`, // invalid statement
}
for _, q := range misses {
fs, err := rel.ParseFilterQL(q)
assert.Equal(t, nil, err)
match, _ := vm.Matches(incctx, fs)
assert.True(t, !match, q)
match, _ = vm.MatchesExpr(incctx, fs.Filter)
assert.True(t, !match, q)
}
// Filter Select Statements
filterSelects := []fsel{
{`select name, zip FROM mycontext FILTER name == "Yoda"`, map[string]interface{}{"name": "Yoda", "zip": 5}},
{`
SELECT
name
, zip IF zip > 2
FROM mycontext
FILTER name == "Yoda"`, map[string]interface{}{"name": "Yoda", "zip": 5}},
{`
SELECT
name
, zip IF zip > 200
FROM mycontext
FILTER name == "Yoda"`, map[string]interface{}{"name": "Yoda"}},
{`
SELECT
name IF name < true
FROM mycontext
FILTER name == "Yoda"`, nil},
{`
SELECT
name IF zip + 5
FROM mycontext
FILTER name == "Yoda"`, nil},
}
for _, test := range filterSelects {
//u.Debugf("about to parse: %v", test.qlText)
sel, err := rel.ParseFilterSelect(test.query)
assert.True(t, err == nil, "expected no error but got ", err, " for ", test.query)
writeContext := datasource.NewContextSimple()
_, ok := vm.EvalFilterSelect(sel, writeContext, incctx)
assert.True(t, ok, "expected no error but got for %s", test.query)
for key, val := range test.expect {
v := value.NewValue(val)
v2, ok := writeContext.Get(key)
assert.True(t, ok, "Get(%q)=%v but got: %#v", key, val, writeContext.Row())
assert.Equal(t, v2.Value(), v.Value(), "?? %s %v!=%v %T %T", key, v.Value(), v2.Value(), v.Value(), v2.Value())
}
}
}
type fsel struct {
query string
expect map[string]interface{}
}
type includer struct {
expr.EvalContext
}
func matchTest(cr expr.EvalContext, stmt *rel.FilterStatement) (bool, bool) {
return vm.Matches(&includer{cr}, stmt)
}
func (includer) Include(name string) (expr.Node, error) {
if name != "test" {
return nil, fmt.Errorf("Expected name 'test' but received: %s", name)
}
f, err := rel.ParseFilterQL("FILTER AND (x > 5)")
if err != nil {
return nil, err
}
return f.Filter, nil
}
func TestInclude(t *testing.T) {
t.Parallel()
e1 := datasource.NewContextSimpleNative(map[string]interface{}{"x": 6, "y": "1"})
e2 := datasource.NewContextSimpleNative(map[string]interface{}{"x": 4, "y": "1"})
q, err := rel.ParseFilterQL("FILTER AND (x < 9000, INCLUDE test)")
assert.Equal(t, nil, err)
{
match, ok := matchTest(e1, q)
assert.True(t, ok)
assert.True(t, match)
}
{
match, ok := matchTest(e2, q)
assert.True(t, ok)
assert.True(t, !match)
}
// Matches should return an error when the query includes an invalid INCLUDE
{
q, err := rel.ParseFilterQL("FILTER AND (x < 9000, INCLUDE shouldfail)")
assert.Equal(t, nil, err)
_, ok := matchTest(e1, q) // Should fail to evaluate because no includer
assert.True(t, !ok)
}
}
type nilincluder struct{}
func (nilincluder) Include(name string) (expr.Node, error) {
return nil, nil
}
// TestFilterContexts ensures we don't panic if an Includer returns nil. They
// shouldn't, but they do, so we need to be defensive.
func TestFilterContexts(t *testing.T) {
t.Parallel()
readCtx := datasource.NewContextSimpleNative(map[string]interface{}{"x": 6, "key": "abc"})
// Test a non-include context
sel, err := rel.ParseFilterSelect("SELECT x FROM context FILTER exists x")
assert.Equal(t, nil, err)
wc := datasource.NewContextSimple()
_, ok := vm.EvalFilterSelect(sel, wc, readCtx)
assert.True(t, ok, "Should be ok")
// Now invalid statement
sel, err = rel.ParseFilterSelect("SELECT x FROM context FILTER key < true ")
assert.Equal(t, nil, err)
_, ok = vm.EvalFilterSelect(sel, wc, readCtx)
assert.Equal(t, false, ok, "Should not be ok")
// Now invalid statement
sel, err = rel.ParseFilterSelect("SELECT x FROM context FILTER EXISTS not_a_key ")
assert.Equal(t, nil, err)
_, ok = vm.EvalFilterSelect(sel, wc, readCtx)
assert.Equal(t, true, ok, "Should be ok")
q, err := rel.ParseFilterQL("FILTER INCLUDE shouldfail")
assert.Equal(t, nil, err)
ctx := expr.NewIncludeContext(readCtx)
err = vm.ResolveIncludes(ctx, q.Filter)
assert.NotEqual(t, err, nil)
_, ok = vm.Matches(ctx, q)
assert.True(t, !ok, "Should not be ok")
//
_, ok = vm.MatchesInc(ctx, readCtx, q)
assert.True(t, !ok, "Should be ok")
}