hdkv
/
jsonpath
1
Fork 0
Code Pull Requests Releases Activity

Initial commit

master
Vladimir Hodakov 2019-02-26 13:05:15 +04:00
commit 0964f1397e
Signed by: Vladimir Hodakov
GPG Key ID: 673980B6882F82C6
26 changed files with 3904 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
LICENSE Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2018 Jumbo Interactive Limited
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

118
README.md Normal file
View File

@ -0,0 +1,118 @@
# The jsonpath library
jsonpath is used to pull values out of a JSON document without unmarshalling the string into an object. At the loss of post-parse random access and conversion to primitive types, you gain faster return speeds and lower memory utilization. If the value you want is located near the start of the json, the evaluator will terminate after reaching and recording its destination.
The evaluator can be initialized with several paths, so you can retrieve multiple sections of the document with just one scan. Naturally, when all paths have been reached, the evaluator will early terminate.
For each value returned by a path, you'll also get the keys & indexes needed to reach that value. Use the `keys` flag to view this in the CLI. The Go package will return an `[]interface{}` of length `n` with indexes `0 - (n-2)` being the keys and the value at index `n-1`.
## The history of a library
This fork is owned and currently maintained by [WTFTeam](https://lab.wtfteam.pro). It's based on [that one](https://github.com/JumboInteractiveLimited/jsonpath) from Jumbo Interactive Limited, which sequentally based/copied from NodePrime/jsonpath Github repository, currently unavailable. The MIT license on this code is inherited from Jumbo Interactive Limited fork, so we can support and maintain this library freely. If there is by any chance someone's proprietary code, you can reach us by ``abuse <at> wtfteam <dot> pro`` with details. Unless proved against all code here is licensed under MIT.
### Go Package
go get lab.wtfteam.pro/wtfteam/jsonpath
```go
paths, err := jsonpath.ParsePaths(pathStrings ...string) {
```
```go
eval, err := jsonpath.EvalPathsInBytes(json []byte, paths)
// OR
eval, err := jsonpath.EvalPathsInReader(r io.Reader, paths)
```
then
```go
for {
if result, ok := eval.Next(); ok {
fmt.Println(result.Pretty(true)) // true -> show keys in pretty string
} else {
break
}
}
if eval.Error != nil {
return eval.Error
}
```
`eval.Next()` will traverse JSON until another value is found. This has the potential of traversing the entire JSON document in an attempt to find one. If you prefer to have more control over traversing, use the `eval.Iterate()` method. It will return after every scanned JSON token and return `([]*Result, bool)`. This array will usually be empty, but occasionally contain results.
### Path Syntax
All paths start from the root node `$`. Similar to getting properties in a JavaScript object, a period `.title` or brackets `["title"]` are used.
Syntax|Meaning|Examples
------|-------|-------
`$`|root of doc|
`.`|property selector |`$.Items`
`["abc"]`|quoted property selector|`$["Items"]`
`*`|wildcard property name|`$.*`
`[n]`|Nth index of array|`[0]` `[1]`
`[n:m]`|Nth index to m-1 index (same as Go slicing)|`[0:1]` `[2:5]`
`[n:]`|Nth index to end of array|`[1:]` `[2:]`
`[*]`|wildcard index of array|`[*]`
`+`|get value at end of path|`$.title+`
`?(expression)`|where clause (expression can reference current json node with @)|`?(@.title == "ABC")`
Expressions
- paths (that start from current node `@`)
- numbers (integers, floats, scientific notation)
- mathematical operators (+ - / * ^)
- numerical comparisos (< <= > >=)
- logic operators (&& || == !=)
- parentheses `(2 < (3 * 5))`
- static values like (`true`, `false`)
- `@.value > 0.5`
Example: this will only return tags of all items that match this expression.
`$.Items[*]?(@.title == "A Tale of Two Cities").tags`
Example:
```javascript
{
"Items":
[
{
"title": "A Midsummer Night's Dream",
"tags":[
"comedy",
"shakespeare",
"play"
]
},{
"title": "A Tale of Two Cities",
"tags":[
"french",
"revolution",
"london"
]
}
]
}
```
Example Paths:
*Paths*
`$.Items[*].title+`
... "A Midsummer Night's Dream"
... "A Tale of Two Cities"
`$.Items[*].tags+`
... ["comedy","shakespeare","play"]
... ["french","revolution","london"]
`$.Items[*].tags[*]+`
... "comedy"
... "shakespeare"
... "play"
... "french"
... "revolution"
... "london"
... = keys/indexes of path

14
constants.go Normal file
View File

@ -0,0 +1,14 @@
package jsonpath
const (
BadStructure = "Bad Structure"
NoMoreResults = "No more results"
UnexpectedToken = "Unexpected token in evaluation"
AbruptTokenStreamEnd = "Token reader is not sending anymore tokens"
)
var (
bytesTrue = []byte{'t', 'r', 'u', 'e'}
bytesFalse = []byte{'f', 'a', 'l', 's', 'e'}
bytesNull = []byte{'n', 'u', 'l', 'l'}
)

335
eval.go Normal file
View File

@ -0,0 +1,335 @@
package jsonpath
import (
"bytes"
"fmt"
)
type queryStateFn func(*query, *Eval, *Item) queryStateFn
type query struct {
Path
state queryStateFn
start int
pos int
firstType int // first json token type in buffer
buffer bytes.Buffer
resultQueue *Results
valLoc stack // capture the current location stack at capture
errors []error
buckets stack // stack of exprBucket
}
type exprBucket struct {
operatorLoc int
expression []Item
queries []*query
results *Results
}
type evalStateFn func(*Eval, *Item) evalStateFn
type Eval struct {
tr tokenReader
levelStack intStack
location stack
queries map[string]*query
state evalStateFn
prevIndex int
nextKey []byte
copyValues bool
resultQueue *Results
Error error
}
func newEvaluation(tr tokenReader, paths ...*Path) *Eval {
e := &Eval{
tr: tr,
location: *newStack(),
levelStack: *newIntStack(),
state: evalRoot,
queries: make(map[string]*query, 0),
prevIndex: -1,
nextKey: nil,
copyValues: true, // depends on which lexer is used
resultQueue: newResults(),
}
for _, p := range paths {
e.queries[p.stringValue] = newQuery(p)
}
// Determine whether to copy emitted item values ([]byte) from lexer
switch tr.(type) {
case *readerLexer:
e.copyValues = true
default:
e.copyValues = false
}
return e
}
func newQuery(p *Path) *query {
return &query{
Path: *p,
state: pathMatchOp,
start: -1,
pos: -1,
buffer: *bytes.NewBuffer(make([]byte, 0, 50)),
valLoc: *newStack(),
errors: make([]error, 0),
resultQueue: newResults(),
buckets: *newStack(),
}
}
func (e *Eval) Iterate() (*Results, bool) {
e.resultQueue.clear()
t, ok := e.tr.next()
if !ok || e.state == nil {
return nil, false
}
// run evaluator function
e.state = e.state(e, t)
anyRunning := false
// run path function for each path
for str, query := range e.queries {
anyRunning = true
query.state = query.state(query, e, t)
if query.state == nil {
delete(e.queries, str)
}
if query.resultQueue.len() > 0 {
e.resultQueue.push(query.resultQueue.Pop())
}
for _, b := range query.buckets.values {
bucket := b.(exprBucket)
for _, dq := range bucket.queries {
dq.state = dq.state(dq, e, t)
if query.resultQueue.len() > 0 {
e.resultQueue.push(query.resultQueue.Pop())
}
}
}
}
if !anyRunning {
return nil, false
}
if e.Error != nil {
return nil, false
}
return e.resultQueue, true
}
func (e *Eval) Next() (*Result, bool) {
if e.resultQueue.len() > 0 {
return e.resultQueue.Pop(), true
}
for {
if _, ok := e.Iterate(); ok {
if e.resultQueue.len() > 0 {
return e.resultQueue.Pop(), true
}
} else {
break
}
}
return nil, false
}
func (q *query) loc() int {
return abs(q.pos-q.start) + q.start
}
func (q *query) trySpillOver() {
if b, ok := q.buckets.peek(); ok {
bucket := b.(exprBucket)
if q.loc() < bucket.operatorLoc {
q.buckets.pop()
exprRes, err := bucket.evaluate()
if err != nil {
q.errors = append(q.errors, err)
}
if exprRes {
next, ok := q.buckets.peek()
var spillover *Results
if !ok {
// fmt.Println("Spilling over into end queue")
spillover = q.resultQueue
} else {
// fmt.Println("Spilling over into lower bucket")
nextBucket := next.(exprBucket)
spillover = nextBucket.results
}
for {
v := bucket.results.Pop()
if v != nil {
spillover.push(v)
} else {
break
}
}
}
}
}
}
func pathMatchOp(q *query, e *Eval, i *Item) queryStateFn {
curLocation := e.location.len() - 1
if q.loc() > curLocation {
q.pos -= 1
q.trySpillOver()
} else if q.loc() <= curLocation {
if q.loc() == curLocation-1 {
if len(q.operators)+q.start >= curLocation {
current, _ := e.location.peek()
nextOp := q.operators[abs(q.loc()-q.start)]
if itemMatchOperator(current, i, nextOp) {
q.pos += 1
if nextOp.whereClauseBytes != nil && len(nextOp.whereClause) > 0 {
bucket := exprBucket{
operatorLoc: q.loc(),
expression: nextOp.whereClause,
queries: make([]*query, len(nextOp.dependentPaths)),
results: newResults(),
}
for i, p := range nextOp.dependentPaths {
bucket.queries[i] = newQuery(p)
bucket.queries[i].pos = q.loc()
bucket.queries[i].start = q.loc()
bucket.queries[i].captureEndValue = true
}
q.buckets.push(bucket)
}
}
}
}
}
if q.loc() == len(q.operators)+q.start && q.loc() <= curLocation {
if q.captureEndValue {
q.firstType = i.typ
q.buffer.Write(i.val)
}
q.valLoc = *e.location.clone()
return pathEndValue
}
if q.loc() < -1 {
return nil
} else {
return pathMatchOp
}
}
func pathEndValue(q *query, e *Eval, i *Item) queryStateFn {
if e.location.len()-1 >= q.loc() {
if q.captureEndValue {
q.buffer.Write(i.val)
}
} else {
r := &Result{Keys: q.valLoc.toArray()}
if q.buffer.Len() > 0 {
val := make([]byte, q.buffer.Len())
copy(val, q.buffer.Bytes())
r.Value = val
switch q.firstType {
case jsonBraceLeft:
r.Type = JsonObject
case jsonString:
r.Type = JsonString
case jsonBracketLeft:
r.Type = JsonArray
case jsonNull:
r.Type = JsonNull
case jsonBool:
r.Type = JsonBool
case jsonNumber:
r.Type = JsonNumber
default:
r.Type = -1
}
}
if q.buckets.len() == 0 {
q.resultQueue.push(r)
} else {
b, _ := q.buckets.peek()
b.(exprBucket).results.push(r)
}
q.valLoc = *newStack()
q.buffer.Truncate(0)
q.pos -= 1
return pathMatchOp
}
return pathEndValue
}
func (b *exprBucket) evaluate() (bool, error) {
values := make(map[string]Item)
for _, q := range b.queries {
result := q.resultQueue.Pop()
if result != nil {
t, err := getJsonTokenType(result.Value)
if err != nil {
return false, err
}
i := Item{
typ: t,
val: result.Value,
}
values[q.Path.stringValue] = i
}
}
res, err := evaluatePostFix(b.expression, values)
if err != nil {
return false, err
}
res_bool, ok := res.(bool)
if !ok {
return false, fmt.Errorf(exprErrorFinalValueNotBool, res)
}
return res_bool, nil
}
func itemMatchOperator(loc interface{}, i *Item, op *operator) bool {
topBytes, isKey := loc.([]byte)
topInt, isIndex := loc.(int)
if isKey {
switch op.typ {
case opTypeNameWild:
return true
case opTypeName, opTypeNameList:
_, found := op.keyStrings[string(topBytes)]
return found
}
} else if isIndex {
switch op.typ {
case opTypeIndexWild:
return true
case opTypeIndex, opTypeIndexRange:
return topInt >= op.indexStart && (!op.hasIndexEnd || topInt <= op.indexEnd)
}
}
return false
}

193
eval_states.go Normal file
View File

@ -0,0 +1,193 @@
package jsonpath
import (
"errors"
"fmt"
)
func evalRoot(e *Eval, i *Item) evalStateFn {
switch i.typ {
case jsonBraceLeft:
e.levelStack.push(i.typ)
return evalObjectAfterOpen
case jsonBracketLeft:
e.levelStack.push(i.typ)
return evalArrayAfterOpen
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func evalObjectAfterOpen(e *Eval, i *Item) evalStateFn {
switch i.typ {
case jsonKey:
c := i.val[1 : len(i.val)-1]
if e.copyValues {
d := make([]byte, len(c))
copy(d, c)
c = d
}
e.nextKey = c
return evalObjectColon
case jsonBraceRight:
return rightBraceOrBracket(e)
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func evalObjectColon(e *Eval, i *Item) evalStateFn {
switch i.typ {
case jsonColon:
return evalObjectValue
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func evalObjectValue(e *Eval, i *Item) evalStateFn {
e.location.push(e.nextKey)
switch i.typ {
case jsonNull, jsonNumber, jsonString, jsonBool:
return evalObjectAfterValue
case jsonBraceLeft:
e.levelStack.push(i.typ)
return evalObjectAfterOpen
case jsonBracketLeft:
e.levelStack.push(i.typ)
return evalArrayAfterOpen
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func evalObjectAfterValue(e *Eval, i *Item) evalStateFn {
e.location.pop()
switch i.typ {
case jsonComma:
return evalObjectAfterOpen
case jsonBraceRight:
return rightBraceOrBracket(e)
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func rightBraceOrBracket(e *Eval) evalStateFn {
e.levelStack.pop()
lowerTyp, ok := e.levelStack.peek()
if !ok {
return evalRootEnd
} else {
switch lowerTyp {
case jsonBraceLeft:
return evalObjectAfterValue
case jsonBracketLeft:
return evalArrayAfterValue
}
}
return nil
}
func evalArrayAfterOpen(e *Eval, i *Item) evalStateFn {
e.prevIndex = -1
switch i.typ {
case jsonNull, jsonNumber, jsonString, jsonBool, jsonBraceLeft, jsonBracketLeft:
return evalArrayValue(e, i)
case jsonBracketRight:
setPrevIndex(e)
return rightBraceOrBracket(e)
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func evalArrayValue(e *Eval, i *Item) evalStateFn {
e.prevIndex++
e.location.push(e.prevIndex)
switch i.typ {
case jsonNull, jsonNumber, jsonString, jsonBool:
return evalArrayAfterValue
case jsonBraceLeft:
e.levelStack.push(i.typ)
return evalObjectAfterOpen
case jsonBracketLeft:
e.levelStack.push(i.typ)
return evalArrayAfterOpen
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func evalArrayAfterValue(e *Eval, i *Item) evalStateFn {
switch i.typ {
case jsonComma:
if val, ok := e.location.pop(); ok {
if valIndex, ok := val.(int); ok {
e.prevIndex = valIndex
}
}
return evalArrayValue
case jsonBracketRight:
e.location.pop()
setPrevIndex(e)
return rightBraceOrBracket(e)
case jsonError:
return evalError(e, i)
default:
e.Error = errors.New(UnexpectedToken)
}
return nil
}
func setPrevIndex(e *Eval) {
e.prevIndex = -1
peeked, ok := e.location.peek()
if ok {
if peekedIndex, intOk := peeked.(int); intOk {
e.prevIndex = peekedIndex
}
}
}
func evalRootEnd(e *Eval, i *Item) evalStateFn {
if i.typ != jsonEOF {
if i.typ == jsonError {
evalError(e, i)
} else {
e.Error = errors.New(BadStructure)
}
}
return nil
}
func evalError(e *Eval, i *Item) evalStateFn {
e.Error = fmt.Errorf("%s at byte index %d", string(i.val), i.pos)
return nil
}

101
eval_test.go Normal file
View File

@ -0,0 +1,101 @@
package jsonpath
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
type test struct {
name string
json string
path string
expected []Result
}
var tests = []test{
test{`key selection`, `{"aKey":32}`, `$.aKey+`, []Result{newResult(`32`, JsonNumber, `aKey`)}},
test{`nested key selection`, `{"aKey":{"bKey":32}}`, `$.aKey+`, []Result{newResult(`{"bKey":32}`, JsonObject, `aKey`)}},
test{`empty array`, `{"aKey":[]}`, `$.aKey+`, []Result{newResult(`[]`, JsonArray, `aKey`)}},
test{`multiple same-level keys, weird spacing`, `{ "aKey" : true , "bKey": [ 1 , 2 ], "cKey" : true } `, `$.bKey+`, []Result{newResult(`[1,2]`, JsonArray, `bKey`)}},
test{`array index selection`, `{"aKey":[123,456]}`, `$.aKey[1]+`, []Result{newResult(`456`, JsonNumber, `aKey`, 1)}},
test{`array wild index selection`, `{"aKey":[123,456]}`, `$.aKey[*]+`, []Result{newResult(`123`, JsonNumber, `aKey`, 0), newResult(`456`, JsonNumber, `aKey`, 1)}},
test{`array range index selection`, `{"aKey":[11,22,33,44]}`, `$.aKey[1:3]+`, []Result{newResult(`22`, JsonNumber, `aKey`, 1), newResult(`33`, JsonNumber, `aKey`, 2)}},
test{`array range (no index) selection`, `{"aKey":[11,22,33,44]}`, `$.aKey[1:1]+`, []Result{}},
test{`array range (no upper bound) selection`, `{"aKey":[11,22,33]}`, `$.aKey[1:]+`, []Result{newResult(`22`, JsonNumber, `aKey`, 1), newResult(`33`, JsonNumber, `aKey`, 2)}},
test{`empty array - try selection`, `{"aKey":[]}`, `$.aKey[1]+`, []Result{}},
test{`null selection`, `{"aKey":[null]}`, `$.aKey[0]+`, []Result{newResult(`null`, JsonNull, `aKey`, 0)}},
test{`empty object`, `{"aKey":{}}`, `$.aKey+`, []Result{newResult(`{}`, JsonObject, `aKey`)}},
test{`object w/ height=2`, `{"aKey":{"bKey":32}}`, `$.aKey.bKey+`, []Result{newResult(`32`, JsonNumber, `aKey`, `bKey`)}},
test{`array of multiple types`, `{"aKey":[1,{"s":true},"asdf"]}`, `$.aKey[1]+`, []Result{newResult(`{"s":true}`, JsonObject, `aKey`, 1)}},
test{`nested array selection`, `{"aKey":{"bKey":[123,456]}}`, `$.aKey.bKey+`, []Result{newResult(`[123,456]`, JsonArray, `aKey`, `bKey`)}},
test{`nested array`, `[[[[[]], [true, false, []]]]]`, `$[0][0][1][2]+`, []Result{newResult(`[]`, JsonArray, 0, 0, 1, 2)}},
test{`index of array selection`, `{"aKey":{"bKey":[123, 456, 789]}}`, `$.aKey.bKey[1]+`, []Result{newResult(`456`, JsonNumber, `aKey`, `bKey`, 1)}},
test{`index of array selection (more than one)`, `{"aKey":{"bKey":[123,456]}}`, `$.aKey.bKey[1]+`, []Result{newResult(`456`, JsonNumber, `aKey`, `bKey`, 1)}},
test{`multi-level object/array`, `{"1Key":{"aKey": null, "bKey":{"trash":[1,2]}, "cKey":[123,456] }, "2Key":false}`, `$.1Key.bKey.trash[0]+`, []Result{newResult(`1`, JsonNumber, `1Key`, `bKey`, `trash`, 0)}},
test{`multi-level array`, `{"aKey":[true,false,null,{"michael":[5,6,7]}, ["s", "3"] ]}`, `$.*[*].michael[1]+`, []Result{newResult(`6`, JsonNumber, `aKey`, 3, `michael`, 1)}},
test{`multi-level array 2`, `{"aKey":[true,false,null,{"michael":[5,6,7]}, ["s", "3"] ]}`, `$.*[*][1]+`, []Result{newResult(`"3"`, JsonString, `aKey`, 4, 1)}},
test{`evaluation literal equality`, `{"items":[ {"name":"alpha", "value":11}]}`, `$.items[*]?("bravo" == "bravo").value+`, []Result{newResult(`11`, JsonNumber, `items`, 0, `value`)}},
test{`evaluation based on string equal to path value`, `{"items":[ {"name":"alpha", "value":11}, {"name":"bravo", "value":22}, {"name":"charlie", "value":33} ]}`, `$.items[*]?(@.name == "bravo").value+`, []Result{newResult(`22`, JsonNumber, `items`, 1, `value`)}},
}
func TestPathQuery(t *testing.T) {
as := assert.New(t)
for _, t := range tests {
paths, err := ParsePaths(t.path)
if as.NoError(err) {
eval, err := EvalPathsInBytes([]byte(t.json), paths)
if as.NoError(err, "Testing: %s", t.name) {
res := toResultArray(eval)
if as.NoError(eval.Error) {
as.EqualValues(t.expected, res, "Testing of %q", t.name)
}
}
eval_reader, err := EvalPathsInReader(strings.NewReader(t.json), paths)
if as.NoError(err, "Testing: %s", t.name) {
res := toResultArray(eval_reader)
if as.NoError(eval.Error) {
as.EqualValues(t.expected, res, "Testing of %q", t.name)
}
}
}
}
}
func newResult(value string, typ int, keys ...interface{}) Result {
keysChanged := make([]interface{}, len(keys))
for i, k := range keys {
switch v := k.(type) {
case string:
keysChanged[i] = []byte(v)
default:
keysChanged[i] = v
}
}
return Result{
Value: []byte(value),
Keys: keysChanged,
Type: typ,
}
}
func toResultArray(e *Eval) []Result {
vals := make([]Result, 0)
for {
if r, ok := e.Next(); ok {
if r != nil {
vals = append(vals, *r)
}
} else {
break
}
}
return vals
}

429
expression.go Normal file
View File

@ -0,0 +1,429 @@
package jsonpath
import (
"errors"
"fmt"
"math"
"reflect"
"strconv"
)
const (
exprErrorMismatchedParens = "Mismatched parentheses"
exprErrorBadExpression = "Bad Expression"
exprErrorFinalValueNotBool = "Expression evaluated to a non-bool: %v"
exprErrorNotEnoughOperands = "Not enough operands for operation %q"
exprErrorValueNotFound = "Value for %q not found"
exprErrorBadValue = "Bad value %q for type %q"
exprErrorPathValueNotScalar = "Path value must be scalar value"
exprErrorBadOperandType = "Operand type expected to be %q for operation %q"
)
type exprErrorBadTypeComparison struct {
valueType string
expectedType string
}
func (e exprErrorBadTypeComparison) Error() string {
return fmt.Sprintf("Type %s cannot be compared to type %s", e.valueType, e.expectedType)
}
// Lowest priority = lowest #
var opa = map[int]struct {
prec int
rAssoc bool
}{
exprOpAnd: {1, false},
exprOpOr: {1, false},
exprOpEq: {2, false},
exprOpNeq: {2, false},
exprOpLt: {3, false},
exprOpLe: {3, false},
exprOpGt: {3, false},
exprOpGe: {3, false},
exprOpPlus: {4, false},
exprOpMinus: {4, false},
exprOpSlash: {5, false},
exprOpStar: {5, false},
exprOpPercent: {5, false},
exprOpHat: {6, false},
exprOpNot: {7, true},
exprOpPlusUn: {7, true},
exprOpMinusUn: {7, true},
}
// Shunting-yard Algorithm (infix -> postfix)
// http://rosettacode.org/wiki/Parsing/Shunting-yard_algorithm#Go
func infixToPostFix(items []Item) (out []Item, err error) {
stack := newStack()
for _, i := range items {
switch i.typ {
case exprParenLeft:
stack.push(i) // push "(" to stack
case exprParenRight:
found := false
for {
// pop item ("(" or operator) from stack
op_interface, ok := stack.pop()
if !ok {
return nil, errors.New(exprErrorMismatchedParens)
}
op := op_interface.(Item)
if op.typ == exprParenLeft {
found = true
break // discard "("
}
out = append(out, op) // add operator to result
}
if !found {
return nil, errors.New(exprErrorMismatchedParens)
}
default:
if o1, isOp := opa[i.typ]; isOp {
// token is an operator
for stack.len() > 0 {
// consider top item on stack
op_int, _ := stack.peek()
op := op_int.(Item)
if o2, isOp := opa[op.typ]; !isOp || o1.prec > o2.prec ||
o1.prec == o2.prec && o1.rAssoc {
break
}
// top item is an operator that needs to come off
stack.pop() // pop it
out = append(out, op) // add it to result
}
// push operator (the new one) to stack
stack.push(i)
} else { // token is an operand
out = append(out, i) // add operand to result
}
}
}
// drain stack to result
for stack.len() > 0 {
op_int, _ := stack.pop()
op := op_int.(Item)
if op.typ == exprParenLeft {
return nil, errors.New(exprErrorMismatchedParens)
}
out = append(out, op)
}
return
}
func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface{}, error) {
s := newStack()
if len(postFixItems) == 0 {
return false, errors.New(exprErrorBadExpression)
}
for _, item := range postFixItems {
switch item.typ {
// VALUES
case exprBool:
val, err := strconv.ParseBool(string(item.val))
if err != nil {
return false, fmt.Errorf(exprErrorBadValue, string(item.val), exprTokenNames[exprBool])
}
s.push(val)
case exprNumber:
val, err := strconv.ParseFloat(string(item.val), 64)
if err != nil {
return false, fmt.Errorf(exprErrorBadValue, string(item.val), exprTokenNames[exprNumber])
}
s.push(val)
case exprPath:
// TODO: Handle datatypes of JSON
i, ok := pathValues[string(item.val)]
if !ok {
return false, fmt.Errorf(exprErrorValueNotFound, string(item.val))
}
switch i.typ {
case jsonNull:
s.push(nil)
case jsonNumber:
val_float, err := strconv.ParseFloat(string(i.val), 64)
if err != nil {
return false, fmt.Errorf(exprErrorBadValue, string(item.val), jsonTokenNames[jsonNumber])
}
s.push(val_float)
case jsonKey, jsonString:
s.push(i.val)
default:
return false, fmt.Errorf(exprErrorPathValueNotScalar)
}
case exprString:
s.push(item.val)
case exprNull:
s.push(nil)
// OPERATORS
case exprOpAnd:
a, b, err := take2Bool(s, item.typ)
if err != nil {
return false, err
}
s.push(a && b)
case exprOpEq:
p, ok := s.peek()
if !ok {
return false, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[item.typ])
}
switch p.(type) {
case nil:
err := take2Null(s, item.typ)
if err != nil {
return false, err
} else {
s.push(true)
}
case bool:
a, b, err := take2Bool(s, item.typ)
if err != nil {
return false, err
}
s.push(a == b)
case float64:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(a == b)
case []byte:
a, b, err := take2ByteSlice(s, item.typ)
if err != nil {
return false, err
}
s.push(byteSlicesEqual(a, b))
}
case exprOpNeq:
p, ok := s.peek()
if !ok {
return false, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[item.typ])
}
switch p.(type) {
case nil:
err := take2Null(s, item.typ)
if err != nil {
return true, err
} else {
s.push(false)
}
case bool:
a, b, err := take2Bool(s, item.typ)
if err != nil {
return false, err
}
s.push(a != b)
case float64:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(a != b)
case []byte:
a, b, err := take2ByteSlice(s, item.typ)
if err != nil {
return false, err
}
s.push(!byteSlicesEqual(a, b))
}
case exprOpNot:
a, err := take1Bool(s, item.typ)
if err != nil {
return false, err
}
s.push(!a)
case exprOpOr:
a, b, err := take2Bool(s, item.typ)
if err != nil {
return false, err
}
s.push(a || b)
case exprOpGt:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(b > a)
case exprOpGe:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(b >= a)
case exprOpLt:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(b < a)
case exprOpLe:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(b <= a)
case exprOpPlus:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(b + a)
case exprOpPlusUn:
a, err := take1Float(s, item.typ)
if err != nil {
return false, err
}
s.push(a)
case exprOpMinus:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(b - a)
case exprOpMinusUn:
a, err := take1Float(s, item.typ)
if err != nil {
return false, err
}
s.push(0 - a)
case exprOpSlash:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
if a == 0.0 {
return false, errors.New("Cannot divide by zero")
}
s.push(b / a)
case exprOpStar:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(b * a)
case exprOpPercent:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(math.Mod(b, a))
case exprOpHat:
a, b, err := take2Float(s, item.typ)
if err != nil {
return false, err
}
s.push(math.Pow(b, a))
case exprOpExclam:
a, err := take1Bool(s, item.typ)
if err != nil {
return false, err
}
s.push(!a)
// Other
default:
return false, fmt.Errorf("Token not supported in evaluator: %v", exprTokenNames[item.typ])
}
}
if s.len() != 1 {
return false, fmt.Errorf(exprErrorBadExpression)
}
end_int, _ := s.pop()
return end_int, nil
}
func take1Bool(s *stack, op int) (bool, error) {
t := exprBool
val, ok := s.pop()
if !ok {
return false, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
}
b, ok := val.(bool)
if !ok {
return false, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()}
}
return b, nil
}
func take2Bool(s *stack, op int) (bool, bool, error) {
a, a_err := take1Bool(s, op)
b, b_err := take1Bool(s, op)
return a, b, firstError(a_err, b_err)
}
func take1Float(s *stack, op int) (float64, error) {
t := exprNumber
val, ok := s.pop()
if !ok {
return 0.0, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
}
b, ok := val.(float64)
if !ok {
return 0.0, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()}
}
return b, nil
}
func take2Float(s *stack, op int) (float64, float64, error) {
a, a_err := take1Float(s, op)
b, b_err := take1Float(s, op)
return a, b, firstError(a_err, b_err)
}
func take1ByteSlice(s *stack, op int) ([]byte, error) {
t := exprNumber
val, ok := s.pop()
if !ok {
return nil, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
}
b, ok := val.([]byte)
if !ok {
return nil, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()}
}
return b, nil
}
func take2ByteSlice(s *stack, op int) ([]byte, []byte, error) {
a, a_err := take1ByteSlice(s, op)
b, b_err := take1ByteSlice(s, op)
return a, b, firstError(a_err, b_err)
}
func take1Null(s *stack, op int) error {
t := exprNull
val, ok := s.pop()
if !ok {
return fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
}
if v := reflect.TypeOf(val); v != nil {
return exprErrorBadTypeComparison{exprTokenNames[t], v.String()}
}
return nil
}
func take2Null(s *stack, op int) error {
a_err := take1Null(s, op)
b_err := take1Null(s, op)
return firstError(a_err, b_err)
}

287
expression_states.go Normal file
View File

@ -0,0 +1,287 @@
package jsonpath
const (
exprError = iota
exprEOF
exprParenLeft
exprParenRight
exprNumber
exprPath
exprBool
exprNull
exprString
exprOperators
exprOpEq
exprOpNeq
exprOpNot
exprOpLt
exprOpLe
exprOpGt
exprOpGe
exprOpAnd
exprOpOr
exprOpPlus
exprOpPlusUn
exprOpMinus
exprOpMinusUn
exprOpSlash
exprOpStar
exprOpHat
exprOpPercent
exprOpExclam
)
var exprTokenNames = map[int]string{
exprError: "error",
exprEOF: "EOF",
exprParenLeft: "(",
exprParenRight: ")",
exprNumber: "number",
exprPath: "path",
exprBool: "bool",
exprNull: "null",
exprString: "string",
exprOpEq: "==",
exprOpNeq: "!=",
exprOpNot: "!",
exprOpLt: "<",
exprOpLe: "<=",
exprOpGt: ">",
exprOpGe: ">=",
exprOpAnd: "&&",
exprOpOr: "||",
exprOpPlus: "+",
exprOpPlusUn: "(+)",
exprOpMinus: "-",
exprOpMinusUn: "(-)",
exprOpSlash: "/",
exprOpStar: "*",
exprOpHat: "^",
exprOpPercent: "%",
exprOpExclam: "!",
}
var EXPRESSION = lexExprText
func lexExprText(l lexer, state *intStack) stateFn {
ignoreSpaceRun(l)
cur := l.peek()
var next stateFn
switch cur {
case '(':
l.take()
state.push(exprParenLeft)
l.emit(exprParenLeft)
next = lexExprText
case ')':
if top, ok := state.peek(); ok && top != exprParenLeft {
next = l.errorf("Received %#U but has no matching (", cur)
break
}
state.pop()
l.take()
l.emit(exprParenRight)
next = lexOneValue
case '!':
l.take()
l.emit(exprOpNot)
next = lexExprText
case '+':
l.take()
l.emit(exprOpPlusUn)
next = lexExprText
case '-':
l.take()
l.emit(exprOpMinusUn)
next = lexExprText
case '@': //, '$': // Only support current key location
l.take()
takePath(l)
l.emit(exprPath)
next = lexOneValue
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeNumeric(l)
l.emit(exprNumber)
next = lexOneValue
case 't':
takeExactSequence(l, bytesTrue)
l.emit(exprBool)
next = lexOneValue
case 'f':
takeExactSequence(l, bytesFalse)
l.emit(exprBool)
next = lexOneValue
case 'n':
takeExactSequence(l, bytesNull)
l.emit(exprNull)
next = lexOneValue
case '"':
err := l.takeString()
if err != nil {
return l.errorf("Could not take string because %q", err)
}
l.emit(exprString)
next = lexOneValue
case eof:
l.emit(exprEOF)
// next = nil
default:
return l.errorf("Unrecognized sequence in expression: %#U", cur)
}
return next
}
func lexOneValue(l lexer, state *intStack) stateFn {
var next stateFn
cur := l.peek()
switch cur {
case '+':
l.take()
l.emit(exprOpPlus)
next = lexExprText
case '-':
l.take()
l.emit(exprOpMinus)
next = lexExprText
case '*':
l.take()
l.emit(exprOpStar)
next = lexExprText
case '/':
l.take()
l.emit(exprOpSlash)
next = lexExprText
case '%':
l.take()
l.emit(exprOpPercent)
next = lexExprText
case '^':
l.take()
l.emit(exprOpHat)
next = lexExprText
case '<':
l.take()
cur = l.peek()
if cur == '=' {
l.take()
l.emit(exprOpLe)
} else {
l.emit(exprOpLt)
}
next = lexExprText
case '>':
l.take()
cur = l.peek()
if cur == '=' {
l.take()
l.emit(exprOpGe)
} else {
l.emit(exprOpGt)
}
next = lexExprText
case '&':
l.take()
cur = l.take()
if cur != '&' {
return l.errorf("Expected double & instead of %#U", cur)
}
l.emit(exprOpAnd)
next = lexExprText
case '|':
l.take()
cur = l.take()
if cur != '|' {
return l.errorf("Expected double | instead of %#U", cur)
}
l.emit(exprOpOr)
next = lexExprText
case '=':
l.take()
cur = l.take()
if cur != '=' {
return l.errorf("Expected double = instead of %#U", cur)
}
l.emit(exprOpEq)
next = lexExprText
case '!':
l.take()
cur = l.take()
if cur != '=' {
return l.errorf("Expected = for != instead of %#U", cur)
}
l.emit(exprOpNeq)
next = lexExprText
case ')':
if top, ok := state.peek(); ok && top != exprParenLeft {
next = l.errorf("Received %#U but has no matching (", cur)
break
}
state.pop()
l.take()
l.emit(exprParenRight)
next = lexOneValue
case eof:
l.emit(exprEOF)
default:
return l.errorf("Unrecognized sequence in expression: %#U", cur)
}
return next
}
func takeNumeric(l lexer) {
takeDigits(l)
if l.peek() == '.' {
l.take()
takeDigits(l)
}
if l.peek() == 'e' || l.peek() == 'E' {
l.take()
if l.peek() == '+' || l.peek() == '-' {
l.take()
takeDigits(l)
} else {
takeDigits(l)
}
}
}
func takePath(l lexer) {
inQuotes := false
var prev int = 0
// capture until end of path - ugly
takeLoop:
for {
cur := l.peek()
switch cur {
case '"':
if prev != '\\' {
inQuotes = !inQuotes
}
l.take()
case ' ':
if !inQuotes {
break takeLoop
}
l.take()
case eof:
break takeLoop
default:
l.take()
}
prev = cur
}
}
func lexExprEnd(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != eof {
return l.errorf("Expected EOF but received %#U", cur)
}
l.emit(exprEOF)
return nil
}

40
expression_states_test.go Normal file
View File

@ -0,0 +1,40 @@
package jsonpath
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
var expressionTests = []lexTest{
{"empty", "", []int{exprEOF}},
{"spaces", " \t\r\n", []int{exprEOF}},
{"numbers", " 1.3e10 ", []int{exprNumber, exprEOF}},
// {"numbers with signs", "+1 -2.23", []int{exprNumber, exprOpPlus, exprNumber, exprEOF}},
{"paths", " @.aKey[2].bKey ", []int{exprPath, exprEOF}},
{"addition with mixed sign", "4+-19", []int{exprNumber, exprOpPlus, exprOpMinusUn, exprNumber, exprEOF}},
{"addition", "4+19", []int{exprNumber, exprOpPlus, exprNumber, exprEOF}},
{"subtraction", "4-19", []int{exprNumber, exprOpMinus, exprNumber, exprEOF}},
{"parens", "( () + () )", []int{exprParenLeft, exprParenLeft, exprParenRight, exprOpPlus, exprParenLeft, exprParenRight, exprParenRight, exprEOF}},
{"equals", "true ==", []int{exprBool, exprOpEq, exprEOF}},
{"numerical comparisons", "3.4 <", []int{exprNumber, exprOpLt, exprEOF}},
}
func TestExpressionTokens(t *testing.T) {
as := assert.New(t)
for _, test := range expressionTests {
lexer := NewSliceLexer([]byte(test.input), EXPRESSION)
items := readerToArray(lexer)
types := itemsToTypes(items)
for _, i := range items {
if i.typ == exprError {
fmt.Println(string(i.val))
}
}
as.EqualValues(types, test.tokenTypes, "Testing of %s: \nactual\n\t%+v\nexpected\n\t%v", test.name, typesDescription(types, exprTokenNames), typesDescription(test.tokenTypes, exprTokenNames))
}
}

226
expression_test.go Normal file
View File

@ -0,0 +1,226 @@
package jsonpath
import (
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
var exprTests = []struct {
input string
fields map[string]Item
expectedValue interface{}
}{
// &&
{"true && true", nil, true},
{"false && true", nil, false},
{"false && false", nil, false},
// ||
{"true || true", nil, true},
{"true || false", nil, true},
{"false || false", nil, false},
// LT
{"10 < 20", nil, true},
{"10 < 10", nil, false},
{"100 < 20", nil, false},
{"@a < 50", map[string]Item{"@a": genValue(`49`, jsonNumber)}, true},
{"@a < 50", map[string]Item{"@a": genValue(`50`, jsonNumber)}, false},
{"@a < 50", map[string]Item{"@a": genValue(`51`, jsonNumber)}, false},
// LE
{"10 <= 20", nil, true},
{"10 <= 10", nil, true},
{"100 <= 20", nil, false},
{"@a <= 54", map[string]Item{"@a": genValue(`53`, jsonNumber)}, true},
{"@a <= 54", map[string]Item{"@a": genValue(`54`, jsonNumber)}, true},
{"@a <= 54", map[string]Item{"@a": genValue(`55`, jsonNumber)}, false},
// GT
{"30 > 20", nil, true},
{"20 > 20", nil, false},
{"10 > 20", nil, false},
{"@a > 50", map[string]Item{"@a": genValue(`49`, jsonNumber)}, false},
{"@a > 50", map[string]Item{"@a": genValue(`50`, jsonNumber)}, false},
{"@a > 50", map[string]Item{"@a": genValue(`51`, jsonNumber)}, true},
// GE
{"30 >= 20", nil, true},
{"20 >= 20", nil, true},
{"10 >= 20", nil, false},
{"@a >= 50", map[string]Item{"@a": genValue(`49`, jsonNumber)}, false},
{"@a >= 50", map[string]Item{"@a": genValue(`50`, jsonNumber)}, true},
{"@a >= 50", map[string]Item{"@a": genValue(`51`, jsonNumber)}, true},
// EQ
{"20 == 20", nil, true},
{"20 == 21", nil, false},
{"true == true", nil, true},
{"true == false", nil, false},
{"@a == @b", map[string]Item{"@a": genValue(`"one"`, jsonString), "@b": genValue(`"one"`, jsonString)}, true},
{"@a == @b", map[string]Item{"@a": genValue(`"one"`, jsonString), "@b": genValue(`"two"`, jsonString)}, false},
{`"fire" == "fire"`, nil, true},
{`"fire" == "water"`, nil, false},
{`@a == "toronto"`, map[string]Item{"@a": genValue(`"toronto"`, jsonString)}, true},
{`@a == "toronto"`, map[string]Item{"@a": genValue(`"los angeles"`, jsonString)}, false},
{`@a == 3.4`, map[string]Item{"@a": genValue(`3.4`, jsonNumber)}, true},
{`@a == 3.4`, map[string]Item{"@a": genValue(`3.41`, jsonNumber)}, false},
{`@a == null`, map[string]Item{"@a": genValue(`null`, jsonNull)}, true},
// NEQ
{"20 != 20", nil, false},
{"20 != 21", nil, true},
{"true != true", nil, false},
{"true != false", nil, true},
{"@a != @b", map[string]Item{"@a": genValue(`"one"`, jsonString), "@b": genValue(`"one"`, jsonString)}, false},
{"@a != @b", map[string]Item{"@a": genValue(`"one"`, jsonString), "@b": genValue(`"two"`, jsonString)}, true},
{`"fire" != "fire"`, nil, false},
{`"fire" != "water"`, nil, true},
{`@a != "toronto"`, map[string]Item{"@a": genValue(`"toronto"`, jsonString)}, false},
{`@a != "toronto"`, map[string]Item{"@a": genValue(`"los angeles"`, jsonString)}, true},
{`@a != 3.4`, map[string]Item{"@a": genValue(`3.4`, jsonNumber)}, false},
{`@a != 3.4`, map[string]Item{"@a": genValue(`3.41`, jsonNumber)}, true},
{`@a != null`, map[string]Item{"@a": genValue(`null`, jsonNull)}, false},
// Plus
{"20 + 7", nil, 27},
{"20 + 6.999999", nil, 26.999999},
// Minus
{"20 - 7", nil, 13},
{"20 - 7.11111", nil, 12.88889},
// Minus Unary
{"-27", nil, -27},
{"30 - -3", nil, 33},
{"30 + -3", nil, 27},
{"2 +++++ 3", nil, 5},
{"2+--3", nil, 5},
// Star
{"20 * 7", nil, 140},
{"20 * 6.999999", nil, 139.99998},
{"20 * -7", nil, -140},
{"-20 * -7", nil, 140},
// Slash
{"20 / 5", nil, 4},
{"20 / 6.999999 - 2.85714326531 <= 0.00000001", nil, true},
// Hat
{"7 ^ 4", nil, 2401},
{"2 ^ -2", nil, 0.25},
{"((7 ^ -4) - 0.00041649312) <= 0.0001", nil, true},
// Mod
{"7.5 % 4", nil, 3.5},
{"2 % -2", nil, 0},
{"11 % 22", nil, 11},
// Negate
{"!true", nil, false},
{"!false", nil, true},
// Mix
{"20 >= 20 || 2 == 2", nil, true},
{"20 > @.test && @.test < 13 && @.test > 1.99994", map[string]Item{"@.test": genValue(`10.23423`, jsonNumber)}, true},
{"20 > @.test && @.test < 13 && @.test > 1.99994", map[string]Item{"@.test": genValue(`15.3423`, jsonNumber)}, false},
}
func genValue(val string, typ int) Item {
return Item{
val: []byte(val),
typ: typ,
}
}
func TestExpressions(t *testing.T) {
as := assert.New(t)
emptyFields := map[string]Item{}
for _, test := range exprTests {
if test.fields == nil {
test.fields = emptyFields
}
lexer := NewSliceLexer([]byte(test.input), EXPRESSION)
items := readerToArray(lexer)
// trim EOF
items = items[0 : len(items)-1]
items_post, err := infixToPostFix(items)
if as.NoError(err, "Could not transform to postfix\nTest: %q", test.input) {
val, err := evaluatePostFix(items_post, test.fields)
if as.NoError(err, "Could not evaluate postfix\nTest Input: %q\nTest Values:%q\nError:%q", test.input, test.fields, err) {
as.EqualValues(test.expectedValue, val, "\nTest: %q\nActual: %v \nExpected %v\n", test.input, val, test.expectedValue)
}
}
}
}
var exprErrorTests = []struct {
input string
fields map[string]Item
expectedErrorSubstring string
}{
{"@a == @b", map[string]Item{"@a": genValue(`"one"`, jsonString), "@b": genValue("3.4", jsonNumber)}, "cannot be compared"},
{")(", nil, "Mismatched parentheses"},
{")123", nil, "Mismatched parentheses"},
{"20 == null", nil, "cannot be compared"},
{`"toronto" == null`, nil, "cannot be compared"},
{`false == 20`, nil, "cannot be compared"},
{`"nick" == 20`, nil, "cannot be compared"},
{"20 != null", nil, "cannot be compared"},
{`"toronto" != null`, nil, "cannot be compared"},
{`false != 20`, nil, "cannot be compared"},
{`"nick" != 20`, nil, "cannot be compared"},
{``, nil, "Bad Expression"},
{`==`, nil, "Bad Expression"},
{`!=`, nil, "Not enough operands"},
{`!23`, nil, "cannot be compared"},
{`"nick" || true`, nil, "cannot be compared"},
{`"nick" >= 3.2`, nil, "cannot be compared"},
{`"nick" >3.2`, nil, "cannot be compared"},
{`"nick" <= 3.2`, nil, "cannot be compared"},
{`"nick" < 3.2`, nil, "cannot be compared"},
{`"nick" + 3.2`, nil, "cannot be compared"},
{`"nick" - 3.2`, nil, "cannot be compared"},
{`"nick" / 3.2`, nil, "cannot be compared"},
{`"nick" * 3.2`, nil, "cannot be compared"},
{`"nick" % 3.2`, nil, "cannot be compared"},
{`"nick"+`, nil, "cannot be compared"},
{`"nick"-`, nil, "cannot be compared"},
{`"nick"^3.2`, nil, "cannot be compared"},
{`@a == null`, map[string]Item{"@a": genValue(`3.41`, jsonNumber)}, "cannot be compared"},
}
func TestBadExpressions(t *testing.T) {
as := assert.New(t)
emptyFields := map[string]Item{}
for _, test := range exprErrorTests {
if test.fields == nil {
test.fields = emptyFields
}
lexer := NewSliceLexer([]byte(test.input), EXPRESSION)
items := readerToArray(lexer)
// trim EOF
items = items[0 : len(items)-1]
items_post, err := infixToPostFix(items)
if err != nil {
as.True(strings.Contains(err.Error(), test.expectedErrorSubstring), "Test Input: %q\nError %q does not contain %q", test.input, err.Error(), test.expectedErrorSubstring)
continue
}
if as.NoError(err, "Could not transform to postfix\nTest: %q", test.input) {
_, err := evaluatePostFix(items_post, test.fields)
if as.Error(err, "Could not evaluate postfix\nTest Input: %q\nTest Values:%q\nError:%s", test.input, test.fields, err) {
as.True(strings.Contains(err.Error(), test.expectedErrorSubstring), "Test Input: %q\nError %s does not contain %q", test.input, err.Error(), test.expectedErrorSubstring)
}
}
}
}

266
json_states.go Normal file
View File

@ -0,0 +1,266 @@
package jsonpath
const (
jsonError = iota
jsonEOF
jsonBraceLeft
jsonBraceRight
jsonBracketLeft
jsonBracketRight
jsonColon
jsonComma
jsonNumber
jsonString
jsonNull
jsonKey
jsonBool
)
var trueBytes = []byte{'t', 'r', 'u', 'e'}
var falseBytes = []byte{'f', 'a', 'l', 's', 'e'}
var nullBytes = []byte{'n', 'u', 'l', 'l'}
var jsonTokenNames = map[int]string{
jsonEOF: "EOF",
jsonError: "ERROR",
jsonBraceLeft: "{",
jsonBraceRight: "}",
jsonBracketLeft: "[",
jsonBracketRight: "]",
jsonColon: ":",
jsonComma: ",",
jsonNumber: "NUMBER",
jsonString: "STRING",
jsonNull: "NULL",
jsonKey: "KEY",
jsonBool: "BOOL",
}
var JSON = lexJsonRoot
func lexJsonRoot(l lexer, state *intStack) stateFn {
ignoreSpaceRun(l)
cur := l.peek()
var next stateFn
switch cur {
case '{':
next = stateJsonObjectOpen
case '[':
next = stateJsonArrayOpen
default:
next = l.errorf("Expected '{' or '[' at root of JSON instead of %#U", cur)
}
return next
}
func stateJsonObjectOpen(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != '{' {
return l.errorf("Expected '{' as start of object instead of %#U", cur)
}
l.emit(jsonBraceLeft)
state.push(jsonBraceLeft)
return stateJsonObject
}
func stateJsonArrayOpen(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != '[' {
return l.errorf("Expected '[' as start of array instead of %#U", cur)
}
l.emit(jsonBracketLeft)
state.push(jsonBracketLeft)
return stateJsonArray
}
func stateJsonObject(l lexer, state *intStack) stateFn {
var next stateFn
cur := l.peek()
switch cur {
case '}':
if top, ok := state.peek(); ok && top != jsonBraceLeft {
next = l.errorf("Received %#U but has no matching '{'", cur)
break
}
l.take()
l.emit(jsonBraceRight)
state.pop()
next = stateJsonAfterValue
case '"':
next = stateJsonKey
default:
next = l.errorf("Expected '}' or \" within an object instead of %#U", cur)
}
return next
}
func stateJsonArray(l lexer, state *intStack) stateFn {
var next stateFn
cur := l.peek()
switch cur {
case ']':
if top, ok := state.peek(); ok && top != jsonBracketLeft {
next = l.errorf("Received %#U but has no matching '['", cur)
break
}
l.take()
l.emit(jsonBracketRight)
state.pop()
next = stateJsonAfterValue
default:
next = stateJsonValue
}
return next
}
func stateJsonAfterValue(l lexer, state *intStack) stateFn {
cur := l.take()
top, ok := state.peek()
topVal := noValue
if ok {
topVal = top
}
switch cur {
case ',':
l.emit(jsonComma)
switch topVal {
case jsonBraceLeft:
return stateJsonKey
case jsonBracketLeft:
return stateJsonValue
case noValue:
return l.errorf("Found %#U outside of array or object", cur)
default:
return l.errorf("Unexpected character in lexer stack: %#U", cur)
}
case '}':
l.emit(jsonBraceRight)
state.pop()
switch topVal {
case jsonBraceLeft:
return stateJsonAfterValue
case jsonBracketLeft:
return l.errorf("Unexpected %#U in array", cur)
case noValue:
return stateJsonAfterRoot
}
case ']':
l.emit(jsonBracketRight)
state.pop()
switch topVal {
case jsonBraceLeft:
return l.errorf("Unexpected %#U in object", cur)
case jsonBracketLeft:
return stateJsonAfterValue
case noValue:
return stateJsonAfterRoot
}
case eof:
if state.len() == 0 {
l.emit(jsonEOF)
return nil
} else {
return l.errorf("Unexpected EOF instead of value")
}
default:
return l.errorf("Unexpected character after json value token: %#U", cur)
}
return nil
}
func stateJsonKey(l lexer, state *intStack) stateFn {
if err := l.takeString(); err != nil {
return l.errorf(err.Error())
}
l.emit(jsonKey)
return stateJsonColon
}
func stateJsonColon(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != ':' {
return l.errorf("Expected ':' after key string instead of %#U", cur)
}
l.emit(jsonColon)
return stateJsonValue
}
func stateJsonValue(l lexer, state *intStack) stateFn {
cur := l.peek()
switch cur {
case eof:
return l.errorf("Unexpected EOF instead of value")
case '"':
return stateJsonString
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
return stateJsonNumber
case 't', 'f':
return stateJsonBool
case 'n':
return stateJsonNull
case '{':
return stateJsonObjectOpen
case '[':
return stateJsonArrayOpen
default:
return l.errorf("Unexpected character as start of value: %#U", cur)
}
}
func stateJsonString(l lexer, state *intStack) stateFn {
if err := l.takeString(); err != nil {
return l.errorf(err.Error())
}
l.emit(jsonString)
return stateJsonAfterValue
}
func stateJsonNumber(l lexer, state *intStack) stateFn {
if err := takeJSONNumeric(l); err != nil {
return l.errorf(err.Error())
}
l.emit(jsonNumber)
return stateJsonAfterValue
}
func stateJsonBool(l lexer, state *intStack) stateFn {
cur := l.peek()
var match []byte
switch cur {
case 't':
match = trueBytes
case 'f':
match = falseBytes
}
if !takeExactSequence(l, match) {
return l.errorf("Could not parse value as 'true' or 'false'")
}
l.emit(jsonBool)
return stateJsonAfterValue
}
func stateJsonNull(l lexer, state *intStack) stateFn {
if !takeExactSequence(l, nullBytes) {
return l.errorf("Could not parse value as 'null'")
}
l.emit(jsonNull)
return stateJsonAfterValue
}
func stateJsonAfterRoot(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != eof {
return l.errorf("Expected EOF instead of %#U", cur)
}
l.emit(jsonEOF)
return nil
}

172
json_states_test.go Normal file
View File

@ -0,0 +1,172 @@
package jsonpath
import (
"testing"
"github.com/stretchr/testify/assert"
)
var jsonTests = []lexTest{
{"empty object", `{}`, []int{jsonBraceLeft, jsonBraceRight, jsonEOF}},
{"empty array", `[]`, []int{jsonBracketLeft, jsonBracketRight, jsonEOF}},
{"key string", `{"key" :"value"}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonString, jsonBraceRight, jsonEOF}},
{"multiple pairs", `{"key" :"value","key2" :"value"}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonString, jsonComma, jsonKey, jsonColon, jsonString, jsonBraceRight, jsonEOF}},
{"key number", `{"key" : 12.34e+56}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonNumber, jsonBraceRight, jsonEOF}},
{"key true", `{"key" :true}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBool, jsonBraceRight, jsonEOF}},
{"key false", `{"key" :false}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBool, jsonBraceRight, jsonEOF}},
{"key null", `{"key" :null}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonNull, jsonBraceRight, jsonEOF}},
{"key arrayOf number", `{"key" :[23]}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBracketLeft, jsonNumber, jsonBracketRight, jsonBraceRight, jsonEOF}},
{"key array", `{"key" :[23,"45",67]}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBracketLeft, jsonNumber, jsonComma, jsonString, jsonComma, jsonNumber, jsonBracketRight, jsonBraceRight, jsonEOF}},
{"key array", `{"key" :["45",{}]}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBracketLeft, jsonString, jsonComma, jsonBraceLeft, jsonBraceRight, jsonBracketRight, jsonBraceRight, jsonEOF}},
{"key nestedObject", `{"key" :{"innerkey":"value"}}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBraceLeft, jsonKey, jsonColon, jsonString, jsonBraceRight, jsonBraceRight, jsonEOF}},
{"key nestedArray", `[1,["a","b"]]`, []int{jsonBracketLeft, jsonNumber, jsonComma, jsonBracketLeft, jsonString, jsonComma, jsonString, jsonBracketRight, jsonBracketRight, jsonEOF}},
}
func TestValidJson(t *testing.T) {
as := assert.New(t)
for _, test := range jsonTests {
lexer := NewSliceLexer([]byte(test.input), JSON)
types := itemsToTypes(readerToArray(lexer))
as.EqualValues(types, test.tokenTypes, "Testing of %q: \nactual\n\t%+v\nexpected\n\t%v", test.name, typesDescription(types, jsonTokenNames), typesDescription(test.tokenTypes, jsonTokenNames))
}
}
var errorJsonTests = []lexTest{
{"Missing end brace", `{`, []int{jsonBraceLeft, jsonError}},
{"Missing start brace", `}`, []int{jsonError}},
{"Missing key start quote", `{key":true}`, []int{jsonBraceLeft, jsonError}},
{"Missing key end quote", `{"key:true}`, []int{jsonBraceLeft, jsonError}},
{"Missing colon", `{"key"true}`, []int{jsonBraceLeft, jsonKey, jsonError}},
{"Missing value", `{"key":}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonError}},
{"Missing string start quote", `{"key":test"}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonError}},
{"Missing embedded array bracket", `{"key":[}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBracketLeft, jsonError}},
{"Missing values in array", `{"key":[,]`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBracketLeft, jsonError}},
{"Missing value after comma", `{"key":[343,]}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBracketLeft, jsonNumber, jsonComma, jsonError}},
{"Missing comma in array", `{"key":[234 424]}`, []int{jsonBraceLeft, jsonKey, jsonColon, jsonBracketLeft, jsonNumber, jsonError}},
}
func TestMalformedJson(t *testing.T) {
as := assert.New(t)
for _, test := range errorJsonTests {
lexer := NewSliceLexer([]byte(test.input), JSON)
types := itemsToTypes(readerToArray(lexer))
as.EqualValues(types, test.tokenTypes, "Testing of %q: \nactual\n\t%+v\nexpected\n\t%v", test.name, typesDescription(types, jsonTokenNames), typesDescription(test.tokenTypes, jsonTokenNames))
}
}
func itemsToTypes(items []Item) []int {
types := make([]int, len(items))
for i, item := range items {
types[i] = item.typ
}
return types
}
// func TestEarlyTerminationForJSON(t *testing.T) {
// as := assert.New(t)
// wg := sync.WaitGroup{}
// lexer := NewSliceLexer(`{"key":"value", "key2":{"ikey":3}, "key3":[1,2,3,4]}`)
// wg.Add(1)
// go func() {
// lexer.Run(JSON)
// wg.Done()
// }()
// // Pop a few items
// <-lexer.items
// <-lexer.items
// // Kill command
// close(lexer.kill)
// wg.Wait()
// remainingItems := readerToArray(lexer.items)
// // TODO: Occasionally fails - rethink this
// _ = as
// _ = remainingItems
// // as.True(len(remainingItems) <= bufferSize, "Count of remaining items should be less than buffer size: %d", len(remainingItems))
// }
var examples = []string{
`{"items":[
{
"name": "example document for wicked fast parsing of huge json docs",
"integer": 123,
"totally sweet scientific notation": -123.123e-2,
"unicode? you betcha!": "ú™£¢∞§\u2665",
"zero character": "0",
"null is boring": null
},
{
"name": "another object",
"cooler than first object?": true,
"nested object": {
"nested object?": true,
"is nested array the same combination i have on my luggage?": true,
"nested array": [1,2,3,4,5]
},
"false": false
}
]}`,
`{
"$schema": "http://json-schema.org/draft-04/schema#",
"title": "Product set",
"type": "array",
"items": {
"title": "Product",
"type": "object",
"properties": {
"id": {
"description": "The unique identifier for a product",
"type": "number"
},
"name": {
"type": "string"
},
"price": {
"type": "number",
"minimum": 0,
"exclusiveMinimum": true
},
"tags": {
"type": "array",
"items": {
"type": "string"
},
"minItems": 1,
"uniqueItems": true
},
"dimensions": {
"type": "object",
"properties": {
"length": {"type": "number"},
"width": {"type": "number"},
"height": {"type": "number"}
},
"required": ["length", "width", "height"]
},
"warehouseLocation": {
"description": "Coordinates of the warehouse with the product",
"$ref": "http://json-schema.org/geo"
}
},
"required": ["id", "name", "price"]
}
}`,
}
func TestMixedCaseJson(t *testing.T) {
as := assert.New(t)
for _, json := range examples {
lexer := NewSliceLexer([]byte(json), JSON)
items := readerToArray(lexer)
for _, i := range items {
as.False(i.typ == jsonError, "Found error while parsing: %q", i.val)
}
}
}

86
lexer.go Normal file
View File

@ -0,0 +1,86 @@
package jsonpath
type Pos int
type stateFn func(lexer, *intStack) stateFn
const (
lexError = 0 // must match jsonError and pathError
lexEOF = 1
eof = -1
noValue = -2
)
type Item struct {
typ int
pos Pos // The starting position, in bytes, of this Item in the input string.
val []byte
}
// Used by evaluator
type tokenReader interface {
next() (*Item, bool)
}
// Used by state functions
type lexer interface {
tokenReader
take() int
takeString() error
peek() int
emit(int)
ignore()
errorf(string, ...interface{}) stateFn
reset()
}
type lex struct {
initialState stateFn
currentStateFn stateFn
item Item
hasItem bool
stack intStack
}
func newLex(initial stateFn) lex {
return lex{
initialState: initial,
currentStateFn: initial,
item: Item{},
stack: *newIntStack(),
}
}
func (i *Item) clone() *Item {
ic := Item{
typ: i.typ,
pos: i.pos,
val: make([]byte, len(i.val)),
}
copy(ic.val, i.val)
return &ic
}
func itemsDescription(items []Item, nameMap map[int]string) []string {
vals := make([]string, len(items))
for i, item := range items {
vals[i] = itemDescription(&item, nameMap)
}
return vals
}
func itemDescription(item *Item, nameMap map[int]string) string {
var found bool
val, found := nameMap[item.typ]
if !found {
return string(item.val)
}
return val
}
func typesDescription(types []int, nameMap map[int]string) []string {
vals := make([]string, len(types))
for i, val := range types {
vals[i] = nameMap[val]
}
return vals
}

161
lexer_reader.go Normal file
View File

@ -0,0 +1,161 @@
package jsonpath
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
)
type readerLexer struct {
lex
bufInput *bufio.Reader
input io.Reader
pos Pos
nextByte int
lexeme *bytes.Buffer
}
func NewReaderLexer(rr io.Reader, initial stateFn) *readerLexer {
l := readerLexer{
input: rr,
bufInput: bufio.NewReader(rr),
nextByte: noValue,
lex: newLex(initial),
lexeme: bytes.NewBuffer(make([]byte, 0, 100)),
}
return &l
}
func (l *readerLexer) take() int {
if l.nextByte == noValue {
l.peek()
}
nr := l.nextByte
l.nextByte = noValue
l.lexeme.WriteByte(byte(nr))
return nr
}
func (l *readerLexer) takeString() error {
cur := l.take()
if cur != '"' {
return fmt.Errorf("Expected \" as start of string instead of %#U", cur)
}
var previous byte
looper:
for {
curByte, err := l.bufInput.ReadByte()
if err == io.EOF {
return errors.New("Unexpected EOF in string")
}
l.lexeme.WriteByte(curByte)
if curByte == '"' {
if previous != '\\' {
break looper
} else {
curByte, err = l.bufInput.ReadByte()
if err == io.EOF {
return errors.New("Unexpected EOF in string")
}
l.lexeme.WriteByte(curByte)
}
}
previous = curByte
}
return nil
}
func (l *readerLexer) peek() int {
if l.nextByte != noValue {
return l.nextByte
}
r, err := l.bufInput.ReadByte()
if err == io.EOF {
l.nextByte = eof
return eof
}
l.nextByte = int(r)
return l.nextByte
}
func (l *readerLexer) emit(t int) {
l.setItem(t, l.pos, l.lexeme.Bytes())
l.pos += Pos(l.lexeme.Len())
l.hasItem = true
if t == lexEOF {
// Do not capture eof character to match slice_lexer
l.item.val = []byte{}
}
// Ignore whitespace after this token
if l.nextByte == noValue {
l.peek()
}
// ignore white space
for l.nextByte != eof {
if l.nextByte == ' ' || l.nextByte == '\t' || l.nextByte == '\r' || l.nextByte == '\n' {
l.pos++
r, err := l.bufInput.ReadByte()
if err == io.EOF {
l.nextByte = eof
} else {
l.nextByte = int(r)
}
} else {
break
}
}
}
func (l *readerLexer) setItem(typ int, pos Pos, val []byte) {
l.item.typ = typ
l.item.pos = pos
l.item.val = val
}
func (l *readerLexer) ignore() {
l.pos += Pos(l.lexeme.Len())
l.lexeme.Reset()
}
func (l *readerLexer) next() (*Item, bool) {
l.lexeme.Reset()
for {
if l.currentStateFn == nil {
break
}
l.currentStateFn = l.currentStateFn(l, &l.stack)
if l.hasItem {
l.hasItem = false
return &l.item, true
}
}
return &l.item, false
}
func (l *readerLexer) errorf(format string, args ...interface{}) stateFn {
l.setItem(lexError, l.pos, []byte(fmt.Sprintf(format, args...)))
l.lexeme.Truncate(0)
l.hasItem = true
return nil
}
func (l *readerLexer) reset() {
l.bufInput.Reset(l.input)
l.lexeme.Reset()
l.nextByte = noValue
l.pos = 0
l.lex = newLex(l.initialState)
}

131
lexer_slice.go Normal file
View File

@ -0,0 +1,131 @@
package jsonpath
import (
"errors"
"fmt"
)
type sliceLexer struct {
lex
input []byte // the []byte being scanned.
start Pos // start position of this Item.
pos Pos // current position in the input
}
func NewSliceLexer(input []byte, initial stateFn) *sliceLexer {
l := &sliceLexer{
lex: newLex(initial),
input: input,
}
return l
}
func (l *sliceLexer) take() int {
if int(l.pos) >= len(l.input) {
return eof
}
r := int(l.input[l.pos])
l.pos += 1
return r
}
func (l *sliceLexer) takeString() error {
curPos := l.pos
inputLen := len(l.input)
if int(curPos) >= inputLen {
return errors.New("End of file where string expected")
}
cur := int(l.input[curPos])
curPos++
if cur != '"' {
l.pos = curPos
return fmt.Errorf("Expected \" as start of string instead of %#U", cur)
}
var previous int
looper:
for {
if int(curPos) >= inputLen {
l.pos = curPos
return errors.New("End of file where string expected")
}
cur := int(l.input[curPos])
curPos++
if cur == '"' {
if previous == noValue || previous != '\\' {
break looper
} else {
l.take()
}
}
previous = cur
}
l.pos = curPos
return nil
}
func (l *sliceLexer) peek() int {
if int(l.pos) >= len(l.input) {
return eof
}
return int(l.input[l.pos])
}
func (l *sliceLexer) emit(t int) {
l.setItem(t, l.start, l.input[l.start:l.pos])
l.hasItem = true
// Ignore whitespace after this token
for int(l.pos) < len(l.input) {
r := l.input[l.pos]
if r == ' ' || r == '\t' || r == '\r' || r == '\n' {
l.pos++
} else {
break
}
}
l.start = l.pos
}
func (l *sliceLexer) setItem(typ int, pos Pos, val []byte) {
l.item.typ = typ
l.item.pos = pos
l.item.val = val
}
func (l *sliceLexer) ignore() {
l.start = l.pos
}
func (l *sliceLexer) next() (*Item, bool) {
for {
if l.currentStateFn == nil {
break
}
l.currentStateFn = l.currentStateFn(l, &l.stack)
if l.hasItem {
l.hasItem = false
return &l.item, true
}
}
return &l.item, false
}
func (l *sliceLexer) errorf(format string, args ...interface{}) stateFn {
l.setItem(lexError, l.start, []byte(fmt.Sprintf(format, args...)))
l.start = l.pos
l.hasItem = true
return nil
}
func (l *sliceLexer) reset() {
l.start = 0
l.pos = 0
l.lex = newLex(l.initialState)
}

309
lexer_test.go Normal file
View File

File diff suppressed because one or more lines are too long

181
misc.go Normal file
View File

@ -0,0 +1,181 @@
package jsonpath
import (
"errors"
"fmt"
)
func takeExponent(l lexer) error {
r := l.peek()
if r != 'e' && r != 'E' {
return nil
}
l.take()
r = l.take()
switch r {
case '+', '-':
// Check digit immediately follows sign
if d := l.peek(); !(d >= '0' && d <= '9') {
return fmt.Errorf("Expected digit after numeric sign instead of %#U", d)
}
takeDigits(l)
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeDigits(l)
default:
return fmt.Errorf("Expected digit after 'e' instead of %#U", r)
}
return nil
}
func takeJSONNumeric(l lexer) error {
cur := l.take()
switch cur {
case '-':
// Check digit immediately follows sign
if d := l.peek(); !(d >= '0' && d <= '9') {
return fmt.Errorf("Expected digit after dash instead of %#U", d)
}
takeDigits(l)
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeDigits(l)
default:
return fmt.Errorf("Expected digit or dash instead of %#U", cur)
}
// fraction or exponent
cur = l.peek()
switch cur {
case '.':
l.take()
// Check digit immediately follows period
if d := l.peek(); !(d >= '0' && d <= '9') {
return fmt.Errorf("Expected digit after '.' instead of %#U", d)
}
takeDigits(l)
if err := takeExponent(l); err != nil {
return err
}
case 'e', 'E':
if err := takeExponent(l); err != nil {
return err
}
}
return nil
}
func takeDigits(l lexer) {
for {
d := l.peek()
if d >= '0' && d <= '9' {
l.take()
} else {
break
}
}
}
// Only used at the very beginning of parsing. After that, the emit() function
// automatically skips whitespace.
func ignoreSpaceRun(l lexer) {
for {
r := l.peek()
if r == ' ' || r == '\t' || r == '\r' || r == '\n' {
l.take()
} else {
break
}
}
l.ignore()
}
func takeExactSequence(l lexer, str []byte) bool {
for _, r := range str {
v := l.take()
if v != int(r) {
return false
}
}
return true
}
func readerToArray(tr tokenReader) []Item {
vals := make([]Item, 0)
for {
i, ok := tr.next()
if !ok {
break
}
v := *i
s := make([]byte, len(v.val))
copy(s, v.val)
v.val = s
vals = append(vals, v)
}
return vals
}
func findErrors(items []Item) (Item, bool) {
for _, i := range items {
if i.typ == lexError {
return i, true
}
}
return Item{}, false
}
func byteSlicesEqual(a, b []byte) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
func firstError(errors ...error) error {
for _, e := range errors {
if e != nil {
return e
}
}
return nil
}
func abs(x int) int {
switch {
case x < 0:
return -x
case x == 0:
return 0 // return correctly abs(-0)
}
return x
}
//TODO: Kill the need for this
func getJsonTokenType(val []byte) (int, error) {
if len(val) == 0 {
return -1, errors.New("No Value")
}
switch val[0] {
case '{':
return jsonBraceLeft, nil
case '"':
return jsonString, nil
case '[':
return jsonBracketLeft, nil
case 'n':
return jsonNull, nil
case 't', 'b':
return jsonBool, nil
case '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
return jsonNumber, nil
default:
return -1, errors.New("Unrecognized Json Value")
}
}

208
path.go Normal file
View File

@ -0,0 +1,208 @@
package jsonpath
import (
"errors"
"fmt"
"strconv"
)
const (
opTypeIndex = iota
opTypeIndexRange
opTypeIndexWild
opTypeName
opTypeNameList
opTypeNameWild
)
type Path struct {
stringValue string
operators []*operator
captureEndValue bool
}
type operator struct {
typ int
indexStart int
indexEnd int
hasIndexEnd bool
keyStrings map[string]struct{}
whereClauseBytes []byte
dependentPaths []*Path
whereClause []Item
}
func genIndexKey(tr tokenReader) (*operator, error) {
k := &operator{}
var t *Item
var ok bool
if t, ok = tr.next(); !ok {
return nil, errors.New("Expected number, key, or *, but got none")
}
switch t.typ {
case pathWildcard:
k.typ = opTypeIndexWild
k.indexStart = 0
if t, ok = tr.next(); !ok {
return nil, errors.New("Expected ] after *, but got none")
}
if t.typ != pathBracketRight {
return nil, fmt.Errorf("Expected ] after * instead of %q", t.val)
}
case pathIndex:
v, err := strconv.Atoi(string(t.val))
if err != nil {
return nil, fmt.Errorf("Could not parse %q into int64", t.val)
}
k.indexStart = v
k.indexEnd = v
k.hasIndexEnd = true
if t, ok = tr.next(); !ok {
return nil, errors.New("Expected number or *, but got none")
}
switch t.typ {
case pathIndexRange:
if t, ok = tr.next(); !ok {
return nil, errors.New("Expected number or *, but got none")
}
switch t.typ {
case pathIndex:
v, err := strconv.Atoi(string(t.val))
if err != nil {
return nil, fmt.Errorf("Could not parse %q into int64", t.val)
}
k.indexEnd = v - 1
k.hasIndexEnd = true
if t, ok = tr.next(); !ok || t.typ != pathBracketRight {
return nil, errors.New("Expected ], but got none")
}
case pathBracketRight:
k.hasIndexEnd = false
default:
return nil, fmt.Errorf("Unexpected value within brackets after index: %q", t.val)
}
k.typ = opTypeIndexRange
case pathBracketRight:
k.typ = opTypeIndex
default:
return nil, fmt.Errorf("Unexpected value within brackets after index: %q", t.val)
}
case pathKey:
k.keyStrings = map[string]struct{}{string(t.val[1 : len(t.val)-1]): struct{}{}}
k.typ = opTypeName
if t, ok = tr.next(); !ok || t.typ != pathBracketRight {
return nil, errors.New("Expected ], but got none")
}
default:
return nil, fmt.Errorf("Unexpected value within brackets: %q", t.val)
}
return k, nil
}
func parsePath(pathString string) (*Path, error) {
lexer := NewSliceLexer([]byte(pathString), PATH)
p, err := tokensToOperators(lexer)
if err != nil {
return nil, err
}
p.stringValue = pathString
//Generate dependent paths
for _, op := range p.operators {
if len(op.whereClauseBytes) > 0 {
var err error
trimmed := op.whereClauseBytes[1 : len(op.whereClauseBytes)-1]
whereLexer := NewSliceLexer(trimmed, EXPRESSION)
items := readerToArray(whereLexer)
if errItem, found := findErrors(items); found {
return nil, errors.New(string(errItem.val))
}
// transform expression into postfix form
op.whereClause, err = infixToPostFix(items[:len(items)-1]) // trim EOF
if err != nil {
return nil, err
}
op.dependentPaths = make([]*Path, 0)
// parse all paths in expression
for _, item := range op.whereClause {
if item.typ == exprPath {
p, err := parsePath(string(item.val))
if err != nil {
return nil, err
}
op.dependentPaths = append(op.dependentPaths, p)
}
}
}
}
return p, nil
}
func tokensToOperators(tr tokenReader) (*Path, error) {
q := &Path{
stringValue: "",
captureEndValue: false,
operators: make([]*operator, 0),
}
for {
p, ok := tr.next()
if !ok {
break
}
switch p.typ {
case pathRoot:
if len(q.operators) != 0 {
return nil, errors.New("Unexpected root node after start")
}
continue
case pathCurrent:
if len(q.operators) != 0 {
return nil, errors.New("Unexpected current node after start")
}
continue
case pathPeriod:
continue
case pathBracketLeft:
k, err := genIndexKey(tr)
if err != nil {
return nil, err
}
q.operators = append(q.operators, k)
case pathKey:
keyName := p.val
if len(p.val) == 0 {
return nil, fmt.Errorf("Key length is zero at %d", p.pos)
}
if p.val[0] == '"' && p.val[len(p.val)-1] == '"' {
keyName = p.val[1 : len(p.val)-1]
}
q.operators = append(q.operators, &operator{typ: opTypeName, keyStrings: map[string]struct{}{string(keyName): struct{}{}}})
case pathWildcard:
q.operators = append(q.operators, &operator{typ: opTypeNameWild})
case pathValue:
q.captureEndValue = true
case pathWhere:
case pathExpression:
if len(q.operators) == 0 {
return nil, errors.New("Cannot add where clause on last key")
}
last := q.operators[len(q.operators)-1]
if last.whereClauseBytes != nil {
return nil, errors.New("Expression on last key already set")
}
last.whereClauseBytes = p.val
case pathError:
return q, errors.New(string(p.val))
}
}
return q, nil
}

213
path_states.go Normal file
View File

@ -0,0 +1,213 @@
package jsonpath
const (
pathError = iota
pathEOF
pathRoot
pathCurrent
pathKey
pathBracketLeft
pathBracketRight
pathIndex
pathOr
pathIndexRange
pathLength
pathWildcard
pathPeriod
pathValue
pathWhere
pathExpression
)
var pathTokenNames = map[int]string{
pathError: "ERROR",
pathEOF: "EOF",
pathRoot: "$",
pathCurrent: "@",
pathKey: "KEY",
pathBracketLeft: "[",
pathBracketRight: "]",
pathIndex: "INDEX",
pathOr: "|",
pathIndexRange: ":",
pathLength: "LENGTH",
pathWildcard: "*",
pathPeriod: ".",
pathValue: "+",
pathWhere: "?",
pathExpression: "EXPRESSION",
}
var PATH = lexPathStart
func lexPathStart(l lexer, state *intStack) stateFn {
ignoreSpaceRun(l)
cur := l.take()
switch cur {
case '$':
l.emit(pathRoot)
case '@':
l.emit(pathCurrent)
default:
return l.errorf("Expected $ or @ at start of path instead of %#U", cur)
}
return lexPathAfterKey
}
func lexPathAfterKey(l lexer, state *intStack) stateFn {
cur := l.take()
switch cur {
case '.':
l.emit(pathPeriod)
return lexKey
case '[':
l.emit(pathBracketLeft)
return lexPathBracketOpen
case '+':
l.emit(pathValue)
return lexPathAfterValue
case '?':
l.emit(pathWhere)
return lexPathExpression
case eof:
l.emit(pathEOF)
default:
return l.errorf("Unrecognized rune after path element %#U", cur)
}
return nil
}
func lexPathExpression(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != '(' {
return l.errorf("Expected $ at start of path instead of %#U", cur)
}
parenLeftCount := 1
for {
cur = l.take()
switch cur {
case '(':
parenLeftCount++
case ')':
parenLeftCount--
case eof:
return l.errorf("Unexpected EOF within expression")
}
if parenLeftCount == 0 {
break
}
}
l.emit(pathExpression)
return lexPathAfterKey
}
func lexPathBracketOpen(l lexer, state *intStack) stateFn {
switch l.peek() {
case '*':
l.take()
l.emit(pathWildcard)
return lexPathBracketClose
case '"':
l.takeString()
l.emit(pathKey)
return lexPathBracketClose
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
l.take()
takeDigits(l)
l.emit(pathIndex)
return lexPathIndexRange
case eof:
l.emit(pathEOF)
}
return nil
}
func lexPathBracketClose(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != ']' {
return l.errorf("Expected ] instead of %#U", cur)
}
l.emit(pathBracketRight)
return lexPathAfterKey
}
func lexKey(l lexer, state *intStack) stateFn {
// TODO: Support globbing of keys
switch l.peek() {
case '*':
l.take()
l.emit(pathWildcard)
return lexPathAfterKey
case '"':
l.takeString()
l.emit(pathKey)
return lexPathAfterKey
case eof:
l.take()
l.emit(pathEOF)
return nil
default:
for {
v := l.peek()
if v == '.' || v == '[' || v == '+' || v == '?' || v == eof {
break
}
l.take()
}
l.emit(pathKey)
return lexPathAfterKey
}
}
func lexPathIndexRange(l lexer, state *intStack) stateFn {
// TODO: Expand supported operations
// Currently only supports single index or wildcard (1 or all)
cur := l.peek()
switch cur {
case ':':
l.take()
l.emit(pathIndexRange)
return lexPathIndexRangeSecond
case ']':
return lexPathBracketClose
default:
return l.errorf("Expected digit or ] instead of %#U", cur)
}
}
func lexPathIndexRangeSecond(l lexer, state *intStack) stateFn {
cur := l.peek()
switch cur {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeDigits(l)
l.emit(pathIndex)
return lexPathBracketClose
case ']':
return lexPathBracketClose
default:
return l.errorf("Expected digit or ] instead of %#U", cur)
}
}
func lexPathArrayClose(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != ']' {
return l.errorf("Expected ] instead of %#U", cur)
}
l.emit(pathBracketRight)
return lexPathAfterKey
}
func lexPathAfterValue(l lexer, state *intStack) stateFn {
cur := l.take()
if cur != eof {
return l.errorf("Expected EOF instead of %#U", cur)
}
l.emit(pathEOF)
return nil
}

30
path_states_test.go Normal file
View File

@ -0,0 +1,30 @@
package jsonpath
import (
"testing"
"github.com/stretchr/testify/assert"
)
var pathTests = []lexTest{
{"simple root node", `$.akey`, []int{pathRoot, pathPeriod, pathKey, pathEOF}},
{"simple current node", `@.akey`, []int{pathCurrent, pathPeriod, pathKey, pathEOF}},
{"simple root node w/ value", `$.akey+`, []int{pathRoot, pathPeriod, pathKey, pathValue, pathEOF}},
{"nested object", `$.akey.akey2`, []int{pathRoot, pathPeriod, pathKey, pathPeriod, pathKey, pathEOF}},
{"nested objects", `$.akey.akey2.akey3`, []int{pathRoot, pathPeriod, pathKey, pathPeriod, pathKey, pathPeriod, pathKey, pathEOF}},
{"quoted keys", `$.akey["akey2"].akey3`, []int{pathRoot, pathPeriod, pathKey, pathBracketLeft, pathKey, pathBracketRight, pathPeriod, pathKey, pathEOF}},
{"wildcard key", `$.akey.*.akey3`, []int{pathRoot, pathPeriod, pathKey, pathPeriod, pathWildcard, pathPeriod, pathKey, pathEOF}},
{"wildcard index", `$.akey[*]`, []int{pathRoot, pathPeriod, pathKey, pathBracketLeft, pathWildcard, pathBracketRight, pathEOF}},
{"key with where expression", `$.akey?(@.ten = 5)`, []int{pathRoot, pathPeriod, pathKey, pathWhere, pathExpression, pathEOF}},
{"bracket notation", `$["aKey"][*][32][23:42]`, []int{pathRoot, pathBracketLeft, pathKey, pathBracketRight, pathBracketLeft, pathWildcard, pathBracketRight, pathBracketLeft, pathIndex, pathBracketRight, pathBracketLeft, pathIndex, pathIndexRange, pathIndex, pathBracketRight, pathEOF}},
}
func TestValidPaths(t *testing.T) {
as := assert.New(t)
for _, test := range pathTests {
lexer := NewSliceLexer([]byte(test.input), PATH)
types := itemsToTypes(readerToArray(lexer))
as.EqualValues(types, test.tokenTypes, "Testing of %s: \nactual\n\t%+v\nexpected\n\t%v", test.name, typesDescription(types, pathTokenNames), typesDescription(test.tokenTypes, pathTokenNames))
}
}

40
path_test.go Normal file
View File

@ -0,0 +1,40 @@
package jsonpath
import (
"testing"
"github.com/stretchr/testify/assert"
)
type optest struct {
name string
path string
expected []int
}
var optests = []optest{
optest{"single key (period) ", `$.aKey`, []int{opTypeName}},
optest{"single key (bracket)", `$["aKey"]`, []int{opTypeName}},
optest{"single key (period) ", `$.*`, []int{opTypeNameWild}},
optest{"single index", `$[12]`, []int{opTypeIndex}},
optest{"single key", `$[23:45]`, []int{opTypeIndexRange}},
optest{"single key", `$[*]`, []int{opTypeIndexWild}},
optest{"double key", `$["aKey"]["bKey"]`, []int{opTypeName, opTypeName}},
optest{"double key", `$["aKey"].bKey`, []int{opTypeName, opTypeName}},
}
func TestQueryOperators(t *testing.T) {
as := assert.New(t)
for _, t := range optests {
path, err := parsePath(t.path)
as.NoError(err)
as.EqualValues(len(t.expected), len(path.operators))
for x, op := range t.expected {
as.EqualValues(pathTokenNames[op], pathTokenNames[path.operators[x].typ])
}
}
}

55
queue.go Normal file
View File

@ -0,0 +1,55 @@
package jsonpath
type Results struct {
nodes []*Result
head int
tail int
count int
}
func newResults() *Results {
return &Results{
nodes: make([]*Result, 3, 3),
}
}
func (q *Results) push(n *Result) {
if q.head == q.tail && q.count > 0 {
nodes := make([]*Result, len(q.nodes)*2, len(q.nodes)*2)
copy(nodes, q.nodes[q.head:])
copy(nodes[len(q.nodes)-q.head:], q.nodes[:q.head])
q.head = 0
q.tail = len(q.nodes)
q.nodes = nodes
}
q.nodes[q.tail] = n
q.tail = (q.tail + 1) % len(q.nodes)
q.count++
}
func (q *Results) Pop() *Result {
if q.count == 0 {
return nil
}
node := q.nodes[q.head]
q.head = (q.head + 1) % len(q.nodes)
q.count--
return node
}
func (q *Results) peek() *Result {
if q.count == 0 {
return nil
}
return q.nodes[q.head]
}
func (q *Results) len() int {
return q.count
}
func (q *Results) clear() {
q.head = 0
q.count = 0
q.tail = 0
}

57
result.go Normal file
View File

@ -0,0 +1,57 @@
package jsonpath
import (
"bytes"
"fmt"
)
const (
JsonObject = iota
JsonArray
JsonString
JsonNumber
JsonNull
JsonBool
)
type Result struct {
Keys []interface{}
Value []byte
Type int
}
func (r *Result) Pretty(showPath bool) string {
b := bytes.NewBufferString("")
printed := false
if showPath {
for _, k := range r.Keys {
switch v := k.(type) {
case int:
b.WriteString(fmt.Sprintf("%d", v))
default:
b.WriteString(fmt.Sprintf("%q", v))
}
b.WriteRune('\t')
printed = true
}
} else if r.Value == nil {
if len(r.Keys) > 0 {
printed = true
switch v := r.Keys[len(r.Keys)-1].(type) {
case int:
b.WriteString(fmt.Sprintf("%d", v))
default:
b.WriteString(fmt.Sprintf("%q", v))
}
}
}
if r.Value != nil {
printed = true
b.WriteString(fmt.Sprintf("%s", r.Value))
}
if printed {
b.WriteRune('\n')
}
return b.String()
}

27
run.go Normal file
View File

@ -0,0 +1,27 @@
package jsonpath
import "io"
func EvalPathsInBytes(input []byte, paths []*Path) (*Eval, error) {
lexer := NewSliceLexer(input, JSON)
eval := newEvaluation(lexer, paths...)
return eval, nil
}
func EvalPathsInReader(r io.Reader, paths []*Path) (*Eval, error) {
lexer := NewReaderLexer(r, JSON)
eval := newEvaluation(lexer, paths...)
return eval, nil
}
func ParsePaths(pathStrings ...string) ([]*Path, error) {
paths := make([]*Path, len(pathStrings))
for x, p := range pathStrings {
path, err := parsePath(p)
if err != nil {
return nil, err
}
paths[x] = path
}
return paths, nil
}

148
stack.go Normal file
View File

@ -0,0 +1,148 @@
package jsonpath
// Integer Stack
type intStack struct {
values []int
}
func newIntStack() *intStack {
return &intStack{
values: make([]int, 0, 100),
}
}
func (s *intStack) len() int {
return len(s.values)
}
func (s *intStack) push(r int) {
s.values = append(s.values, r)
}
func (s *intStack) pop() (int, bool) {
if s.len() == 0 {
return 0, false
}
v, _ := s.peek()
s.values = s.values[:len(s.values)-1]
return v, true
}
func (s *intStack) peek() (int, bool) {
if s.len() == 0 {
return 0, false
}
v := s.values[len(s.values)-1]
return v, true
}
func (s *intStack) clone() *intStack {
d := intStack{
values: make([]int, s.len()),
}
copy(d.values, s.values)
return &d
}
func (s *intStack) toArray() []int {
return s.values
}
// Result Stack
type resultStack struct {
values []Result
}
func newResultStack() *resultStack {
return &resultStack{
values: make([]Result, 0),
}
}
func (s *resultStack) len() int {
return len(s.values)
}
func (s *resultStack) push(r Result) {
s.values = append(s.values, r)
}
func (s *resultStack) pop() (Result, bool) {
if s.len() == 0 {
return Result{}, false
}
v, _ := s.peek()
s.values = s.values[:len(s.values)-1]
return v, true
}
func (s *resultStack) peek() (Result, bool) {
if s.len() == 0 {
return Result{}, false
}
v := s.values[len(s.values)-1]
return v, true
}
func (s *resultStack) clone() *resultStack {
d := resultStack{
values: make([]Result, s.len()),
}
copy(d.values, s.values)
return &d
}
func (s *resultStack) toArray() []Result {
return s.values
}
// Interface Stack
type stack struct {
values []interface{}
}
func newStack() *stack {
return &stack{
values: make([]interface{}, 0, 100),
}
}
func (s *stack) len() int {
return len(s.values)
}
func (s *stack) push(r interface{}) {
s.values = append(s.values, r)
}
func (s *stack) pop() (interface{}, bool) {
if s.len() == 0 {
return nil, false
}
v, _ := s.peek()
s.values = s.values[:len(s.values)-1]
return v, true
}
func (s *stack) peek() (interface{}, bool) {
if s.len() == 0 {
return nil, false
}
v := s.values[len(s.values)-1]
return v, true
}
func (s *stack) clone() *stack {
d := stack{
values: make([]interface{}, s.len()),
}
copy(d.values, s.values)
return &d
}
func (s *stack) toArray() []interface{} {
return s.values
}

56
stack_test.go Normal file
View File

@ -0,0 +1,56 @@
package jsonpath
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestStackPush(t *testing.T) {
as := assert.New(t)
s := newIntStack()
s.push(5)
as.EqualValues(s.len(), 1)
s.push(12)
as.EqualValues(s.len(), 2)
}
func TestStackPop(t *testing.T) {
as := assert.New(t)
s := newIntStack()
s.push(99)
as.EqualValues(s.len(), 1)
v, ok := s.pop()
as.True(ok)
as.EqualValues(99, v)
as.EqualValues(s.len(), 0)
}
func TestStackPeek(t *testing.T) {
as := assert.New(t)
s := newIntStack()
s.push(99)
v, ok := s.peek()
as.True(ok)
as.EqualValues(99, v)
s.push(54)
v, ok = s.peek()
as.True(ok)
as.EqualValues(54, v)
s.pop()
v, ok = s.peek()
as.True(ok)
as.EqualValues(99, v)
s.pop()
_, ok = s.peek()
as.False(ok)
}