Fix wsl linter warnings

master
Vladimir Hodakov 2019-10-19 02:14:19 +04:00
parent 1f47c5b4a2
commit 3c690ef129
Signed by: Vladimir Hodakov
GPG Key ID: 673980B6882F82C6
22 changed files with 263 additions and 29 deletions

View File

@ -7,6 +7,7 @@ linters:
- gocritic - gocritic
- funlen - funlen
- godox - godox
- gocyclo # is deprecated
linters-settings: linters-settings:
lll: lll:
line-length: 180 line-length: 180

View File

@ -326,6 +326,7 @@ func (b *exprBucket) evaluate() (bool, error) {
func itemMatchOperator(loc interface{}, op *operator) bool { func itemMatchOperator(loc interface{}, op *operator) bool {
topBytes, isKey := loc.([]byte) topBytes, isKey := loc.([]byte)
topInt, isIndex := loc.(int) topInt, isIndex := loc.(int)
if isKey { if isKey {
switch op.typ { switch op.typ {
case opTypeNameWild: case opTypeNameWild:
@ -342,5 +343,6 @@ func itemMatchOperator(loc interface{}, op *operator) bool {
return topInt >= op.indexStart && (!op.hasIndexEnd || topInt <= op.indexEnd) return topInt >= op.indexStart && (!op.hasIndexEnd || topInt <= op.indexEnd)
} }
} }
return false return false
} }

View File

@ -131,6 +131,7 @@ func evalArrayAfterOpen(e *Eval, i *Item) evalStateFn {
default: default:
e.Error = errors.New(UnexpectedToken) e.Error = errors.New(UnexpectedToken)
} }
return nil return nil
} }
@ -152,6 +153,7 @@ func evalArrayValue(e *Eval, i *Item) evalStateFn {
default: default:
e.Error = errors.New(UnexpectedToken) e.Error = errors.New(UnexpectedToken)
} }
return nil return nil
} }
@ -163,21 +165,25 @@ func evalArrayAfterValue(e *Eval, i *Item) evalStateFn {
e.prevIndex = valIndex e.prevIndex = valIndex
} }
} }
return evalArrayValue return evalArrayValue
case jsonBracketRight: case jsonBracketRight:
e.location.pop() e.location.pop()
setPrevIndex(e) setPrevIndex(e)
return rightBraceOrBracket(e) return rightBraceOrBracket(e)
case jsonError: case jsonError:
return evalError(e, i) return evalError(e, i)
default: default:
e.Error = errors.New(UnexpectedToken) e.Error = errors.New(UnexpectedToken)
} }
return nil return nil
} }
func setPrevIndex(e *Eval) { func setPrevIndex(e *Eval) {
e.prevIndex = -1 e.prevIndex = -1
peeked, ok := e.location.peek() peeked, ok := e.location.peek()
if ok { if ok {
if peekedIndex, intOk := peeked.(int); intOk { if peekedIndex, intOk := peeked.(int); intOk {
@ -194,6 +200,7 @@ func evalRootEnd(e *Eval, i *Item) evalStateFn {
e.Error = errors.New(BadStructure) e.Error = errors.New(BadStructure)
} }
} }
return nil return nil
} }

View File

@ -72,6 +72,7 @@ func TestPathQuery(t *testing.T) {
func newResult(value string, typ int, keys ...interface{}) Result { func newResult(value string, typ int, keys ...interface{}) Result {
keysChanged := make([]interface{}, len(keys)) keysChanged := make([]interface{}, len(keys))
for i, k := range keys { for i, k := range keys {
switch v := k.(type) { switch v := k.(type) {
case string: case string:
@ -90,6 +91,7 @@ func newResult(value string, typ int, keys ...interface{}) Result {
func toResultArray(e *Eval) []Result { func toResultArray(e *Eval) []Result {
vals := make([]Result, 0) vals := make([]Result, 0)
for { for {
if r, ok := e.Next(); ok { if r, ok := e.Next(); ok {
if r != nil { if r != nil {
@ -99,5 +101,6 @@ func toResultArray(e *Eval) []Result {
break break
} }
} }
return vals return vals
} }

View File

@ -53,7 +53,8 @@ var opa = map[int]struct {
// Shunting-yard Algorithm (infix -> postfix) // Shunting-yard Algorithm (infix -> postfix)
// http://rosettacode.org/wiki/Parsing/Shunting-yard_algorithm#Go // http://rosettacode.org/wiki/Parsing/Shunting-yard_algorithm#Go
func infixToPostFix(items []Item) (out []Item, err error) { func infixToPostFix(items []Item) ([]Item, error) {
out := make([]Item, 0)
stack := newStack() stack := newStack()
for _, i := range items { for _, i := range items {
@ -62,19 +63,23 @@ func infixToPostFix(items []Item) (out []Item, err error) {
stack.push(i) // push "(" to stack stack.push(i) // push "(" to stack
case exprParenRight: case exprParenRight:
found := false found := false
for { for {
// pop item ("(" or operator) from stack // pop item ("(" or operator) from stack
opInterface, ok := stack.pop() opInterface, ok := stack.pop()
if !ok { if !ok {
return nil, errors.New(exprErrorMismatchedParens) return nil, errors.New(exprErrorMismatchedParens)
} }
op := opInterface.(Item) op := opInterface.(Item)
if op.typ == exprParenLeft { if op.typ == exprParenLeft {
found = true found = true
break // discard "(" break // discard "("
} }
out = append(out, op) // add operator to result out = append(out, op) // add operator to result
} }
if !found { if !found {
return nil, errors.New(exprErrorMismatchedParens) return nil, errors.New(exprErrorMismatchedParens)
} }
@ -85,12 +90,14 @@ func infixToPostFix(items []Item) (out []Item, err error) {
// consider top item on stack // consider top item on stack
opInt, _ := stack.peek() opInt, _ := stack.peek()
op := opInt.(Item) op := opInt.(Item)
if o2, isOp := opa[op.typ]; !isOp || o1.prec > o2.prec || if o2, isOp := opa[op.typ]; !isOp || o1.prec > o2.prec ||
o1.prec == o2.prec && o1.rAssoc { o1.prec == o2.prec && o1.rAssoc {
break break
} }
// top item is an operator that needs to come off // top item is an operator that needs to come off
stack.pop() // pop it stack.pop() // pop it
out = append(out, op) // add it to result out = append(out, op) // add it to result
} }
// push operator (the new one) to stack // push operator (the new one) to stack
@ -104,14 +111,18 @@ func infixToPostFix(items []Item) (out []Item, err error) {
for stack.len() > 0 { for stack.len() > 0 {
opInt, _ := stack.pop() opInt, _ := stack.pop()
op := opInt.(Item) op := opInt.(Item)
if op.typ == exprParenLeft { if op.typ == exprParenLeft {
return nil, errors.New(exprErrorMismatchedParens) return nil, errors.New(exprErrorMismatchedParens)
} }
out = append(out, op) out = append(out, op)
} }
return
return out, nil
} }
// nolint:gocognit
func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface{}, error) { func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface{}, error) {
s := newStack() s := newStack()
@ -121,19 +132,20 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
for _, item := range postFixItems { for _, item := range postFixItems {
switch item.typ { switch item.typ {
// VALUES // VALUES
case exprBool: case exprBool:
val, err := strconv.ParseBool(string(item.val)) val, err := strconv.ParseBool(string(item.val))
if err != nil { if err != nil {
return false, fmt.Errorf(exprErrorBadValue, string(item.val), exprTokenNames[exprBool]) return false, fmt.Errorf(exprErrorBadValue, string(item.val), exprTokenNames[exprBool])
} }
s.push(val) s.push(val)
case exprNumber: case exprNumber:
val, err := strconv.ParseFloat(string(item.val), 64) val, err := strconv.ParseFloat(string(item.val), 64)
if err != nil { if err != nil {
return false, fmt.Errorf(exprErrorBadValue, string(item.val), exprTokenNames[exprNumber]) return false, fmt.Errorf(exprErrorBadValue, string(item.val), exprTokenNames[exprNumber])
} }
s.push(val) s.push(val)
case exprPath: case exprPath:
// TODO: Handle datatypes of JSON // TODO: Handle datatypes of JSON
@ -150,6 +162,7 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
if err != nil { if err != nil {
return false, fmt.Errorf(exprErrorBadValue, string(item.val), jsonTokenNames[jsonNumber]) return false, fmt.Errorf(exprErrorBadValue, string(item.val), jsonTokenNames[jsonNumber])
} }
s.push(valFloat) s.push(valFloat)
case jsonKey, jsonString: case jsonKey, jsonString:
s.push(i.val) s.push(i.val)
@ -160,8 +173,6 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
s.push(item.val) s.push(item.val)
case exprNull: case exprNull:
s.push(nil) s.push(nil)
// OPERATORS
case exprOpAnd: case exprOpAnd:
a, b, err := take2Bool(s, item.typ) a, b, err := take2Bool(s, item.typ)
if err != nil { if err != nil {
@ -188,18 +199,21 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(a == b) s.push(a == b)
case float64: case float64:
a, b, err := take2Float(s, item.typ) a, b, err := take2Float(s, item.typ)
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(a == b) s.push(a == b)
case []byte: case []byte:
a, b, err := take2ByteSlice(s, item.typ) a, b, err := take2ByteSlice(s, item.typ)
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(byteSlicesEqual(a, b)) s.push(byteSlicesEqual(a, b))
} }
case exprOpNeq: case exprOpNeq:
@ -221,18 +235,21 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(a != b) s.push(a != b)
case float64: case float64:
a, b, err := take2Float(s, item.typ) a, b, err := take2Float(s, item.typ)
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(a != b) s.push(a != b)
case []byte: case []byte:
a, b, err := take2ByteSlice(s, item.typ) a, b, err := take2ByteSlice(s, item.typ)
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(!byteSlicesEqual(a, b)) s.push(!byteSlicesEqual(a, b))
} }
case exprOpNot: case exprOpNot:
@ -282,24 +299,28 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(b + a) s.push(b + a)
case exprOpPlusUn: case exprOpPlusUn:
a, err := take1Float(s, item.typ) a, err := take1Float(s, item.typ)
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(a) s.push(a)
case exprOpMinus: case exprOpMinus:
a, b, err := take2Float(s, item.typ) a, b, err := take2Float(s, item.typ)
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(b - a) s.push(b - a)
case exprOpMinusUn: case exprOpMinusUn:
a, err := take1Float(s, item.typ) a, err := take1Float(s, item.typ)
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(0 - a) s.push(0 - a)
case exprOpSlash: case exprOpSlash:
a, b, err := take2Float(s, item.typ) a, b, err := take2Float(s, item.typ)
@ -310,6 +331,7 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
if a == 0.0 { if a == 0.0 {
return false, errors.New("cannot divide by zero") return false, errors.New("cannot divide by zero")
} }
s.push(b / a) s.push(b / a)
case exprOpStar: case exprOpStar:
a, b, err := take2Float(s, item.typ) a, b, err := take2Float(s, item.typ)
@ -337,8 +359,8 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
if err != nil { if err != nil {
return false, err return false, err
} }
s.push(!a) s.push(!a)
// Other
default: default:
return false, fmt.Errorf("token not supported in evaluator: %v", exprTokenNames[item.typ]) return false, fmt.Errorf("token not supported in evaluator: %v", exprTokenNames[item.typ])
} }
@ -349,11 +371,13 @@ func evaluatePostFix(postFixItems []Item, pathValues map[string]Item) (interface
} }
endInt, _ := s.pop() endInt, _ := s.pop()
return endInt, nil return endInt, nil
} }
func take1Bool(s *stack, op int) (bool, error) { func take1Bool(s *stack, op int) (bool, error) {
t := exprBool t := exprBool
val, ok := s.pop() val, ok := s.pop()
if !ok { if !ok {
return false, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op]) return false, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
@ -363,17 +387,20 @@ func take1Bool(s *stack, op int) (bool, error) {
if !ok { if !ok {
return false, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()} return false, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()}
} }
return b, nil return b, nil
} }
func take2Bool(s *stack, op int) (bool, bool, error) { func take2Bool(s *stack, op int) (bool, bool, error) {
a, aErr := take1Bool(s, op) a, aErr := take1Bool(s, op)
b, bErr := take1Bool(s, op) b, bErr := take1Bool(s, op)
return a, b, firstError(aErr, bErr) return a, b, firstError(aErr, bErr)
} }
func take1Float(s *stack, op int) (float64, error) { func take1Float(s *stack, op int) (float64, error) {
t := exprNumber t := exprNumber
val, ok := s.pop() val, ok := s.pop()
if !ok { if !ok {
return 0.0, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op]) return 0.0, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
@ -383,17 +410,20 @@ func take1Float(s *stack, op int) (float64, error) {
if !ok { if !ok {
return 0.0, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()} return 0.0, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()}
} }
return b, nil return b, nil
} }
func take2Float(s *stack, op int) (float64, float64, error) { func take2Float(s *stack, op int) (float64, float64, error) {
a, aErr := take1Float(s, op) a, aErr := take1Float(s, op)
b, bErr := take1Float(s, op) b, bErr := take1Float(s, op)
return a, b, firstError(aErr, bErr) return a, b, firstError(aErr, bErr)
} }
func take1ByteSlice(s *stack, op int) ([]byte, error) { func take1ByteSlice(s *stack, op int) ([]byte, error) {
t := exprNumber t := exprNumber
val, ok := s.pop() val, ok := s.pop()
if !ok { if !ok {
return nil, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op]) return nil, fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
@ -403,17 +433,20 @@ func take1ByteSlice(s *stack, op int) ([]byte, error) {
if !ok { if !ok {
return nil, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()} return nil, exprErrorBadTypeComparison{exprTokenNames[t], (reflect.TypeOf(val)).String()}
} }
return b, nil return b, nil
} }
func take2ByteSlice(s *stack, op int) ([]byte, []byte, error) { func take2ByteSlice(s *stack, op int) ([]byte, []byte, error) {
a, aErr := take1ByteSlice(s, op) a, aErr := take1ByteSlice(s, op)
b, bErr := take1ByteSlice(s, op) b, bErr := take1ByteSlice(s, op)
return a, b, firstError(aErr, bErr) return a, b, firstError(aErr, bErr)
} }
func take1Null(s *stack, op int) error { func take1Null(s *stack, op int) error {
t := exprNull t := exprNull
val, ok := s.pop() val, ok := s.pop()
if !ok { if !ok {
return fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op]) return fmt.Errorf(exprErrorNotEnoughOperands, exprTokenNames[op])
@ -422,11 +455,13 @@ func take1Null(s *stack, op int) error {
if v := reflect.TypeOf(val); v != nil { if v := reflect.TypeOf(val); v != nil {
return exprErrorBadTypeComparison{exprTokenNames[t], v.String()} return exprErrorBadTypeComparison{exprTokenNames[t], v.String()}
} }
return nil return nil
} }
func take2Null(s *stack, op int) error { func take2Null(s *stack, op int) error {
aErr := take1Null(s, op) aErr := take1Null(s, op)
bErr := take1Null(s, op) bErr := take1Null(s, op)
return firstError(aErr, bErr) return firstError(aErr, bErr)
} }

View File

@ -65,20 +65,24 @@ var exprTokenNames = map[int]string{
var EXPRESSION = lexExprText var EXPRESSION = lexExprText
func lexExprText(l lexer, state *intStack) stateFn { func lexExprText(l lexer, state *intStack) stateFn {
ignoreSpaceRun(l)
cur := l.peek()
var next stateFn var next stateFn
ignoreSpaceRun(l)
cur := l.peek()
switch cur { switch cur {
case '(': case '(':
l.take() l.take()
state.push(exprParenLeft) state.push(exprParenLeft)
l.emit(exprParenLeft) l.emit(exprParenLeft)
next = lexExprText next = lexExprText
case ')': case ')':
if top, ok := state.peek(); ok && top != exprParenLeft { if top, ok := state.peek(); ok && top != exprParenLeft {
next = l.errorf("Received %#U but has no matching (", cur) next = l.errorf("Received %#U but has no matching (", cur)
break break
} }
state.pop() state.pop()
l.take() l.take()
l.emit(exprParenRight) l.emit(exprParenRight)
@ -87,82 +91,100 @@ func lexExprText(l lexer, state *intStack) stateFn {
case '!': case '!':
l.take() l.take()
l.emit(exprOpNot) l.emit(exprOpNot)
next = lexExprText next = lexExprText
case '+': case '+':
l.take() l.take()
l.emit(exprOpPlusUn) l.emit(exprOpPlusUn)
next = lexExprText next = lexExprText
case '-': case '-':
l.take() l.take()
l.emit(exprOpMinusUn) l.emit(exprOpMinusUn)
next = lexExprText next = lexExprText
case '@': //, '$': // Only support current key location case '@': //, '$': // Only support current key location
l.take() l.take()
takePath(l) takePath(l)
l.emit(exprPath) l.emit(exprPath)
next = lexOneValue next = lexOneValue
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeNumeric(l) takeNumeric(l)
l.emit(exprNumber) l.emit(exprNumber)
next = lexOneValue next = lexOneValue
case 't': case 't':
takeExactSequence(l, bytesTrue) takeExactSequence(l, bytesTrue)
l.emit(exprBool) l.emit(exprBool)
next = lexOneValue next = lexOneValue
case 'f': case 'f':
takeExactSequence(l, bytesFalse) takeExactSequence(l, bytesFalse)
l.emit(exprBool) l.emit(exprBool)
next = lexOneValue next = lexOneValue
case 'n': case 'n':
takeExactSequence(l, bytesNull) takeExactSequence(l, bytesNull)
l.emit(exprNull) l.emit(exprNull)
next = lexOneValue next = lexOneValue
case '"': case '"':
err := l.takeString() err := l.takeString()
if err != nil { if err != nil {
return l.errorf("Could not take string because %q", err) return l.errorf("Could not take string because %q", err)
} }
l.emit(exprString) l.emit(exprString)
next = lexOneValue next = lexOneValue
case eof: case eof:
l.emit(exprEOF) l.emit(exprEOF) // next = nil
// next = nil
default: default:
return l.errorf("Unrecognized sequence in expression: %#U", cur) return l.errorf("Unrecognized sequence in expression: %#U", cur)
} }
return next return next
} }
func lexOneValue(l lexer, state *intStack) stateFn { func lexOneValue(l lexer, state *intStack) stateFn {
var next stateFn var next stateFn
cur := l.peek() cur := l.peek()
switch cur { switch cur {
case '+': case '+':
l.take() l.take()
l.emit(exprOpPlus) l.emit(exprOpPlus)
next = lexExprText next = lexExprText
case '-': case '-':
l.take() l.take()
l.emit(exprOpMinus) l.emit(exprOpMinus)
next = lexExprText next = lexExprText
case '*': case '*':
l.take() l.take()
l.emit(exprOpStar) l.emit(exprOpStar)
next = lexExprText next = lexExprText
case '/': case '/':
l.take() l.take()
l.emit(exprOpSlash) l.emit(exprOpSlash)
next = lexExprText next = lexExprText
case '%': case '%':
l.take() l.take()
l.emit(exprOpPercent) l.emit(exprOpPercent)
next = lexExprText next = lexExprText
case '^': case '^':
l.take() l.take()
l.emit(exprOpHat) l.emit(exprOpHat)
next = lexExprText next = lexExprText
case '<': case '<':
l.take() l.take()
cur = l.peek() cur = l.peek()
if cur == '=' { if cur == '=' {
l.take() l.take()
@ -170,9 +192,11 @@ func lexOneValue(l lexer, state *intStack) stateFn {
} else { } else {
l.emit(exprOpLt) l.emit(exprOpLt)
} }
next = lexExprText next = lexExprText
case '>': case '>':
l.take() l.take()
cur = l.peek() cur = l.peek()
if cur == '=' { if cur == '=' {
l.take() l.take()
@ -180,44 +204,58 @@ func lexOneValue(l lexer, state *intStack) stateFn {
} else { } else {
l.emit(exprOpGt) l.emit(exprOpGt)
} }
next = lexExprText next = lexExprText
case '&': case '&':
l.take() l.take()
cur = l.take() cur = l.take()
if cur != '&' { if cur != '&' {
return l.errorf("Expected double & instead of %#U", cur) return l.errorf("expected double & instead of %#U", cur)
} }
l.emit(exprOpAnd) l.emit(exprOpAnd)
next = lexExprText next = lexExprText
case '|': case '|':
l.take() l.take()
cur = l.take() cur = l.take()
if cur != '|' { if cur != '|' {
return l.errorf("Expected double | instead of %#U", cur) return l.errorf("expected double | instead of %#U", cur)
} }
l.emit(exprOpOr) l.emit(exprOpOr)
next = lexExprText next = lexExprText
case '=': case '=':
l.take() l.take()
cur = l.take() cur = l.take()
if cur != '=' { if cur != '=' {
return l.errorf("Expected double = instead of %#U", cur) return l.errorf("expected double = instead of %#U", cur)
} }
l.emit(exprOpEq) l.emit(exprOpEq)
next = lexExprText next = lexExprText
case '!': case '!':
l.take() l.take()
cur = l.take() cur = l.take()
if cur != '=' { if cur != '=' {
return l.errorf("Expected = for != instead of %#U", cur) return l.errorf("expected = for != instead of %#U", cur)
} }
l.emit(exprOpNeq) l.emit(exprOpNeq)
next = lexExprText next = lexExprText
case ')': case ')':
if top, ok := state.peek(); ok && top != exprParenLeft { if top, ok := state.peek(); ok && top != exprParenLeft {
next = l.errorf("Received %#U but has no matching (", cur) next = l.errorf("received %#U but has no matching (", cur)
break break
} }
state.pop() state.pop()
l.take() l.take()
l.emit(exprParenRight) l.emit(exprParenRight)
@ -226,19 +264,23 @@ func lexOneValue(l lexer, state *intStack) stateFn {
case eof: case eof:
l.emit(exprEOF) l.emit(exprEOF)
default: default:
return l.errorf("Unrecognized sequence in expression: %#U", cur) return l.errorf("unrecognized sequence in expression: %#U", cur)
} }
return next return next
} }
func takeNumeric(l lexer) { func takeNumeric(l lexer) {
takeDigits(l) takeDigits(l)
if l.peek() == '.' { if l.peek() == '.' {
l.take() l.take()
takeDigits(l) takeDigits(l)
} }
if l.peek() == 'e' || l.peek() == 'E' { if l.peek() == 'e' || l.peek() == 'E' {
l.take() l.take()
if l.peek() == '+' || l.peek() == '-' { if l.peek() == '+' || l.peek() == '-' {
l.take() l.take()
takeDigits(l) takeDigits(l)

View File

@ -24,6 +24,7 @@ var expressionTests = []lexTest{
func TestExpressionTokens(t *testing.T) { func TestExpressionTokens(t *testing.T) {
as := assert.New(t) as := assert.New(t)
for _, test := range expressionTests { for _, test := range expressionTests {
lexer := NewSliceLexer([]byte(test.input), EXPRESSION) lexer := NewSliceLexer([]byte(test.input), EXPRESSION)
items := readerToArray(lexer) items := readerToArray(lexer)

View File

@ -150,6 +150,7 @@ func TestExpressions(t *testing.T) {
// trim EOF // trim EOF
items = items[0 : len(items)-1] items = items[0 : len(items)-1]
itemsPost, err := infixToPostFix(items) itemsPost, err := infixToPostFix(items)
if as.NoError(err, "Could not transform to postfix\nTest: %q", test.input) { if as.NoError(err, "Could not transform to postfix\nTest: %q", test.input) {
val, err := evaluatePostFix(itemsPost, test.fields) val, err := evaluatePostFix(itemsPost, test.fields)
if as.NoError(err, "Could not evaluate postfix\nTest Input: %q\nTest Values:%q\nError:%q", test.input, test.fields, err) { if as.NoError(err, "Could not evaluate postfix\nTest Input: %q\nTest Values:%q\nError:%q", test.input, test.fields, err) {
@ -210,17 +211,18 @@ func TestBadExpressions(t *testing.T) {
items := readerToArray(lexer) items := readerToArray(lexer)
// trim EOF // trim EOF
items = items[0 : len(items)-1] items = items[0 : len(items)-1]
itemsPost, err := infixToPostFix(items) itemsPost, err := infixToPostFix(items)
if err != nil { if err != nil {
as.True(strings.Contains(err.Error(), test.expectedErrorSubstring), "Test Input: %q\nError %q does not contain %q", test.input, err.Error(), test.expectedErrorSubstring) as.True(strings.Contains(err.Error(), test.expectedErrorSubstring), "Test Input: %q\nError %q does not contain %q", test.input, err.Error(), test.expectedErrorSubstring)
continue continue
} }
if as.NoError(err, "Could not transform to postfix\nTest: %q", test.input) { if as.NoError(err, "Could not transform to postfix\nTest: %q", test.input) {
_, err := evaluatePostFix(itemsPost, test.fields) _, err := evaluatePostFix(itemsPost, test.fields)
if as.Error(err, "Could not evaluate postfix\nTest Input: %q\nTest Values:%q\nError:%s", test.input, test.fields, err) { if as.Error(err, "Could not evaluate postfix\nTest Input: %q\nTest Values:%q\nError:%s", test.input, test.fields, err) {
as.True(strings.Contains(err.Error(), test.expectedErrorSubstring), "Test Input: %q\nError %s does not contain %q", test.input, err.Error(), test.expectedErrorSubstring) as.True(strings.Contains(err.Error(), test.expectedErrorSubstring), "Test Input: %q\nError %s does not contain %q", test.input, err.Error(), test.expectedErrorSubstring)
} }
} }
} }
} }

View File

@ -41,11 +41,11 @@ var jsonTokenNames = map[int]string{
var JSON = lexJSONRoot var JSON = lexJSONRoot
func lexJSONRoot(l lexer, state *intStack) stateFn { func lexJSONRoot(l lexer, state *intStack) stateFn {
ignoreSpaceRun(l)
cur := l.peek()
var next stateFn var next stateFn
ignoreSpaceRun(l)
cur := l.peek()
switch cur { switch cur {
case '{': case '{':
next = stateJSONObjectOpen next = stateJSONObjectOpen
@ -54,6 +54,7 @@ func lexJSONRoot(l lexer, state *intStack) stateFn {
default: default:
next = l.errorf("Expected '{' or '[' at root of JSON instead of %#U", cur) next = l.errorf("Expected '{' or '[' at root of JSON instead of %#U", cur)
} }
return next return next
} }
@ -62,6 +63,7 @@ func stateJSONObjectOpen(l lexer, state *intStack) stateFn {
if cur != '{' { if cur != '{' {
return l.errorf("Expected '{' as start of object instead of %#U", cur) return l.errorf("Expected '{' as start of object instead of %#U", cur)
} }
l.emit(jsonBraceLeft) l.emit(jsonBraceLeft)
state.push(jsonBraceLeft) state.push(jsonBraceLeft)
@ -73,6 +75,7 @@ func stateJSONArrayOpen(l lexer, state *intStack) stateFn {
if cur != '[' { if cur != '[' {
return l.errorf("Expected '[' as start of array instead of %#U", cur) return l.errorf("Expected '[' as start of array instead of %#U", cur)
} }
l.emit(jsonBracketLeft) l.emit(jsonBracketLeft)
state.push(jsonBracketLeft) state.push(jsonBracketLeft)
@ -81,6 +84,7 @@ func stateJSONArrayOpen(l lexer, state *intStack) stateFn {
func stateJSONObject(l lexer, state *intStack) stateFn { func stateJSONObject(l lexer, state *intStack) stateFn {
var next stateFn var next stateFn
cur := l.peek() cur := l.peek()
switch cur { switch cur {
case '}': case '}':
@ -88,20 +92,24 @@ func stateJSONObject(l lexer, state *intStack) stateFn {
next = l.errorf("Received %#U but has no matching '{'", cur) next = l.errorf("Received %#U but has no matching '{'", cur)
break break
} }
l.take() l.take()
l.emit(jsonBraceRight) l.emit(jsonBraceRight)
state.pop() state.pop()
next = stateJSONAfterValue next = stateJSONAfterValue
case '"': case '"':
next = stateJSONKey next = stateJSONKey
default: default:
next = l.errorf("Expected '}' or \" within an object instead of %#U", cur) next = l.errorf("Expected '}' or \" within an object instead of %#U", cur)
} }
return next return next
} }
func stateJSONArray(l lexer, state *intStack) stateFn { func stateJSONArray(l lexer, state *intStack) stateFn {
var next stateFn var next stateFn
cur := l.peek() cur := l.peek()
switch cur { switch cur {
case ']': case ']':
@ -109,20 +117,24 @@ func stateJSONArray(l lexer, state *intStack) stateFn {
next = l.errorf("Received %#U but has no matching '['", cur) next = l.errorf("Received %#U but has no matching '['", cur)
break break
} }
l.take() l.take()
l.emit(jsonBracketRight) l.emit(jsonBracketRight)
state.pop() state.pop()
next = stateJSONAfterValue next = stateJSONAfterValue
default: default:
next = stateJSONValue next = stateJSONValue
} }
return next return next
} }
func stateJSONAfterValue(l lexer, state *intStack) stateFn { func stateJSONAfterValue(l lexer, state *intStack) stateFn {
cur := l.take() cur := l.take()
top, ok := state.peek()
topVal := noValue topVal := noValue
top, ok := state.peek()
if ok { if ok {
topVal = top topVal = top
} }
@ -130,6 +142,7 @@ func stateJSONAfterValue(l lexer, state *intStack) stateFn {
switch cur { switch cur {
case ',': case ',':
l.emit(jsonComma) l.emit(jsonComma)
switch topVal { switch topVal {
case jsonBraceLeft: case jsonBraceLeft:
return stateJSONKey return stateJSONKey
@ -143,6 +156,7 @@ func stateJSONAfterValue(l lexer, state *intStack) stateFn {
case '}': case '}':
l.emit(jsonBraceRight) l.emit(jsonBraceRight)
state.pop() state.pop()
switch topVal { switch topVal {
case jsonBraceLeft: case jsonBraceLeft:
return stateJSONAfterValue return stateJSONAfterValue
@ -154,6 +168,7 @@ func stateJSONAfterValue(l lexer, state *intStack) stateFn {
case ']': case ']':
l.emit(jsonBracketRight) l.emit(jsonBracketRight)
state.pop() state.pop()
switch topVal { switch topVal {
case jsonBraceLeft: case jsonBraceLeft:
return l.errorf("unexpected %#U in object", cur) return l.errorf("unexpected %#U in object", cur)
@ -172,6 +187,7 @@ func stateJSONAfterValue(l lexer, state *intStack) stateFn {
default: default:
return l.errorf("unexpected character after json value token: %#U", cur) return l.errorf("unexpected character after json value token: %#U", cur)
} }
return nil return nil
} }
@ -179,6 +195,7 @@ func stateJSONKey(l lexer, state *intStack) stateFn {
if err := l.takeString(); err != nil { if err := l.takeString(); err != nil {
return l.errorf(err.Error()) return l.errorf(err.Error())
} }
l.emit(jsonKey) l.emit(jsonKey)
return stateJSONColon return stateJSONColon
@ -187,8 +204,9 @@ func stateJSONKey(l lexer, state *intStack) stateFn {
func stateJSONColon(l lexer, state *intStack) stateFn { func stateJSONColon(l lexer, state *intStack) stateFn {
cur := l.take() cur := l.take()
if cur != ':' { if cur != ':' {
return l.errorf("Expected ':' after key string instead of %#U", cur) return l.errorf("expected ':' after key string instead of %#U", cur)
} }
l.emit(jsonColon) l.emit(jsonColon)
return stateJSONValue return stateJSONValue
@ -213,7 +231,7 @@ func stateJSONValue(l lexer, state *intStack) stateFn {
case '[': case '[':
return stateJSONArrayOpen return stateJSONArrayOpen
default: default:
return l.errorf("Unexpected character as start of value: %#U", cur) return l.errorf("unexpected character as start of value: %#U", cur)
} }
} }
@ -221,7 +239,9 @@ func stateJSONString(l lexer, state *intStack) stateFn {
if err := l.takeString(); err != nil { if err := l.takeString(); err != nil {
return l.errorf(err.Error()) return l.errorf(err.Error())
} }
l.emit(jsonString) l.emit(jsonString)
return stateJSONAfterValue return stateJSONAfterValue
} }
@ -229,13 +249,16 @@ func stateJSONNumber(l lexer, state *intStack) stateFn {
if err := takeJSONNumeric(l); err != nil { if err := takeJSONNumeric(l); err != nil {
return l.errorf(err.Error()) return l.errorf(err.Error())
} }
l.emit(jsonNumber) l.emit(jsonNumber)
return stateJSONAfterValue return stateJSONAfterValue
} }
func stateJSONBool(l lexer, state *intStack) stateFn { func stateJSONBool(l lexer, state *intStack) stateFn {
cur := l.peek()
var match []byte var match []byte
cur := l.peek()
switch cur { switch cur {
case 't': case 't':
match = trueBytes match = trueBytes
@ -246,7 +269,9 @@ func stateJSONBool(l lexer, state *intStack) stateFn {
if !takeExactSequence(l, match) { if !takeExactSequence(l, match) {
return l.errorf("Could not parse value as 'true' or 'false'") return l.errorf("Could not parse value as 'true' or 'false'")
} }
l.emit(jsonBool) l.emit(jsonBool)
return stateJSONAfterValue return stateJSONAfterValue
} }
@ -254,7 +279,9 @@ func stateJSONNull(l lexer, state *intStack) stateFn {
if !takeExactSequence(l, nullBytes) { if !takeExactSequence(l, nullBytes) {
return l.errorf("Could not parse value as 'null'") return l.errorf("Could not parse value as 'null'")
} }
l.emit(jsonNull) l.emit(jsonNull)
return stateJSONAfterValue return stateJSONAfterValue
} }
@ -263,6 +290,8 @@ func stateJSONAfterRoot(l lexer, state *intStack) stateFn {
if cur != eof { if cur != eof {
return l.errorf("Expected EOF instead of %#U", cur) return l.errorf("Expected EOF instead of %#U", cur)
} }
l.emit(jsonEOF) l.emit(jsonEOF)
return nil return nil
} }

View File

@ -63,6 +63,7 @@ func itemsToTypes(items []Item) []int {
for i, item := range items { for i, item := range items {
types[i] = item.typ types[i] = item.typ
} }
return types return types
} }
@ -161,6 +162,7 @@ var examples = []string{
func TestMixedCaseJSON(t *testing.T) { func TestMixedCaseJSON(t *testing.T) {
as := assert.New(t) as := assert.New(t)
for _, json := range examples { for _, json := range examples {
lexer := NewSliceLexer([]byte(json), JSON) lexer := NewSliceLexer([]byte(json), JSON)
items := readerToArray(lexer) items := readerToArray(lexer)

View File

@ -56,5 +56,6 @@ func typesDescription(types []int, nameMap map[int]string) []string {
for i, val := range types { for i, val := range types {
vals[i] = nameMap[val] vals[i] = nameMap[val]
} }
return vals return vals
} }

View File

@ -25,6 +25,7 @@ func NewReaderLexer(rr io.Reader, initial stateFn) *ReaderLexer {
lex: newLex(initial), lex: newLex(initial),
lexeme: bytes.NewBuffer(make([]byte, 0, 100)), lexeme: bytes.NewBuffer(make([]byte, 0, 100)),
} }
return &l return &l
} }
@ -36,6 +37,7 @@ func (l *ReaderLexer) take() int {
nr := l.nextByte nr := l.nextByte
l.nextByte = noValue l.nextByte = noValue
l.lexeme.WriteByte(byte(nr)) l.lexeme.WriteByte(byte(nr))
return nr return nr
} }
@ -52,6 +54,7 @@ looper:
if err == io.EOF { if err == io.EOF {
return errors.New("unexpected EOF in string") return errors.New("unexpected EOF in string")
} }
l.lexeme.WriteByte(curByte) l.lexeme.WriteByte(curByte)
if curByte == '"' { if curByte == '"' {
@ -62,12 +65,14 @@ looper:
if err == io.EOF { if err == io.EOF {
return errors.New("unexpected EOF in string") return errors.New("unexpected EOF in string")
} }
l.lexeme.WriteByte(curByte) l.lexeme.WriteByte(curByte)
} }
} }
previous = curByte previous = curByte
} }
return nil return nil
} }
@ -83,14 +88,16 @@ func (l *ReaderLexer) peek() int {
} }
l.nextByte = int(r) l.nextByte = int(r)
return l.nextByte return l.nextByte
} }
func (l *ReaderLexer) emit(t int) { func (l *ReaderLexer) emit(t int) {
l.setItem(t, l.pos, l.lexeme.Bytes()) l.setItem(t, l.pos, l.lexeme.Bytes())
l.pos += Pos(l.lexeme.Len())
l.hasItem = true l.hasItem = true
l.pos += Pos(l.lexeme.Len())
if t == lexEOF { if t == lexEOF {
// Do not capture eof character to match slice_lexer // Do not capture eof character to match slice_lexer
l.item.val = []byte{} l.item.val = []byte{}
@ -105,6 +112,7 @@ func (l *ReaderLexer) emit(t int) {
for l.nextByte != eof { for l.nextByte != eof {
if l.nextByte == ' ' || l.nextByte == '\t' || l.nextByte == '\r' || l.nextByte == '\n' { if l.nextByte == ' ' || l.nextByte == '\t' || l.nextByte == '\r' || l.nextByte == '\n' {
l.pos++ l.pos++
r, err := l.bufInput.ReadByte() r, err := l.bufInput.ReadByte()
if err == io.EOF { if err == io.EOF {
l.nextByte = eof l.nextByte = eof
@ -130,6 +138,7 @@ func (l *ReaderLexer) ignore() {
func (l *ReaderLexer) next() (*Item, bool) { func (l *ReaderLexer) next() (*Item, bool) {
l.lexeme.Reset() l.lexeme.Reset()
for { for {
if l.currentStateFn == nil { if l.currentStateFn == nil {
break break
@ -142,19 +151,23 @@ func (l *ReaderLexer) next() (*Item, bool) {
return &l.item, true return &l.item, true
} }
} }
return &l.item, false return &l.item, false
} }
func (l *ReaderLexer) errorf(format string, args ...interface{}) stateFn { func (l *ReaderLexer) errorf(format string, args ...interface{}) stateFn {
l.setItem(lexError, l.pos, []byte(fmt.Sprintf(format, args...))) l.setItem(lexError, l.pos, []byte(fmt.Sprintf(format, args...)))
l.lexeme.Truncate(0) l.lexeme.Truncate(0)
l.hasItem = true l.hasItem = true
return nil return nil
} }
func (l *ReaderLexer) reset() { func (l *ReaderLexer) reset() {
l.bufInput.Reset(l.input) l.bufInput.Reset(l.input)
l.lexeme.Reset() l.lexeme.Reset()
l.nextByte = noValue l.nextByte = noValue
l.pos = 0 l.pos = 0
l.lex = newLex(l.initialState) l.lex = newLex(l.initialState)

View File

@ -17,6 +17,7 @@ func NewSliceLexer(input []byte, initial stateFn) *SliceLexer {
lex: newLex(initial), lex: newLex(initial),
input: input, input: input,
} }
return l return l
} }
@ -24,8 +25,10 @@ func (l *SliceLexer) take() int {
if int(l.pos) >= len(l.input) { if int(l.pos) >= len(l.input) {
return eof return eof
} }
r := int(l.input[l.pos]) r := int(l.input[l.pos])
l.pos++ l.pos++
return r return r
} }
@ -39,6 +42,7 @@ func (l *SliceLexer) takeString() error {
cur := int(l.input[curPos]) cur := int(l.input[curPos])
curPos++ curPos++
if cur != '"' { if cur != '"' {
l.pos = curPos l.pos = curPos
return fmt.Errorf("expected \" as start of string instead of %#U", cur) return fmt.Errorf("expected \" as start of string instead of %#U", cur)
@ -63,7 +67,9 @@ looper:
previous = cur previous = cur
} }
l.pos = curPos l.pos = curPos
return nil return nil
} }
@ -71,11 +77,13 @@ func (l *SliceLexer) peek() int {
if int(l.pos) >= len(l.input) { if int(l.pos) >= len(l.input) {
return eof return eof
} }
return int(l.input[l.pos]) return int(l.input[l.pos])
} }
func (l *SliceLexer) emit(t int) { func (l *SliceLexer) emit(t int) {
l.setItem(t, l.start, l.input[l.start:l.pos]) l.setItem(t, l.start, l.input[l.start:l.pos])
l.hasItem = true l.hasItem = true
// Ignore whitespace after this token // Ignore whitespace after this token
@ -114,13 +122,16 @@ func (l *SliceLexer) next() (*Item, bool) {
return &l.item, true return &l.item, true
} }
} }
return &l.item, false return &l.item, false
} }
func (l *SliceLexer) errorf(format string, args ...interface{}) stateFn { func (l *SliceLexer) errorf(format string, args ...interface{}) stateFn {
l.setItem(lexError, l.start, []byte(fmt.Sprintf(format, args...))) l.setItem(lexError, l.start, []byte(fmt.Sprintf(format, args...)))
l.start = l.pos l.start = l.pos
l.hasItem = true l.hasItem = true
return nil return nil
} }

View File

@ -14,6 +14,7 @@ import (
func testLexerMethods(l lexer, as *assert.Assertions) { func testLexerMethods(l lexer, as *assert.Assertions) {
s := l.peek() s := l.peek()
as.EqualValues('{', s, "First rune should match") as.EqualValues('{', s, "First rune should match")
r := l.take() r := l.take()
as.EqualValues('{', r, "First rune should match") as.EqualValues('{', r, "First rune should match")
r = l.take() r = l.take()
@ -114,6 +115,7 @@ func TestReaderLexerReset(t *testing.T) {
lexer.reset() lexer.reset()
reader.Seek(0, 0) reader.Seek(0, 0)
ritems2 := readerToArray(lexer) ritems2 := readerToArray(lexer)
as.EqualValues(ritems, ritems2, "Item slices are not equal") as.EqualValues(ritems, ritems2, "Item slices are not equal")
@ -133,6 +135,7 @@ func TestLexersAgainstEachOther(t *testing.T) {
func TestLargeJSON(t *testing.T) { func TestLargeJSON(t *testing.T) {
as := assert.New(t) as := assert.New(t)
input, err := ioutil.ReadFile("large.test") input, err := ioutil.ReadFile("large.test")
if err != nil { if err != nil {
t.SkipNow() t.SkipNow()
@ -140,12 +143,16 @@ func TestLargeJSON(t *testing.T) {
} }
lexer := NewSliceLexer(input, JSON) lexer := NewSliceLexer(input, JSON)
for { for {
i, ok := lexer.next() i, ok := lexer.next()
if i.typ == jsonError { if i.typ == jsonError {
as.Fail(string(i.val)) as.Fail(string(i.val))
} }
_ = i _ = i
if !ok { if !ok {
break break
} }
@ -154,7 +161,9 @@ func TestLargeJSON(t *testing.T) {
func benchmarkBytesLexer(input []byte, b *testing.B) { func benchmarkBytesLexer(input []byte, b *testing.B) {
lexer := NewSliceLexer(input, JSON) lexer := NewSliceLexer(input, JSON)
b.ResetTimer() b.ResetTimer()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
for { for {
_, ok := lexer.next() _, ok := lexer.next()
@ -169,7 +178,9 @@ func benchmarkBytesLexer(input []byte, b *testing.B) {
func benchmarkReaderLexer(input []byte, b *testing.B) { func benchmarkReaderLexer(input []byte, b *testing.B) {
reader := bytes.NewReader(input) reader := bytes.NewReader(input)
lexer := NewReaderLexer(reader, JSON) lexer := NewReaderLexer(reader, JSON)
b.ResetTimer() b.ResetTimer()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
for { for {
_, ok := lexer.next() _, ok := lexer.next()
@ -185,9 +196,12 @@ func benchmarkReaderLexer(input []byte, b *testing.B) {
func benchmarkStdLibDecode(input []byte, b *testing.B) { func benchmarkStdLibDecode(input []byte, b *testing.B) {
reader := bytes.NewReader(input) reader := bytes.NewReader(input)
dec := json.NewDecoder(reader) dec := json.NewDecoder(reader)
b.ResetTimer() b.ResetTimer()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
var x struct{} var x struct{}
dec.Decode(&x) dec.Decode(&x)
reader.Seek(0, 0) reader.Seek(0, 0)
} }
@ -196,8 +210,10 @@ func benchmarkStdLibDecode(input []byte, b *testing.B) {
// Not comparable to previous benchmarks // Not comparable to previous benchmarks
func benchmarkStdUnmarshal(input []byte, b *testing.B) { func benchmarkStdUnmarshal(input []byte, b *testing.B) {
b.ResetTimer() b.ResetTimer()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
var x interface{} var x interface{}
err := json.Unmarshal(input, &x) err := json.Unmarshal(input, &x)
if err != nil { if err != nil {
b.Error(err) b.Error(err)
@ -210,19 +226,24 @@ func BenchmarkStdUnmarshalLarge(b *testing.B) {
if err != nil { if err != nil {
b.SkipNow() b.SkipNow()
} }
benchmarkStdUnmarshal(input, b) benchmarkStdUnmarshal(input, b)
} }
func BenchmarkStdLibDecodeLarge(b *testing.B) { func BenchmarkStdLibDecodeLarge(b *testing.B) {
input, err := ioutil.ReadFile("large.test") input, err := ioutil.ReadFile("large.test")
reader := bytes.NewReader(input)
if err != nil { if err != nil {
b.SkipNow() b.SkipNow()
} }
reader := bytes.NewReader(input)
dec := json.NewDecoder(reader) dec := json.NewDecoder(reader)
b.ResetTimer() b.ResetTimer()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
var x struct{} var x struct{}
dec.Decode(&x) dec.Decode(&x)
reader.Seek(0, 0) reader.Seek(0, 0)
} }
@ -233,6 +254,7 @@ func BenchmarkSliceLexerLarge(b *testing.B) {
if err != nil { if err != nil {
b.SkipNow() b.SkipNow()
} }
benchmarkBytesLexer(input, b) benchmarkBytesLexer(input, b)
} }
@ -244,7 +266,9 @@ func BenchmarkReaderLexerLarge(b *testing.B) {
// reader := io.NewReader(input) // reader := io.NewReader(input)
// reader, _ := os.Open("large.test") // reader, _ := os.Open("large.test")
lexer := NewReaderLexer(input, JSON) lexer := NewReaderLexer(input, JSON)
b.ResetTimer() b.ResetTimer()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
for { for {
_, ok := lexer.next() _, ok := lexer.next()

15
misc.go
View File

@ -10,7 +10,9 @@ func takeExponent(l lexer) error {
if r != 'e' && r != 'E' { if r != 'e' && r != 'E' {
return nil return nil
} }
l.take() l.take()
r = l.take() r = l.take()
switch r { switch r {
case '+', '-': case '+', '-':
@ -18,12 +20,14 @@ func takeExponent(l lexer) error {
if d := l.peek(); !(d >= '0' && d <= '9') { if d := l.peek(); !(d >= '0' && d <= '9') {
return fmt.Errorf("expected digit after numeric sign instead of %#U", d) return fmt.Errorf("expected digit after numeric sign instead of %#U", d)
} }
takeDigits(l) takeDigits(l)
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeDigits(l) takeDigits(l)
default: default:
return fmt.Errorf("expected digit after 'e' instead of %#U", r) return fmt.Errorf("expected digit after 'e' instead of %#U", r)
} }
return nil return nil
} }
@ -35,6 +39,7 @@ func takeJSONNumeric(l lexer) error {
if d := l.peek(); !(d >= '0' && d <= '9') { if d := l.peek(); !(d >= '0' && d <= '9') {
return fmt.Errorf("expected digit after dash instead of %#U", d) return fmt.Errorf("expected digit after dash instead of %#U", d)
} }
takeDigits(l) takeDigits(l)
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeDigits(l) takeDigits(l)
@ -51,7 +56,9 @@ func takeJSONNumeric(l lexer) error {
if d := l.peek(); !(d >= '0' && d <= '9') { if d := l.peek(); !(d >= '0' && d <= '9') {
return fmt.Errorf("expected digit after '.' instead of %#U", d) return fmt.Errorf("expected digit after '.' instead of %#U", d)
} }
takeDigits(l) takeDigits(l)
if err := takeExponent(l); err != nil { if err := takeExponent(l); err != nil {
return err return err
} }
@ -96,22 +103,27 @@ func takeExactSequence(l lexer, str []byte) bool {
return false return false
} }
} }
return true return true
} }
func readerToArray(tr tokenReader) []Item { func readerToArray(tr tokenReader) []Item {
vals := make([]Item, 0) vals := make([]Item, 0)
for { for {
i, ok := tr.next() i, ok := tr.next()
if !ok { if !ok {
break break
} }
v := *i v := *i
s := make([]byte, len(v.val)) s := make([]byte, len(v.val))
copy(s, v.val) copy(s, v.val)
v.val = s v.val = s
vals = append(vals, v) vals = append(vals, v)
} }
return vals return vals
} }
@ -121,6 +133,7 @@ func findErrors(items []Item) (Item, bool) {
return i, true return i, true
} }
} }
return Item{}, false return Item{}, false
} }
@ -144,6 +157,7 @@ func firstError(errors ...error) error {
return e return e
} }
} }
return nil return nil
} }
@ -154,6 +168,7 @@ func abs(x int) int {
case x == 0: case x == 0:
return 0 // return correctly abs(-0) return 0 // return correctly abs(-0)
} }
return x return x
} }

View File

@ -49,9 +49,11 @@ func genIndexKey(tr tokenReader) (*operator, error) {
case pathWildcard: case pathWildcard:
k.typ = opTypeIndexWild k.typ = opTypeIndexWild
k.indexStart = 0 k.indexStart = 0
if t, ok = tr.next(); !ok { if t, ok = tr.next(); !ok {
return nil, errors.New("expected ] after *, but got none") return nil, errors.New("expected ] after *, but got none")
} }
if t.typ != pathBracketRight { if t.typ != pathBracketRight {
return nil, fmt.Errorf("expected ] after * instead of %q", t.val) return nil, fmt.Errorf("expected ] after * instead of %q", t.val)
} }
@ -196,8 +198,8 @@ func tokensToOperators(tr tokenReader) (*Path, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
q.operators = append(q.operators, k)
q.operators = append(q.operators, k)
case pathKey: case pathKey:
keyName := p.val keyName := p.val
@ -220,7 +222,6 @@ func tokensToOperators(tr tokenReader) (*Path, error) {
) )
case pathWildcard: case pathWildcard:
q.operators = append(q.operators, &operator{typ: opTypeNameWild}) q.operators = append(q.operators, &operator{typ: opTypeNameWild})
case pathValue: case pathValue:
q.captureEndValue = true q.captureEndValue = true
case pathWhere: case pathWhere:

View File

@ -44,6 +44,7 @@ var PATH = lexPathStart
func lexPathStart(l lexer, state *intStack) stateFn { func lexPathStart(l lexer, state *intStack) stateFn {
ignoreSpaceRun(l) ignoreSpaceRun(l)
cur := l.take() cur := l.take()
switch cur { switch cur {
case '$': case '$':
@ -77,6 +78,7 @@ func lexPathAfterKey(l lexer, state *intStack) stateFn {
default: default:
return l.errorf("Unrecognized rune after path element %#U", cur) return l.errorf("Unrecognized rune after path element %#U", cur)
} }
return nil return nil
} }
@ -87,6 +89,7 @@ func lexPathExpression(l lexer, state *intStack) stateFn {
} }
parenLeftCount := 1 parenLeftCount := 1
for { for {
cur = l.take() cur = l.take()
switch cur { switch cur {
@ -103,6 +106,7 @@ func lexPathExpression(l lexer, state *intStack) stateFn {
} }
} }
l.emit(pathExpression) l.emit(pathExpression)
return lexPathAfterKey return lexPathAfterKey
} }
@ -111,19 +115,23 @@ func lexPathBracketOpen(l lexer, state *intStack) stateFn {
case '*': case '*':
l.take() l.take()
l.emit(pathWildcard) l.emit(pathWildcard)
return lexPathBracketClose return lexPathBracketClose
case '"': case '"':
l.takeString() l.takeString()
l.emit(pathKey) l.emit(pathKey)
return lexPathBracketClose return lexPathBracketClose
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
l.take() l.take()
takeDigits(l) takeDigits(l)
l.emit(pathIndex) l.emit(pathIndex)
return lexPathIndexRange return lexPathIndexRange
case eof: case eof:
l.emit(pathEOF) l.emit(pathEOF)
} }
return nil return nil
} }
@ -132,7 +140,9 @@ func lexPathBracketClose(l lexer, state *intStack) stateFn {
if cur != ']' { if cur != ']' {
return l.errorf("Expected ] instead of %#U", cur) return l.errorf("Expected ] instead of %#U", cur)
} }
l.emit(pathBracketRight) l.emit(pathBracketRight)
return lexPathAfterKey return lexPathAfterKey
} }
@ -142,14 +152,17 @@ func lexKey(l lexer, state *intStack) stateFn {
case '*': case '*':
l.take() l.take()
l.emit(pathWildcard) l.emit(pathWildcard)
return lexPathAfterKey return lexPathAfterKey
case '"': case '"':
l.takeString() l.takeString()
l.emit(pathKey) l.emit(pathKey)
return lexPathAfterKey return lexPathAfterKey
case eof: case eof:
l.take() l.take()
l.emit(pathEOF) l.emit(pathEOF)
return nil return nil
default: default:
for { for {
@ -157,9 +170,12 @@ func lexKey(l lexer, state *intStack) stateFn {
if v == '.' || v == '[' || v == '+' || v == '?' || v == eof { if v == '.' || v == '[' || v == '+' || v == '?' || v == eof {
break break
} }
l.take() l.take()
} }
l.emit(pathKey) l.emit(pathKey)
return lexPathAfterKey return lexPathAfterKey
} }
} }
@ -172,6 +188,7 @@ func lexPathIndexRange(l lexer, state *intStack) stateFn {
case ':': case ':':
l.take() l.take()
l.emit(pathIndexRange) l.emit(pathIndexRange)
return lexPathIndexRangeSecond return lexPathIndexRangeSecond
case ']': case ']':
return lexPathBracketClose return lexPathBracketClose
@ -186,6 +203,7 @@ func lexPathIndexRangeSecond(l lexer, state *intStack) stateFn {
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
takeDigits(l) takeDigits(l)
l.emit(pathIndex) l.emit(pathIndex)
return lexPathBracketClose return lexPathBracketClose
case ']': case ']':
return lexPathBracketClose return lexPathBracketClose
@ -199,6 +217,8 @@ func lexPathAfterValue(l lexer, state *intStack) stateFn {
if cur != eof { if cur != eof {
return l.errorf("Expected EOF instead of %#U", cur) return l.errorf("Expected EOF instead of %#U", cur)
} }
l.emit(pathEOF) l.emit(pathEOF)
return nil return nil
} }

View File

@ -21,6 +21,7 @@ var pathTests = []lexTest{
func TestValidPaths(t *testing.T) { func TestValidPaths(t *testing.T) {
as := assert.New(t) as := assert.New(t)
for _, test := range pathTests { for _, test := range pathTests {
lexer := NewSliceLexer([]byte(test.input), PATH) lexer := NewSliceLexer([]byte(test.input), PATH)
types := itemsToTypes(readerToArray(lexer)) types := itemsToTypes(readerToArray(lexer))

View File

@ -18,10 +18,12 @@ func (q *Results) push(n *Result) {
nodes := make([]*Result, len(q.nodes)*2) nodes := make([]*Result, len(q.nodes)*2)
copy(nodes, q.nodes[q.head:]) copy(nodes, q.nodes[q.head:])
copy(nodes[len(q.nodes)-q.head:], q.nodes[:q.head]) copy(nodes[len(q.nodes)-q.head:], q.nodes[:q.head])
q.head = 0 q.head = 0
q.tail = len(q.nodes) q.tail = len(q.nodes)
q.nodes = nodes q.nodes = nodes
} }
q.nodes[q.tail] = n q.nodes[q.tail] = n
q.tail = (q.tail + 1) % len(q.nodes) q.tail = (q.tail + 1) % len(q.nodes)
q.count++ q.count++
@ -31,9 +33,11 @@ func (q *Results) Pop() *Result {
if q.count == 0 { if q.count == 0 {
return nil return nil
} }
node := q.nodes[q.head] node := q.nodes[q.head]
q.head = (q.head + 1) % len(q.nodes) q.head = (q.head + 1) % len(q.nodes)
q.count-- q.count--
return node return node
} }
@ -41,6 +45,7 @@ func (q *Results) Peek() *Result {
if q.count == 0 { if q.count == 0 {
return nil return nil
} }
return q.nodes[q.head] return q.nodes[q.head]
} }

View File

@ -23,6 +23,7 @@ type Result struct {
func (r *Result) Pretty(showPath bool) string { func (r *Result) Pretty(showPath bool) string {
b := bytes.NewBufferString("") b := bytes.NewBufferString("")
printed := false printed := false
if showPath { if showPath {
for _, k := range r.Keys { for _, k := range r.Keys {
switch v := k.(type) { switch v := k.(type) {
@ -32,6 +33,7 @@ func (r *Result) Pretty(showPath bool) string {
b.WriteString(fmt.Sprintf("%q", v)) b.WriteString(fmt.Sprintf("%q", v))
} }
b.WriteRune('\t') b.WriteRune('\t')
printed = true printed = true
} }
} else if r.Value == nil { } else if r.Value == nil {
@ -48,10 +50,13 @@ func (r *Result) Pretty(showPath bool) string {
if r.Value != nil { if r.Value != nil {
printed = true printed = true
b.WriteString(fmt.Sprintf("%s", r.Value)) b.WriteString(fmt.Sprintf("%s", r.Value))
} }
if printed { if printed {
b.WriteRune('\n') b.WriteRune('\n')
} }
return b.String() return b.String()
} }

5
run.go
View File

@ -5,23 +5,28 @@ import "io"
func EvalPathsInBytes(input []byte, paths []*Path) (*Eval, error) { func EvalPathsInBytes(input []byte, paths []*Path) (*Eval, error) {
lexer := NewSliceLexer(input, JSON) lexer := NewSliceLexer(input, JSON)
eval := newEvaluation(lexer, paths...) eval := newEvaluation(lexer, paths...)
return eval, nil return eval, nil
} }
func EvalPathsInReader(r io.Reader, paths []*Path) (*Eval, error) { func EvalPathsInReader(r io.Reader, paths []*Path) (*Eval, error) {
lexer := NewReaderLexer(r, JSON) lexer := NewReaderLexer(r, JSON)
eval := newEvaluation(lexer, paths...) eval := newEvaluation(lexer, paths...)
return eval, nil return eval, nil
} }
func ParsePaths(pathStrings ...string) ([]*Path, error) { func ParsePaths(pathStrings ...string) ([]*Path, error) {
paths := make([]*Path, len(pathStrings)) paths := make([]*Path, len(pathStrings))
for x, p := range pathStrings { for x, p := range pathStrings {
path, err := parsePath(p) path, err := parsePath(p)
if err != nil { if err != nil {
return nil, err return nil, err
} }
paths[x] = path paths[x] = path
} }
return paths, nil return paths, nil
} }

View File

@ -24,8 +24,10 @@ func (s *intStack) pop() (int, bool) {
if s.len() == 0 { if s.len() == 0 {
return 0, false return 0, false
} }
v, _ := s.peek() v, _ := s.peek()
s.values = s.values[:len(s.values)-1] s.values = s.values[:len(s.values)-1]
return v, true return v, true
} }
@ -33,7 +35,9 @@ func (s *intStack) peek() (int, bool) {
if s.len() == 0 { if s.len() == 0 {
return 0, false return 0, false
} }
v := s.values[len(s.values)-1] v := s.values[len(s.values)-1]
return v, true return v, true
} }
@ -61,8 +65,10 @@ func (s *stack) pop() (interface{}, bool) {
if s.len() == 0 { if s.len() == 0 {
return nil, false return nil, false
} }
v, _ := s.peek() v, _ := s.peek()
s.values = s.values[:len(s.values)-1] s.values = s.values[:len(s.values)-1]
return v, true return v, true
} }
@ -70,7 +76,9 @@ func (s *stack) peek() (interface{}, bool) {
if s.len() == 0 { if s.len() == 0 {
return nil, false return nil, false
} }
v := s.values[len(s.values)-1] v := s.values[len(s.values)-1]
return v, true return v, true
} }
@ -79,6 +87,7 @@ func (s *stack) clone() *stack {
values: make([]interface{}, s.len()), values: make([]interface{}, s.len()),
} }
copy(d.values, s.values) copy(d.values, s.values)
return &d return &d
} }