go.mod: github.com/buger/jsonparser v1.1.1

Fix CVE-2020-35381

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
This commit is contained in:
Sebastiaan van Stijn
2021-03-12 17:28:04 +01:00
parent c3d01539d5
commit f4d2925220
13 changed files with 382 additions and 90 deletions

View File

@ -5,6 +5,8 @@
*.mprof
.idea
vendor/github.com/buger/goterm/
prof.cpu
prof.mem

View File

@ -1,3 +1,11 @@
language: go
go: 1.7
arch:
- amd64
- ppc64le
go:
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- 1.11.x
script: go test -v ./.

View File

@ -1,5 +1,5 @@
[![Go Report Card](https://goreportcard.com/badge/github.com/buger/jsonparser)](https://goreportcard.com/report/github.com/buger/jsonparser) ![License](https://img.shields.io/dub/l/vibe-d.svg)
# Alternative JSON parser for Go (so far fastest)
# Alternative JSON parser for Go (10x times faster standard library)
It does not require you to know the structure of the payload (eg. create structs), and allows accessing fields by providing the path to them. It is up to **10 times faster** than standard `encoding/json` package (depending on payload size and usage), **allocates no memory**. See benchmarks below.
@ -61,7 +61,7 @@ jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, off
}, "person", "avatars")
// Or use can access fields by index!
jsonparser.GetInt("person", "avatars", "[0]", "url")
jsonparser.GetString(data, "person", "avatars", "[0]", "url")
// You can use `ObjectEach` helper to iterate objects { "key1":object1, "key2":object2, .... "keyN":objectN }
jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {

View File

@ -1,9 +1,16 @@
package jsonparser
// About 3x faster then strconv.ParseInt because does not check for range error and support only base 10, which is enough for JSON
func parseInt(bytes []byte) (v int64, ok bool) {
import (
bio "bytes"
)
// minInt64 '-9223372036854775808' is the smallest representable number in int64
const minInt64 = `9223372036854775808`
// About 2x faster then strconv.ParseInt because it only supports base 10, which is enough for JSON
func parseInt(bytes []byte) (v int64, ok bool, overflow bool) {
if len(bytes) == 0 {
return 0, false
return 0, false, false
}
var neg bool = false
@ -12,17 +19,29 @@ func parseInt(bytes []byte) (v int64, ok bool) {
bytes = bytes[1:]
}
var b int64 = 0
for _, c := range bytes {
if c >= '0' && c <= '9' {
v = (10 * v) + int64(c-'0')
b = (10 * v) + int64(c-'0')
} else {
return 0, false
return 0, false, false
}
if overflow = (b < v); overflow {
break
}
v = b
}
if overflow {
if neg && bio.Equal(bytes, []byte(minInt64)) {
return b, true, false
}
return 0, false, true
}
if neg {
return -v, true
return -v, true, false
} else {
return v, true
return v, true, false
}
}

View File

@ -6,6 +6,7 @@ import (
"reflect"
"strconv"
"unsafe"
"runtime"
)
//
@ -32,11 +33,12 @@ func bytesToString(b *[]byte) string {
}
func StringToBytes(s string) []byte {
b := make([]byte, 0, 0)
bh := (*reflect.SliceHeader)(unsafe.Pointer(&b))
sh := (*reflect.StringHeader)(unsafe.Pointer(&s))
bh := reflect.SliceHeader{
Data: sh.Data,
Len: sh.Len,
Cap: sh.Len,
}
return *(*[]byte)(unsafe.Pointer(&bh))
bh.Data = sh.Data
bh.Cap = sh.Len
bh.Len = sh.Len
runtime.KeepAlive(s)
return b
}

117
vendor/github.com/buger/jsonparser/fuzz.go generated vendored Normal file
View File

@ -0,0 +1,117 @@
package jsonparser
func FuzzParseString(data []byte) int {
r, err := ParseString(data)
if err != nil || r == "" {
return 0
}
return 1
}
func FuzzEachKey(data []byte) int {
paths := [][]string{
{"name"},
{"order"},
{"nested", "a"},
{"nested", "b"},
{"nested2", "a"},
{"nested", "nested3", "b"},
{"arr", "[1]", "b"},
{"arrInt", "[3]"},
{"arrInt", "[5]"},
{"nested"},
{"arr", "["},
{"a\n", "b\n"},
}
EachKey(data, func(idx int, value []byte, vt ValueType, err error) {}, paths...)
return 1
}
func FuzzDelete(data []byte) int {
Delete(data, "test")
return 1
}
func FuzzSet(data []byte) int {
_, err := Set(data, []byte(`"new value"`), "test")
if err != nil {
return 0
}
return 1
}
func FuzzObjectEach(data []byte) int {
_ = ObjectEach(data, func(key, value []byte, valueType ValueType, off int) error {
return nil
})
return 1
}
func FuzzParseFloat(data []byte) int {
_, err := ParseFloat(data)
if err != nil {
return 0
}
return 1
}
func FuzzParseInt(data []byte) int {
_, err := ParseInt(data)
if err != nil {
return 0
}
return 1
}
func FuzzParseBool(data []byte) int {
_, err := ParseBoolean(data)
if err != nil {
return 0
}
return 1
}
func FuzzTokenStart(data []byte) int {
_ = tokenStart(data)
return 1
}
func FuzzGetString(data []byte) int {
_, err := GetString(data, "test")
if err != nil {
return 0
}
return 1
}
func FuzzGetFloat(data []byte) int {
_, err := GetFloat(data, "test")
if err != nil {
return 0
}
return 1
}
func FuzzGetInt(data []byte) int {
_, err := GetInt(data, "test")
if err != nil {
return 0
}
return 1
}
func FuzzGetBoolean(data []byte) int {
_, err := GetBoolean(data, "test")
if err != nil {
return 0
}
return 1
}
func FuzzGetUnsafeString(data []byte) int {
_, err := GetUnsafeString(data, "test")
if err != nil {
return 0
}
return 1
}

4
vendor/github.com/buger/jsonparser/go.mod generated vendored Normal file
View File

@ -0,0 +1,4 @@
module github.com/buger/jsonparser
go 1.13

0
vendor/github.com/buger/jsonparser/go.sum generated vendored Normal file
View File

47
vendor/github.com/buger/jsonparser/oss-fuzz-build.sh generated vendored Normal file
View File

@ -0,0 +1,47 @@
#!/bin/bash -eu
git clone https://github.com/dvyukov/go-fuzz-corpus
zip corpus.zip go-fuzz-corpus/json/corpus/*
cp corpus.zip $OUT/fuzzparsestring_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzParseString fuzzparsestring
cp corpus.zip $OUT/fuzzeachkey_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzEachKey fuzzeachkey
cp corpus.zip $OUT/fuzzdelete_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzDelete fuzzdelete
cp corpus.zip $OUT/fuzzset_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzSet fuzzset
cp corpus.zip $OUT/fuzzobjecteach_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzObjectEach fuzzobjecteach
cp corpus.zip $OUT/fuzzparsefloat_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzParseFloat fuzzparsefloat
cp corpus.zip $OUT/fuzzparseint_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzParseInt fuzzparseint
cp corpus.zip $OUT/fuzzparsebool_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzParseBool fuzzparsebool
cp corpus.zip $OUT/fuzztokenstart_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzTokenStart fuzztokenstart
cp corpus.zip $OUT/fuzzgetstring_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzGetString fuzzgetstring
cp corpus.zip $OUT/fuzzgetfloat_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzGetFloat fuzzgetfloat
cp corpus.zip $OUT/fuzzgetint_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzGetInt fuzzgetint
cp corpus.zip $OUT/fuzzgetboolean_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzGetBoolean fuzzgetboolean
cp corpus.zip $OUT/fuzzgetunsafestring_seed_corpus.zip
compile_go_fuzzer github.com/buger/jsonparser FuzzGetUnsafeString fuzzgetunsafestring

View File

@ -4,7 +4,6 @@ import (
"bytes"
"errors"
"fmt"
"math"
"strconv"
)
@ -17,6 +16,7 @@ var (
MalformedArrayError = errors.New("Value is array, but can't find closing ']' symbol")
MalformedObjectError = errors.New("Value looks like object, but can't find closing '}' symbol")
MalformedValueError = errors.New("Value looks like Number/Boolean/None, but can't find its end: ',' or '}' symbol")
OverflowIntegerError = errors.New("Value is number, but overflowed while parsing")
MalformedStringEscapeError = errors.New("Encountered an invalid escape sequence in a string")
)
@ -97,9 +97,15 @@ func findKeyStart(data []byte, key string) (int, error) {
}
case '[':
i = blockEnd(data[i:], data[i], ']') + i
end := blockEnd(data[i:], data[i], ']')
if end != -1 {
i = i + end
}
case '{':
i = blockEnd(data[i:], data[i], '}') + i
end := blockEnd(data[i:], data[i], '}')
if end != -1 {
i = i + end
}
}
i++
}
@ -213,6 +219,7 @@ func searchKeys(data []byte, keys ...string) int {
i := 0
ln := len(data)
lk := len(keys)
lastMatched := true
if lk == 0 {
return 0
@ -240,8 +247,8 @@ func searchKeys(data []byte, keys ...string) int {
i += valueOffset
// if string is a key, and key level match
if data[i] == ':' && keyLevel == level-1 {
// if string is a key
if data[i] == ':' {
if level < 1 {
return -1
}
@ -259,18 +266,40 @@ func searchKeys(data []byte, keys ...string) int {
keyUnesc = ku
}
if equalStr(&keyUnesc, keys[level-1]) {
keyLevel++
// If we found all keys in path
if keyLevel == lk {
return i + 1
if level <= len(keys) {
if equalStr(&keyUnesc, keys[level-1]) {
lastMatched = true
// if key level match
if keyLevel == level-1 {
keyLevel++
// If we found all keys in path
if keyLevel == lk {
return i + 1
}
}
} else {
lastMatched = false
}
} else {
return -1
}
} else {
i--
}
case '{':
level++
// in case parent key is matched then only we will increase the level otherwise can directly
// can move to the end of this block
if !lastMatched {
end := blockEnd(data[i:], '{', '}')
if end == -1 {
return -1
}
i += end - 1
} else {
level++
}
case '}':
level--
if level == keyLevel {
@ -279,7 +308,11 @@ func searchKeys(data []byte, keys ...string) int {
case '[':
// If we want to get array element by index
if keyLevel == level && keys[level][0] == '[' {
aIdx, err := strconv.Atoi(keys[level][1 : len(keys[level])-1])
var keyLen = len(keys[level])
if keyLen < 3 || keys[level][0] != '[' || keys[level][keyLen-1] != ']' {
return -1
}
aIdx, err := strconv.Atoi(keys[level][1 : keyLen-1])
if err != nil {
return -1
}
@ -316,6 +349,8 @@ func searchKeys(data []byte, keys ...string) int {
i += arraySkip - 1
}
}
case ':': // If encountered, JSON data is malformed
return -1
}
i++
@ -324,14 +359,6 @@ func searchKeys(data []byte, keys ...string) int {
return -1
}
var bitwiseFlags []int64
func init() {
for i := 0; i < 63; i++ {
bitwiseFlags = append(bitwiseFlags, int64(math.Pow(2, float64(i))))
}
}
func sameTree(p1, p2 []string) bool {
minLen := len(p1)
if len(p2) < minLen {
@ -348,7 +375,8 @@ func sameTree(p1, p2 []string) bool {
}
func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]string) int {
var pathFlags int64
var x struct{}
pathFlags := make([]bool, len(paths))
var level, pathsMatched, i int
ln := len(data)
@ -359,7 +387,6 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
}
}
var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
pathsBuf := make([]string, maxPath)
for i < ln {
@ -393,10 +420,13 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
var keyUnesc []byte
if !keyEscaped {
keyUnesc = key
} else if ku, err := Unescape(key, stackbuf[:]); err != nil {
return -1
} else {
keyUnesc = ku
var stackbuf [unescapeStackBufSize]byte
if ku, err := Unescape(key, stackbuf[:]); err != nil {
return -1
} else {
keyUnesc = ku
}
}
if maxPath >= level {
@ -407,23 +437,18 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
pathsBuf[level-1] = bytesToString(&keyUnesc)
for pi, p := range paths {
if len(p) != level || pathFlags&bitwiseFlags[pi+1] != 0 || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
if len(p) != level || pathFlags[pi] || !equalStr(&keyUnesc, p[level-1]) || !sameTree(p, pathsBuf[:level]) {
continue
}
match = pi
i++
pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true
v, dt, of, e := Get(data[i:])
v, dt, _, e := Get(data[i+1:])
cb(pi, v, dt, e)
if of != -1 {
i += of
}
if pathsMatched == len(paths) {
break
}
@ -457,8 +482,9 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
case '}':
level--
case '[':
var arrIdxFlags int64
var pIdxFlags int64
var ok bool
arrIdxFlags := make(map[int]struct{})
pIdxFlags := make([]bool, len(paths))
if level < 0 {
cb(-1, nil, Unknown, MalformedJsonError)
@ -466,30 +492,31 @@ func EachKey(data []byte, cb func(int, []byte, ValueType, error), paths ...[]str
}
for pi, p := range paths {
if len(p) < level+1 || pathFlags&bitwiseFlags[pi+1] != 0 || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
if len(p) < level+1 || pathFlags[pi] || p[level][0] != '[' || !sameTree(p, pathsBuf[:level]) {
continue
}
aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
arrIdxFlags |= bitwiseFlags[aIdx+1]
pIdxFlags |= bitwiseFlags[pi+1]
if len(p[level]) >= 2 {
aIdx, _ := strconv.Atoi(p[level][1 : len(p[level])-1])
arrIdxFlags[aIdx] = x
pIdxFlags[pi] = true
}
}
if arrIdxFlags > 0 {
if len(arrIdxFlags) > 0 {
level++
var curIdx int
arrOff, _ := ArrayEach(data[i:], func(value []byte, dataType ValueType, offset int, err error) {
if arrIdxFlags&bitwiseFlags[curIdx+1] != 0 {
if _, ok = arrIdxFlags[curIdx]; ok {
for pi, p := range paths {
if pIdxFlags&bitwiseFlags[pi+1] != 0 {
if pIdxFlags[pi] {
aIdx, _ := strconv.Atoi(p[level-1][1 : len(p[level-1])-1])
if curIdx == aIdx {
of := searchKeys(value, p[level:]...)
pathsMatched++
pathFlags |= bitwiseFlags[pi+1]
pathFlags[pi] = true
if of != -1 {
v, dt, _, e := Get(value[of:])
@ -568,46 +595,96 @@ var (
)
func createInsertComponent(keys []string, setValue []byte, comma, object bool) []byte {
var buffer bytes.Buffer
isIndex := string(keys[0][0]) == "["
offset := 0
lk := calcAllocateSpace(keys, setValue, comma, object)
buffer := make([]byte, lk, lk)
if comma {
buffer.WriteString(",")
offset += WriteToBuffer(buffer[offset:], ",")
}
if isIndex {
buffer.WriteString("[")
if isIndex && !comma {
offset += WriteToBuffer(buffer[offset:], "[")
} else {
if object {
buffer.WriteString("{")
offset += WriteToBuffer(buffer[offset:], "{")
}
if !isIndex {
offset += WriteToBuffer(buffer[offset:], "\"")
offset += WriteToBuffer(buffer[offset:], keys[0])
offset += WriteToBuffer(buffer[offset:], "\":")
}
buffer.WriteString("\"")
buffer.WriteString(keys[0])
buffer.WriteString("\":")
}
for i := 1; i < len(keys); i++ {
if string(keys[i][0]) == "[" {
buffer.WriteString("[")
offset += WriteToBuffer(buffer[offset:], "[")
} else {
buffer.WriteString("{\"")
buffer.WriteString(keys[i])
buffer.WriteString("\":")
offset += WriteToBuffer(buffer[offset:], "{\"")
offset += WriteToBuffer(buffer[offset:], keys[i])
offset += WriteToBuffer(buffer[offset:], "\":")
}
}
buffer.Write(setValue)
offset += WriteToBuffer(buffer[offset:], string(setValue))
for i := len(keys) - 1; i > 0; i-- {
if string(keys[i][0]) == "[" {
buffer.WriteString("]")
offset += WriteToBuffer(buffer[offset:], "]")
} else {
buffer.WriteString("}")
offset += WriteToBuffer(buffer[offset:], "}")
}
}
if isIndex {
buffer.WriteString("]")
if isIndex && !comma {
offset += WriteToBuffer(buffer[offset:], "]")
}
if object && !isIndex {
buffer.WriteString("}")
offset += WriteToBuffer(buffer[offset:], "}")
}
return buffer.Bytes()
return buffer
}
func calcAllocateSpace(keys []string, setValue []byte, comma, object bool) int {
isIndex := string(keys[0][0]) == "["
lk := 0
if comma {
// ,
lk += 1
}
if isIndex && !comma {
// []
lk += 2
} else {
if object {
// {
lk += 1
}
if !isIndex {
// "keys[0]"
lk += len(keys[0]) + 3
}
}
lk += len(setValue)
for i := 1; i < len(keys); i++ {
if string(keys[i][0]) == "[" {
// []
lk += 2
} else {
// {"keys[i]":setValue}
lk += len(keys[i]) + 5
}
}
if object && !isIndex {
// }
lk += 1
}
return lk
}
func WriteToBuffer(buffer []byte, str string) int {
copy(buffer, str)
return len(str)
}
/*
@ -687,7 +764,12 @@ func Delete(data []byte, keys ...string) []byte {
newOffset = prevTok + 1
}
data = append(data[:newOffset], data[endOffset:]...)
// We have to make a copy here if we don't want to mangle the original data, because byte slices are
// accessed by reference and not by value
dataCopy := make([]byte, len(data))
copy(dataCopy, data)
data = append(dataCopy[:newOffset], dataCopy[endOffset:]...)
return data
}
@ -730,7 +812,7 @@ func Set(data []byte, setValue []byte, keys ...string) (value []byte, err error)
if endOffset == -1 {
firstToken := nextToken(data)
// We can't set a top-level key if data isn't an object
if len(data) == 0 || data[firstToken] != '{' {
if firstToken < 0 || data[firstToken] != '{' {
return nil, KeyPathNotFoundError
}
// Don't need a comma if the input is an empty object
@ -745,7 +827,9 @@ func Set(data []byte, setValue []byte, keys ...string) (value []byte, err error)
depthOffset := endOffset
if depth != 0 {
// if subpath is a non-empty object, add to it
if data[startOffset] == '{' && data[startOffset+1+nextToken(data[startOffset+1:])] != '}' {
// or if subpath is a non-empty array, add to it
if (data[startOffset] == '{' && data[startOffset+1+nextToken(data[startOffset+1:])] != '}') ||
(data[startOffset] == '[' && data[startOffset+1+nextToken(data[startOffset+1:])] == '{') && keys[depth:][0][0] == 91 {
depthOffset--
startOffset = depthOffset
// otherwise, over-write it with a new object
@ -878,7 +962,7 @@ func internalGet(data []byte, keys ...string) (value []byte, dataType ValueType,
value = value[1 : len(value)-1]
}
return value, dataType, offset, endOffset, nil
return value[:len(value):len(value)], dataType, offset, endOffset, nil
}
// ArrayEach is used when iterating arrays, accepts a callback function with the same return arguments as `Get`.
@ -887,7 +971,12 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
return -1, MalformedObjectError
}
offset = 1
nT := nextToken(data)
if nT == -1 {
return -1, MalformedJsonError
}
offset = nT + 1
if len(keys) > 0 {
if offset = searchKeys(data, keys...); offset == -1 {
@ -963,7 +1052,6 @@ func ArrayEach(data []byte, cb func(value []byte, dataType ValueType, offset int
// ObjectEach iterates over the key-value pairs of a JSON object, invoking a given callback for each such entry
func ObjectEach(data []byte, callback func(key []byte, value []byte, dataType ValueType, offset int) error, keys ...string) (err error) {
var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
offset := 0
// Descend to the desired key, if requested
@ -1017,6 +1105,7 @@ func ObjectEach(data []byte, callback func(key []byte, value []byte, dataType Va
// Unescape the string if needed
if keyEscaped {
var stackbuf [unescapeStackBufSize]byte // stack-allocated array for allocation-free unescaping of small strings
if keyUnescaped, err := Unescape(key, stackbuf[:]); err != nil {
return MalformedStringEscapeError
} else {
@ -1092,7 +1181,7 @@ func GetString(data []byte, keys ...string) (val string, err error) {
return "", fmt.Errorf("Value is not a string: %s", string(v))
}
// If no escapes return raw conten
// If no escapes return raw content
if bytes.IndexByte(v, '\\') == -1 {
return string(v), nil
}
@ -1183,7 +1272,10 @@ func ParseFloat(b []byte) (float64, error) {
// ParseInt parses a Number ValueType into a Go int64
func ParseInt(b []byte) (int64, error) {
if v, ok := parseInt(b); !ok {
if v, ok, overflow := parseInt(b); !ok {
if overflow {
return 0, OverflowIntegerError
}
return 0, MalformedValueError
} else {
return v, nil