diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index 49dfb7544..ad5171488 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -130,19 +130,21 @@ yajl_lex_free(yajl_lexer lxr) return; } -/* a lookup table which lets us quickly determine three things: +/* a lookup table which lets us quickly determine various things: * VEC - valid escaped control char - * note. the solidus '/' may be escaped or not. + * Note: the solidus '/' may be escaped or not. * IJC - invalid json char * VHC - valid hex char * NFP - needs further processing (from a string scanning perspective) * NUC - needs utf8 checking when enabled (from a string scanning perspective) + * VIC - valid identifier char (after the first char) */ #define VEC 0x01 #define IJC 0x02 #define VHC 0x04 #define NFP 0x08 #define NUC 0x10 +#define VIC 0x20 static const char charLookupTable[256] = { @@ -151,20 +153,20 @@ static const char charLookupTable[256] = /*10*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , /*18*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , -/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , 0 , +/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , VIC , 0 , 0 , 0 , /*28*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , VEC , -/*30*/ VHC , VHC , VHC , VHC , VHC , VHC , VHC , VHC , -/*38*/ VHC , VHC , 0 , 0 , 0 , 0 , 0 , 0 , +/*30*/ VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, +/*38*/ VHC|VIC, VHC|VIC, 0 , 0 , 0 , 0 , 0 , 0 , -/*40*/ 0 , VHC , VHC , VHC , VHC , VHC , VHC , 0 , -/*48*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , -/*50*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , -/*58*/ 0 , 0 , 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , +/*40*/ 0 , VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VIC , +/*48*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC , +/*50*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC , +/*58*/ VIC , VIC , VIC , 0 , NFP|VEC|IJC, 0 , 0 , VIC , -/*60*/ 0 , VHC , VEC|VHC, VHC , VHC , VHC , VEC|VHC, 0 , -/*68*/ 0 , 0 , 0 , 0 , 0 , 0 , VEC , 0 , -/*70*/ 0 , 0 , VEC , 0 , VEC , 0 , 0 , 0 , -/*78*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , +/*60*/ 0 , VHC|VIC, VEC|VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VEC|VHC|VIC, VIC, +/*68*/ VIC , VIC , VIC , VIC , VIC , VIC , VEC|VIC, VIC , +/*70*/ VIC , VIC , VEC , VIC , VEC|VIC, VIC , VIC , VIC , +/*78*/ VIC , VIC , VIC , 0 , 0 , 0 , 0 , 0 , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , @@ -376,14 +378,31 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, #define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof; +/* For both identifiers and numbers, we always have to lex one + * character too many to know when they are complete. + */ + +static yajl_tok +yajl_lex_identifier(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset) +{ + unsigned char c; + + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } while (charLookupTable[c] & VIC); + + /* we always go "one too far" */ + unreadChar(lexer, offset); + + return yajl_tok_identifier; +} + static yajl_tok yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t * offset) { - /** XXX: numbers are the only entities in json that we must lex - * _beyond_ in order to know that they are complete. There - * is an ambiguous case for integers at EOF. */ - const char hexDigits[] = "0123456789abcdefABCDEF"; unsigned char c; int numRd = 0; @@ -732,6 +751,122 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, *outLen -= 2; } +#ifdef YAJL_LEXER_DEBUG + if (tok == yajl_tok_error) { + printf("lexical error: %s\n", + yajl_lex_error_to_string(yajl_lex_get_error(lexer))); + } else if (tok == yajl_tok_eof) { + printf("EOF hit\n"); + } else { + printf("lexed %s: '", tokToStr(tok)); + fwrite(*outBuf, 1, *outLen, stdout); + printf("'\n"); + } +#endif + + return tok; +} + +yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen) +{ + yajl_tok tok = yajl_tok_error; + unsigned char c; + size_t startOffset = *offset; + + *outBuf = NULL; + *outLen = 0; + + for (;;) { + assert(*offset <= jsonTextLen); + + if (*offset >= jsonTextLen) { + tok = yajl_tok_eof; + goto lexed; + } + + c = readChar(lexer, jsonText, offset); + + switch (c) { + case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + startOffset++; + break; + case '}': + tok = yajl_tok_right_brace; + goto lexed; + case '"': { + tok = yajl_lex_string(lexer, jsonText, jsonTextLen, offset); + goto lexed; + } + case '/': + /* If comments are disabled this is an error. */ + if (!lexer->allowComments) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_unallowed_comment; + tok = yajl_tok_error; + goto lexed; + } + /* Comments are enabled, so lex it. + * Possible outcomes are: + * - successful lex (tok_comment, which means continue), + * - malformed comment opening (slash not followed by + * '*' or '/') (tok_error) + * - eof hit. (tok_eof) */ + tok = yajl_lex_comment(lexer, jsonText, jsonTextLen, offset); + if (tok == yajl_tok_comment) { + /* "error" is silly, but that's the initial + * state of tok. guilty until proven innocent. */ + tok = yajl_tok_error; + yajl_buf_clear(lexer->buf); + lexer->bufInUse = 0; + startOffset = *offset; + break; + } + /* hit error or eof, bail */ + goto lexed; + default: + if (lexer->allowJson5 && (c == '$' || c == '_' || + (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) { + tok = yajl_lex_identifier(lexer, jsonText, jsonTextLen, offset); + } + else { + lexer->error = yajl_lex_invalid_char; + tok = yajl_tok_error; + } + goto lexed; + } + } + + lexed: + /* need to append to buffer if the buffer is in use or + * if it's an EOF token */ + if (tok == yajl_tok_eof || lexer->bufInUse) { + if (!lexer->bufInUse) yajl_buf_clear(lexer->buf); + lexer->bufInUse = 1; + yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset); + lexer->bufOff = 0; + + if (tok != yajl_tok_eof) { + *outBuf = yajl_buf_data(lexer->buf); + *outLen = yajl_buf_len(lexer->buf); + lexer->bufInUse = 0; + } + } else if (tok != yajl_tok_error) { + *outBuf = jsonText + startOffset; + *outLen = *offset - startOffset; + } + + /* For strings skip the quotes. */ + if (tok == yajl_tok_string || + tok == yajl_tok_string_with_escapes) { + assert(*outLen >= 2); + (*outBuf)++; + *outLen -= 2; + } + else if (tok == yajl_tok_identifier) { + tok = yajl_tok_string; + } #ifdef YAJL_LEXER_DEBUG if (tok == yajl_tok_error) { diff --git a/modules/libcom/src/yajl/yajl_lex.h b/modules/libcom/src/yajl/yajl_lex.h index c5eec8f00..ebe647bd2 100644 --- a/modules/libcom/src/yajl/yajl_lex.h +++ b/modules/libcom/src/yajl/yajl_lex.h @@ -41,7 +41,12 @@ typedef enum { yajl_tok_string, yajl_tok_string_with_escapes, - /* comment tokens are not currently returned to the parser, ever */ + /* These tokens are used within the lexer and never seen by the parser: */ + + /* An unquoted map key, for JSON5 only, returned as yajl_tok_string */ + yajl_tok_identifier, + + /* A comment token, never returned */ yajl_tok_comment } yajl_tok; @@ -84,6 +89,14 @@ yajl_tok yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t * offset, const unsigned char ** outBuf, size_t * outLen); +/** + * A specialized version of yajl_lex_lex for use when the next token is + * a map key, which the parser knows. + */ +yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen); + /** have a peek at the next token, but don't move the lexer forward */ yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t offset); diff --git a/modules/libcom/src/yajl/yajl_parser.c b/modules/libcom/src/yajl/yajl_parser.c index 3f62932b6..b400709a6 100644 --- a/modules/libcom/src/yajl/yajl_parser.c +++ b/modules/libcom/src/yajl/yajl_parser.c @@ -401,8 +401,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, case yajl_state_map_need_key: { /* only difference between these two states is that in * start '}' is valid, whereas in need_key, we've parsed - * a comma, and a string key _must_ follow */ - tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + * a comma, so unless this is JSON5 a key _must_ follow. */ + tok = yajl_lex_key(hand->lexer, jsonText, jsonTextLen, offset, &buf, &bufLen); switch (tok) { case yajl_tok_eof: @@ -439,7 +439,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, } default: yajl_bs_set(hand->stateStack, yajl_state_parse_error); - hand->parseError = + hand->parseError = hand->flags & yajl_allow_json5 ? + "invalid object key (must be a string or identifier)" : "invalid object key (must be a string)"; goto around_again; } diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 4d5b65d49..03f00b206 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -94,6 +94,49 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "map_identifiers", + opts => [ + -5 + ], + input => [ + "{", + " \$:1,", + " _:2,", + " A:3,", + " Z:4,", + " a:5,", + " z:6,", + " \$1:7,", + " _zz:8,", + " ZZ9\$Zalpha:9", + "}", + "" + ], + gives => [ + "map open '{'", + "key: '\$'", + "integer: 1", + "key: '_'", + "integer: 2", + "key: 'A'", + "integer: 3", + "key: 'Z'", + "integer: 4", + "key: 'a'", + "integer: 5", + "key: 'z'", + "integer: 6", + "key: '\$1'", + "integer: 7", + "key: '_zz'", + "integer: 8", + "key: 'ZZ9\$Zalpha'", + "integer: 9", + "map close '}'", + "memory leaks:\t0" + ] + }, { name => "simple_with_comments", opts => [