From e2256d06631e067d40710b5388b10b7eac4555ff Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 00:35:26 -0500 Subject: [PATCH] Accept unquoted identifiers as map keys Adds another lexer entry point for lexing map keys only, adjust parser to use this instead of the general lexer. Also defines another lexer token for internal use only. --- modules/libcom/src/yajl/yajl_lex.c | 169 +++++++++++++++++++++++--- modules/libcom/src/yajl/yajl_lex.h | 15 ++- modules/libcom/src/yajl/yajl_parser.c | 7 +- modules/libcom/test/yajlTestCases.pm | 43 +++++++ 4 files changed, 213 insertions(+), 21 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index 49dfb7544..ad5171488 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -130,19 +130,21 @@ yajl_lex_free(yajl_lexer lxr) return; } -/* a lookup table which lets us quickly determine three things: +/* a lookup table which lets us quickly determine various things: * VEC - valid escaped control char - * note. the solidus '/' may be escaped or not. + * Note: the solidus '/' may be escaped or not. * IJC - invalid json char * VHC - valid hex char * NFP - needs further processing (from a string scanning perspective) * NUC - needs utf8 checking when enabled (from a string scanning perspective) + * VIC - valid identifier char (after the first char) */ #define VEC 0x01 #define IJC 0x02 #define VHC 0x04 #define NFP 0x08 #define NUC 0x10 +#define VIC 0x20 static const char charLookupTable[256] = { @@ -151,20 +153,20 @@ static const char charLookupTable[256] = /*10*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , /*18*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , -/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , 0 , +/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , VIC , 0 , 0 , 0 , /*28*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , VEC , -/*30*/ VHC , VHC , VHC , VHC , VHC , VHC , VHC , VHC , -/*38*/ VHC , VHC , 0 , 0 , 0 , 0 , 0 , 0 , +/*30*/ VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, +/*38*/ VHC|VIC, VHC|VIC, 0 , 0 , 0 , 0 , 0 , 0 , -/*40*/ 0 , VHC , VHC , VHC , VHC , VHC , VHC , 0 , -/*48*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , -/*50*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , -/*58*/ 0 , 0 , 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , +/*40*/ 0 , VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VIC , +/*48*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC , +/*50*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC , +/*58*/ VIC , VIC , VIC , 0 , NFP|VEC|IJC, 0 , 0 , VIC , -/*60*/ 0 , VHC , VEC|VHC, VHC , VHC , VHC , VEC|VHC, 0 , -/*68*/ 0 , 0 , 0 , 0 , 0 , 0 , VEC , 0 , -/*70*/ 0 , 0 , VEC , 0 , VEC , 0 , 0 , 0 , -/*78*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , +/*60*/ 0 , VHC|VIC, VEC|VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VEC|VHC|VIC, VIC, +/*68*/ VIC , VIC , VIC , VIC , VIC , VIC , VEC|VIC, VIC , +/*70*/ VIC , VIC , VEC , VIC , VEC|VIC, VIC , VIC , VIC , +/*78*/ VIC , VIC , VIC , 0 , 0 , 0 , 0 , 0 , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , @@ -376,14 +378,31 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, #define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof; +/* For both identifiers and numbers, we always have to lex one + * character too many to know when they are complete. + */ + +static yajl_tok +yajl_lex_identifier(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset) +{ + unsigned char c; + + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } while (charLookupTable[c] & VIC); + + /* we always go "one too far" */ + unreadChar(lexer, offset); + + return yajl_tok_identifier; +} + static yajl_tok yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t * offset) { - /** XXX: numbers are the only entities in json that we must lex - * _beyond_ in order to know that they are complete. There - * is an ambiguous case for integers at EOF. */ - const char hexDigits[] = "0123456789abcdefABCDEF"; unsigned char c; int numRd = 0; @@ -732,6 +751,122 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, *outLen -= 2; } +#ifdef YAJL_LEXER_DEBUG + if (tok == yajl_tok_error) { + printf("lexical error: %s\n", + yajl_lex_error_to_string(yajl_lex_get_error(lexer))); + } else if (tok == yajl_tok_eof) { + printf("EOF hit\n"); + } else { + printf("lexed %s: '", tokToStr(tok)); + fwrite(*outBuf, 1, *outLen, stdout); + printf("'\n"); + } +#endif + + return tok; +} + +yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen) +{ + yajl_tok tok = yajl_tok_error; + unsigned char c; + size_t startOffset = *offset; + + *outBuf = NULL; + *outLen = 0; + + for (;;) { + assert(*offset <= jsonTextLen); + + if (*offset >= jsonTextLen) { + tok = yajl_tok_eof; + goto lexed; + } + + c = readChar(lexer, jsonText, offset); + + switch (c) { + case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + startOffset++; + break; + case '}': + tok = yajl_tok_right_brace; + goto lexed; + case '"': { + tok = yajl_lex_string(lexer, jsonText, jsonTextLen, offset); + goto lexed; + } + case '/': + /* If comments are disabled this is an error. */ + if (!lexer->allowComments) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_unallowed_comment; + tok = yajl_tok_error; + goto lexed; + } + /* Comments are enabled, so lex it. + * Possible outcomes are: + * - successful lex (tok_comment, which means continue), + * - malformed comment opening (slash not followed by + * '*' or '/') (tok_error) + * - eof hit. (tok_eof) */ + tok = yajl_lex_comment(lexer, jsonText, jsonTextLen, offset); + if (tok == yajl_tok_comment) { + /* "error" is silly, but that's the initial + * state of tok. guilty until proven innocent. */ + tok = yajl_tok_error; + yajl_buf_clear(lexer->buf); + lexer->bufInUse = 0; + startOffset = *offset; + break; + } + /* hit error or eof, bail */ + goto lexed; + default: + if (lexer->allowJson5 && (c == '$' || c == '_' || + (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) { + tok = yajl_lex_identifier(lexer, jsonText, jsonTextLen, offset); + } + else { + lexer->error = yajl_lex_invalid_char; + tok = yajl_tok_error; + } + goto lexed; + } + } + + lexed: + /* need to append to buffer if the buffer is in use or + * if it's an EOF token */ + if (tok == yajl_tok_eof || lexer->bufInUse) { + if (!lexer->bufInUse) yajl_buf_clear(lexer->buf); + lexer->bufInUse = 1; + yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset); + lexer->bufOff = 0; + + if (tok != yajl_tok_eof) { + *outBuf = yajl_buf_data(lexer->buf); + *outLen = yajl_buf_len(lexer->buf); + lexer->bufInUse = 0; + } + } else if (tok != yajl_tok_error) { + *outBuf = jsonText + startOffset; + *outLen = *offset - startOffset; + } + + /* For strings skip the quotes. */ + if (tok == yajl_tok_string || + tok == yajl_tok_string_with_escapes) { + assert(*outLen >= 2); + (*outBuf)++; + *outLen -= 2; + } + else if (tok == yajl_tok_identifier) { + tok = yajl_tok_string; + } #ifdef YAJL_LEXER_DEBUG if (tok == yajl_tok_error) { diff --git a/modules/libcom/src/yajl/yajl_lex.h b/modules/libcom/src/yajl/yajl_lex.h index c5eec8f00..ebe647bd2 100644 --- a/modules/libcom/src/yajl/yajl_lex.h +++ b/modules/libcom/src/yajl/yajl_lex.h @@ -41,7 +41,12 @@ typedef enum { yajl_tok_string, yajl_tok_string_with_escapes, - /* comment tokens are not currently returned to the parser, ever */ + /* These tokens are used within the lexer and never seen by the parser: */ + + /* An unquoted map key, for JSON5 only, returned as yajl_tok_string */ + yajl_tok_identifier, + + /* A comment token, never returned */ yajl_tok_comment } yajl_tok; @@ -84,6 +89,14 @@ yajl_tok yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t * offset, const unsigned char ** outBuf, size_t * outLen); +/** + * A specialized version of yajl_lex_lex for use when the next token is + * a map key, which the parser knows. + */ +yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen); + /** have a peek at the next token, but don't move the lexer forward */ yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t offset); diff --git a/modules/libcom/src/yajl/yajl_parser.c b/modules/libcom/src/yajl/yajl_parser.c index 3f62932b6..b400709a6 100644 --- a/modules/libcom/src/yajl/yajl_parser.c +++ b/modules/libcom/src/yajl/yajl_parser.c @@ -401,8 +401,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, case yajl_state_map_need_key: { /* only difference between these two states is that in * start '}' is valid, whereas in need_key, we've parsed - * a comma, and a string key _must_ follow */ - tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + * a comma, so unless this is JSON5 a key _must_ follow. */ + tok = yajl_lex_key(hand->lexer, jsonText, jsonTextLen, offset, &buf, &bufLen); switch (tok) { case yajl_tok_eof: @@ -439,7 +439,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, } default: yajl_bs_set(hand->stateStack, yajl_state_parse_error); - hand->parseError = + hand->parseError = hand->flags & yajl_allow_json5 ? + "invalid object key (must be a string or identifier)" : "invalid object key (must be a string)"; goto around_again; } diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 4d5b65d49..03f00b206 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -94,6 +94,49 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "map_identifiers", + opts => [ + -5 + ], + input => [ + "{", + " \$:1,", + " _:2,", + " A:3,", + " Z:4,", + " a:5,", + " z:6,", + " \$1:7,", + " _zz:8,", + " ZZ9\$Zalpha:9", + "}", + "" + ], + gives => [ + "map open '{'", + "key: '\$'", + "integer: 1", + "key: '_'", + "integer: 2", + "key: 'A'", + "integer: 3", + "key: 'Z'", + "integer: 4", + "key: 'a'", + "integer: 5", + "key: 'z'", + "integer: 6", + "key: '\$1'", + "integer: 7", + "key: '_zz'", + "integer: 8", + "key: 'ZZ9\$Zalpha'", + "integer: 9", + "map close '}'", + "memory leaks:\t0" + ] + }, { name => "simple_with_comments", opts => [