epics/epics-base
0
0
Fork 0
Code Issues Pull Requests Projects Wiki Activity

Accept unquoted identifiers as map keys

Browse Source 
Adds another lexer entry point for lexing map keys only,
adjust parser to use this instead of the general lexer.
Also defines another lexer token for internal use only.
This commit is contained in:
Andrew Johnson
2020-08-09 00:33:07 -05:00
parent 88e0ced03e
commit e2256d0663
4 changed files with 213 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/libcom/src/yajl/yajl_lex.c
+152 -17
View File
@@ -130,19 +130,21 @@ yajl_lex_free(yajl_lexer lxr)
return;
}
/* a lookup table which lets us quickly determine three things:
/* a lookup table which lets us quickly determine various things:
* VEC - valid escaped control char
* note. the solidus '/' may be escaped or not.
* Note: the solidus '/' may be escaped or not.
* IJC - invalid json char
* VHC - valid hex char
* NFP - needs further processing (from a string scanning perspective)
* NUC - needs utf8 checking when enabled (from a string scanning perspective)
* VIC - valid identifier char (after the first char)
*/
#define VEC 0x01
#define IJC 0x02
#define VHC 0x04
#define NFP 0x08
#define NUC 0x10
#define VIC 0x20
static const char charLookupTable[256] =
{
@@ -151,20 +153,20 @@ static const char charLookupTable[256] =
/*10*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC ,
/*18*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC ,
/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , 0 ,
/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , VIC , 0 , 0 , 0 ,
/*28*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , VEC ,
/*30*/ VHC , VHC , VHC , VHC , VHC , VHC , VHC , VHC ,
/*38*/ VHC , VHC , 0 , 0 , 0 , 0 , 0 , 0 ,
/*30*/ VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC,
/*38*/ VHC|VIC, VHC|VIC, 0 , 0 , 0 , 0 , 0 , 0 ,
/*40*/ 0 , VHC , VHC , VHC , VHC , VHC , VHC , 0 ,
/*48*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
/*50*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
/*58*/ 0 , 0 , 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 ,
/*40*/ 0 , VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VIC ,
/*48*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC ,
/*50*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC ,
/*58*/ VIC , VIC , VIC , 0 , NFP|VEC|IJC, 0 , 0 , VIC ,
/*60*/ 0 , VHC , VEC|VHC, VHC , VHC , VHC , VEC|VHC, 0 ,
/*68*/ 0 , 0 , 0 , 0 , 0 , 0 , VEC , 0 ,
/*70*/ 0 , 0 , VEC , 0 , VEC , 0 , 0 , 0 ,
/*78*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 ,
/*60*/ 0 , VHC|VIC, VEC|VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VEC|VHC|VIC, VIC,
/*68*/ VIC , VIC , VIC , VIC , VIC , VIC , VEC|VIC, VIC ,
/*70*/ VIC , VIC , VEC , VIC , VEC|VIC, VIC , VIC , VIC ,
/*78*/ VIC , VIC , VIC , 0 , 0 , 0 , 0 , 0 ,
NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC ,
NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC ,
@@ -376,14 +378,31 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText,
#define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof;
/* For both identifiers and numbers, we always have to lex one
* character too many to know when they are complete.
*/
static yajl_tok
yajl_lex_identifier(yajl_lexer lexer, const unsigned char * jsonText,
size_t jsonTextLen, size_t * offset)
{
unsigned char c;
do {
RETURN_IF_EOF;
c = readChar(lexer, jsonText, offset);
} while (charLookupTable[c] & VIC);
/* we always go "one too far" */
unreadChar(lexer, offset);
return yajl_tok_identifier;
}
static yajl_tok
yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText,
size_t jsonTextLen, size_t * offset)
{
/** XXX: numbers are the only entities in json that we must lex
* _beyond_ in order to know that they are complete. There
* is an ambiguous case for integers at EOF. */
const char hexDigits[] = "0123456789abcdefABCDEF";
unsigned char c;
int numRd = 0;
@@ -732,6 +751,122 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
*outLen -= 2;
}
#ifdef YAJL_LEXER_DEBUG
if (tok == yajl_tok_error) {
printf("lexical error: %s\n",
yajl_lex_error_to_string(yajl_lex_get_error(lexer)));
} else if (tok == yajl_tok_eof) {
printf("EOF hit\n");
} else {
printf("lexed %s: '", tokToStr(tok));
fwrite(*outBuf, 1, *outLen, stdout);
printf("'\n");
}
#endif
return tok;
}
yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText,
size_t jsonTextLen, size_t * offset,
const unsigned char ** outBuf, size_t * outLen)
{
yajl_tok tok = yajl_tok_error;
unsigned char c;
size_t startOffset = *offset;
*outBuf = NULL;
*outLen = 0;
for (;;) {
assert(*offset <= jsonTextLen);
if (*offset >= jsonTextLen) {
tok = yajl_tok_eof;
goto lexed;
}
c = readChar(lexer, jsonText, offset);
switch (c) {
case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
startOffset++;
break;
case '}':
tok = yajl_tok_right_brace;
goto lexed;
case '"': {
tok = yajl_lex_string(lexer, jsonText, jsonTextLen, offset);
goto lexed;
}
case '/':
/* If comments are disabled this is an error. */
if (!lexer->allowComments) {
unreadChar(lexer, offset);
lexer->error = yajl_lex_unallowed_comment;
tok = yajl_tok_error;
goto lexed;
}
/* Comments are enabled, so lex it.
* Possible outcomes are:
* - successful lex (tok_comment, which means continue),
* - malformed comment opening (slash not followed by
* '*' or '/') (tok_error)
* - eof hit. (tok_eof) */
tok = yajl_lex_comment(lexer, jsonText, jsonTextLen, offset);
if (tok == yajl_tok_comment) {
/* "error" is silly, but that's the initial
* state of tok. guilty until proven innocent. */
tok = yajl_tok_error;
yajl_buf_clear(lexer->buf);
lexer->bufInUse = 0;
startOffset = *offset;
break;
}
/* hit error or eof, bail */
goto lexed;
default:
if (lexer->allowJson5 && (c == '$' || c == '_' ||
(c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) {
tok = yajl_lex_identifier(lexer, jsonText, jsonTextLen, offset);
}
else {
lexer->error = yajl_lex_invalid_char;
tok = yajl_tok_error;
}
goto lexed;
}
}
lexed:
/* need to append to buffer if the buffer is in use or
* if it's an EOF token */
if (tok == yajl_tok_eof || lexer->bufInUse) {
if (!lexer->bufInUse) yajl_buf_clear(lexer->buf);
lexer->bufInUse = 1;
yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset);
lexer->bufOff = 0;
if (tok != yajl_tok_eof) {
*outBuf = yajl_buf_data(lexer->buf);
*outLen = yajl_buf_len(lexer->buf);
lexer->bufInUse = 0;
}
} else if (tok != yajl_tok_error) {
*outBuf = jsonText + startOffset;
*outLen = *offset - startOffset;
}
/* For strings skip the quotes. */
if (tok == yajl_tok_string ||
tok == yajl_tok_string_with_escapes) {
assert(*outLen >= 2);
(*outBuf)++;
*outLen -= 2;
}
else if (tok == yajl_tok_identifier) {
tok = yajl_tok_string;
}
#ifdef YAJL_LEXER_DEBUG
if (tok == yajl_tok_error) {
modules/libcom/src/yajl/yajl_lex.h
+14 -1
View File
@@ -41,7 +41,12 @@ typedef enum {
yajl_tok_string,
yajl_tok_string_with_escapes,
/* comment tokens are not currently returned to the parser, ever */
/* These tokens are used within the lexer and never seen by the parser: */
/* An unquoted map key, for JSON5 only, returned as yajl_tok_string */
yajl_tok_identifier,
/* A comment token, never returned */
yajl_tok_comment
} yajl_tok;
@@ -84,6 +89,14 @@ yajl_tok yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText,
size_t jsonTextLen, size_t * offset,
const unsigned char ** outBuf, size_t * outLen);
/**
* A specialized version of yajl_lex_lex for use when the next token is
* a map key, which the parser knows.
*/
yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText,
size_t jsonTextLen, size_t * offset,
const unsigned char ** outBuf, size_t * outLen);
/** have a peek at the next token, but don't move the lexer forward */
yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText,
size_t jsonTextLen, size_t offset);
modules/libcom/src/yajl/yajl_parser.c
+4 -3
View File
@@ -401,8 +401,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
case yajl_state_map_need_key: {
/* only difference between these two states is that in
* start '}' is valid, whereas in need_key, we've parsed
* a comma, and a string key _must_ follow */
tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen,
* a comma, so unless this is JSON5 a key _must_ follow. */
tok = yajl_lex_key(hand->lexer, jsonText, jsonTextLen,
offset, &buf, &bufLen);
switch (tok) {
case yajl_tok_eof:
@@ -439,7 +439,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText,
}
default:
yajl_bs_set(hand->stateStack, yajl_state_parse_error);
hand->parseError =
hand->parseError = hand->flags & yajl_allow_json5 ?
"invalid object key (must be a string or identifier)" :
"invalid object key (must be a string)";
goto around_again;
}
modules/libcom/test/yajlTestCases.pm
+43
View File
@@ -94,6 +94,49 @@ sub cases {
"memory leaks:\t0"
]
},
{
name => "map_identifiers",
opts => [
-5
],
input => [
"{",
" \$:1,",
" _:2,",
" A:3,",
" Z:4,",
" a:5,",
" z:6,",
" \$1:7,",
" _zz:8,",
" ZZ9\$Zalpha:9",
"}",
""
],
gives => [
"map open '{'",
"key: '\$'",
"integer: 1",
"key: '_'",
"integer: 2",
"key: 'A'",
"integer: 3",
"key: 'Z'",
"integer: 4",
"key: 'a'",
"integer: 5",
"key: 'z'",
"integer: 6",
"key: '\$1'",
"integer: 7",
"key: '_zz'",
"integer: 8",
"key: 'ZZ9\$Zalpha'",
"integer: 9",
"map close '}'",
"memory leaks:\t0"
]
},
{
name => "simple_with_comments",
opts => [