From d381a936b56a13f8b2c77b1e53f86d2e9aa49cce Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Mon, 6 Jul 2020 21:48:10 -0500 Subject: [PATCH 01/29] Fix for yajl#188 potential UB Apparently it is UB to use an enum in va_start() --- modules/libcom/src/yajl/yajl.c | 5 +++-- modules/libcom/src/yajl/yajl_gen.c | 5 +++-- modules/libcom/src/yajl/yajl_gen.h | 2 +- modules/libcom/src/yajl/yajl_parse.h | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/modules/libcom/src/yajl/yajl.c b/modules/libcom/src/yajl/yajl.c index 3fbb87aea..fcaf8accd 100644 --- a/modules/libcom/src/yajl/yajl.c +++ b/modules/libcom/src/yajl/yajl.c @@ -82,11 +82,12 @@ yajl_alloc(const yajl_callbacks * callbacks, } int -yajl_config(yajl_handle h, yajl_option opt, ...) +yajl_config(yajl_handle h, int option, ...) { + yajl_option opt = option; /* UB to use an enum in va_start */ int rv = 1; va_list ap; - va_start(ap, opt); + va_start(ap, option); switch(opt) { case yajl_allow_comments: diff --git a/modules/libcom/src/yajl/yajl_gen.c b/modules/libcom/src/yajl/yajl_gen.c index a40b9760a..80b32b3a6 100644 --- a/modules/libcom/src/yajl/yajl_gen.c +++ b/modules/libcom/src/yajl/yajl_gen.c @@ -48,11 +48,12 @@ struct yajl_gen_t }; int -yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...) +yajl_gen_config(yajl_gen g, int option, ...) { + yajl_gen_option opt = option; /* UB to use an enum in va_start */ int rv = 1; va_list ap; - va_start(ap, opt); + va_start(ap, option); switch(opt) { case yajl_gen_beautify: diff --git a/modules/libcom/src/yajl/yajl_gen.h b/modules/libcom/src/yajl/yajl_gen.h index e4a2c87a3..9afa46267 100644 --- a/modules/libcom/src/yajl/yajl_gen.h +++ b/modules/libcom/src/yajl/yajl_gen.h @@ -105,7 +105,7 @@ extern "C" { * allocation (via yajl_alloc()) * \returns zero in case of errors, non-zero otherwise */ - YAJL_API int yajl_gen_config(yajl_gen g, yajl_gen_option opt, ...); + YAJL_API int yajl_gen_config(yajl_gen hand, int option, ...); /** Allocate a generator handle * \param allocFuncs An optional pointer to a structure which allows diff --git a/modules/libcom/src/yajl/yajl_parse.h b/modules/libcom/src/yajl/yajl_parse.h index e5b565c80..f7cbbe5a5 100644 --- a/modules/libcom/src/yajl/yajl_parse.h +++ b/modules/libcom/src/yajl/yajl_parse.h @@ -162,7 +162,7 @@ extern "C" { * allocation (via yajl_alloc()) * \returns zero in case of errors, non-zero otherwise */ - YAJL_API int yajl_config(yajl_handle h, yajl_option opt, ...); + YAJL_API int yajl_config(yajl_handle h, int option, ...); /** Free a parser handle */ YAJL_API void yajl_free(yajl_handle handle); From ae604b2a5530a77a9090ded9dc9fad27934da00e Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Mon, 6 Jul 2020 23:45:47 -0500 Subject: [PATCH 02/29] Start of JSON5 support Added yajl_allow_json5 config flag, pass it around. Added -5 option to yajl_test and yajlTestConverter.pl --- modules/libcom/src/yajl/yajl.c | 7 +++++-- modules/libcom/src/yajl/yajl_lex.c | 7 ++++++- modules/libcom/src/yajl/yajl_lex.h | 3 ++- modules/libcom/src/yajl/yajl_parse.h | 8 +++++++- modules/libcom/test/yajlTestConverter.pl | 7 +++++-- modules/libcom/test/yajl_test.c | 5 ++++- 6 files changed, 29 insertions(+), 8 deletions(-) diff --git a/modules/libcom/src/yajl/yajl.c b/modules/libcom/src/yajl/yajl.c index fcaf8accd..87795b173 100644 --- a/modules/libcom/src/yajl/yajl.c +++ b/modules/libcom/src/yajl/yajl.c @@ -95,6 +95,7 @@ yajl_config(yajl_handle h, int option, ...) case yajl_allow_trailing_garbage: case yajl_allow_multiple_values: case yajl_allow_partial_values: + case yajl_allow_json5: if (va_arg(ap, int)) h->flags |= opt; else h->flags &= ~opt; break; @@ -128,7 +129,8 @@ yajl_parse(yajl_handle hand, const unsigned char * jsonText, if (hand->lexer == NULL) { hand->lexer = yajl_lex_alloc(&(hand->alloc), hand->flags & yajl_allow_comments, - !(hand->flags & yajl_dont_validate_strings)); + !(hand->flags & yajl_dont_validate_strings), + hand->flags & yajl_allow_json5); } if (hand->lexer == NULL) { return yajl_status_error; @@ -151,7 +153,8 @@ yajl_complete_parse(yajl_handle hand) if (hand->lexer == NULL) { hand->lexer = yajl_lex_alloc(&(hand->alloc), hand->flags & yajl_allow_comments, - !(hand->flags & yajl_dont_validate_strings)); + !(hand->flags & yajl_dont_validate_strings), + hand->flags & yajl_allow_json5); } return yajl_do_finish(hand); diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index e0cc3b539..b311c2b8a 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -87,6 +87,9 @@ struct yajl_lexer_t { /* shall we allow comments? */ unsigned int allowComments; + /* are we parsing JSON5? */ + unsigned int allowJson5; + /* shall we validate utf8 inside strings? */ unsigned int validateUTF8; @@ -102,7 +105,8 @@ struct yajl_lexer_t { yajl_lexer yajl_lex_alloc(yajl_alloc_funcs * alloc, - unsigned int allowComments, unsigned int validateUTF8) + unsigned int allowComments, unsigned int validateUTF8, + unsigned int allowJson5) { yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t)); if (lxr == NULL) { @@ -113,6 +117,7 @@ yajl_lex_alloc(yajl_alloc_funcs * alloc, lxr->buf = yajl_buf_alloc(alloc); lxr->allowComments = allowComments; lxr->validateUTF8 = validateUTF8; + lxr->allowJson5 = allowJson5; lxr->alloc = alloc; return lxr; } diff --git a/modules/libcom/src/yajl/yajl_lex.h b/modules/libcom/src/yajl/yajl_lex.h index 806ea1de0..7112b8765 100644 --- a/modules/libcom/src/yajl/yajl_lex.h +++ b/modules/libcom/src/yajl/yajl_lex.h @@ -53,7 +53,8 @@ extern "C" { yajl_lexer yajl_lex_alloc(yajl_alloc_funcs * alloc, unsigned int allowComments, - unsigned int validateUTF8); + unsigned int validateUTF8, + unsigned int allowJson5); void yajl_lex_free(yajl_lexer lexer); diff --git a/modules/libcom/src/yajl/yajl_parse.h b/modules/libcom/src/yajl/yajl_parse.h index f7cbbe5a5..c9a1ee216 100644 --- a/modules/libcom/src/yajl/yajl_parse.h +++ b/modules/libcom/src/yajl/yajl_parse.h @@ -155,7 +155,13 @@ extern "C" { * yajl will enter an error state (premature EOF). Setting this * flag suppresses that check and the corresponding error. */ - yajl_allow_partial_values = 0x10 + yajl_allow_partial_values = 0x10, + /** + * The JSON5 standard allows additional formats for numbers, strings + * and object keys which are not permitted in the JSON standard. + * Setting this flag enables JSON5 formats in the lexer and parser. + */ + yajl_allow_json5 = 0x20, } yajl_option; /** Allow the modification of parser options subsequent to handle diff --git a/modules/libcom/test/yajlTestConverter.pl b/modules/libcom/test/yajlTestConverter.pl index c3d2b54b6..31413a6bd 100755 --- a/modules/libcom/test/yajlTestConverter.pl +++ b/modules/libcom/test/yajlTestConverter.pl @@ -22,14 +22,17 @@ my $caseFile = 'yajlTestCases.pm'; my @cases; for my $file (@files) { - $file =~ m|/([afn][cgmp]_)?([^/]*)\.json$|; + $file =~ m|/([afn][5cgmp]_)?([^/]*)\.json$|; my $allow = $1; my $name = $2; next if $name eq ''; my $case = { name => $name }; - if ($allow eq 'ac_') { + if ($allow eq 'a5_') { + $case->{opts} = ['-5']; + } + elsif ($allow eq 'ac_') { $case->{opts} = ['-c']; } elsif ($allow eq 'ag_') { diff --git a/modules/libcom/test/yajl_test.c b/modules/libcom/test/yajl_test.c index cacfdcb5c..178ca6e04 100644 --- a/modules/libcom/test/yajl_test.c +++ b/modules/libcom/test/yajl_test.c @@ -154,6 +154,7 @@ static void usage(const char * progname) "usage: %s [options]\n" "Parse input from stdin as JSON and ouput parsing details " "to stdout\n" + " -5 allow JSON5\n" " -b set the read buffer size\n" " -c allow comments\n" " -g allow *g*arbage after valid JSON text\n" @@ -197,7 +198,9 @@ main(int argc, char ** argv) /* check arguments. We expect exactly one! */ for (i=1;i= argc) usage(argv[0]); From 91c5b2fee27b7ae2c2eca50f0f233204c62a301c Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 7 Jul 2020 00:31:14 -0500 Subject: [PATCH 03/29] Trailing commas now require a json5 parser Modifies the yajl test case to add the -5 option --- modules/libcom/src/yajl/yajl_parser.c | 6 ++-- modules/libcom/test/yajlTestCases.pm | 48 ++++++++++++++------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_parser.c b/modules/libcom/src/yajl/yajl_parser.c index cb910f79d..795378704 100644 --- a/modules/libcom/src/yajl/yajl_parser.c +++ b/modules/libcom/src/yajl/yajl_parser.c @@ -332,7 +332,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, case yajl_tok_right_bracket: { yajl_state s = yajl_bs_current(hand->stateStack); if (s == yajl_state_array_start || - s == yajl_state_array_need_val) + ((hand->flags & yajl_allow_json5) && + (s == yajl_state_array_need_val))) { if (hand->callbacks && hand->callbacks->yajl_end_array) @@ -404,7 +405,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, case yajl_tok_right_brace: { yajl_state s = yajl_bs_current(hand->stateStack); if (s == yajl_state_map_start || - s == yajl_state_map_need_key) { + ((hand->flags & yajl_allow_json5) && + (s == yajl_state_map_need_key))) { if (hand->callbacks && hand->callbacks->yajl_end_map) { _CC_CHK(hand->callbacks->yajl_end_map(hand->ctx)); } diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 40eaff6e7..fd26d9952 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -7,6 +7,31 @@ sub cases { my $VAR1 = [ + { + name => "trailing_commas", + opts => [ + -5 + ], + input => [ + "{\"array\":[1,2,],\"map\":{\"a\":1,},}", + "" + ], + gives => [ + "map open '{'", + "key: 'array'", + "array open '['", + "integer: 1", + "integer: 2", + "array close ']'", + "key: 'map'", + "map open '{'", + "key: 'a'", + "integer: 1", + "map close '}'", + "map close '}'", + "memory leaks:\t0" + ] + }, { name => "difficult_json_c_test_case_with_comments", opts => [ @@ -3000,29 +3025,6 @@ sub cases { "memory leaks:\t0" ] }, - { - name => "trailing_commas", - opts => [], - input => [ - "{\"array\":[1,2,],\"map\":{\"a\":1,},}", - "" - ], - gives => [ - "map open '{'", - "key: 'array'", - "array open '['", - "integer: 1", - "integer: 2", - "array close ']'", - "key: 'map'", - "map open '{'", - "key: 'a'", - "integer: 1", - "map close '}'", - "map close '}'", - "memory leaks:\t0" - ] - }, { name => "true", opts => [], From 456e774d857cec455bfa8ea0d241d09830805ecd Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 7 Jul 2020 00:34:50 -0500 Subject: [PATCH 04/29] JSON5: Modified lexer for some number support If configured for JSON5 the lexer now allows a leading or trailing decimal point on doubles, and an explicit leading + sign on integers or double numbers. Includes test cases. --- modules/libcom/src/yajl/yajl_lex.c | 19 ++++++++--- modules/libcom/test/yajlTestCases.pm | 47 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index b311c2b8a..781d8def0 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -117,7 +117,7 @@ yajl_lex_alloc(yajl_alloc_funcs * alloc, lxr->buf = yajl_buf_alloc(alloc); lxr->allowComments = allowComments; lxr->validateUTF8 = validateUTF8; - lxr->allowJson5 = allowJson5; + lxr->allowJson5 = !!allowJson5; lxr->alloc = alloc; return lxr; } @@ -385,27 +385,32 @@ yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, * is an ambiguous case for integers at EOF. */ unsigned char c; + int numRd = 0; yajl_tok tok = yajl_tok_integer; RETURN_IF_EOF; c = readChar(lexer, jsonText, offset); - /* optional leading minus */ - if (c == '-') { + /* optional leading plus/minus */ + if (c == '-' || (lexer->allowJson5 && c == '+')) { RETURN_IF_EOF; c = readChar(lexer, jsonText, offset); } /* a single zero, or a series of integers */ if (c == '0') { + numRd++; RETURN_IF_EOF; c = readChar(lexer, jsonText, offset); } else if (c >= '1' && c <= '9') { do { + numRd++; RETURN_IF_EOF; c = readChar(lexer, jsonText, offset); } while (c >= '0' && c <= '9'); + } else if (lexer->allowJson5 && c == '.') { + goto got_decimal; } else { unreadChar(lexer, offset); lexer->error = yajl_lex_missing_integer_after_minus; @@ -414,10 +419,10 @@ yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, /* optional fraction (indicates this is floating point) */ if (c == '.') { - int numRd = 0; - + got_decimal: RETURN_IF_EOF; c = readChar(lexer, jsonText, offset); + if (!lexer->allowJson5) numRd = 0; while (c >= '0' && c <= '9') { numRd++; @@ -607,6 +612,9 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, jsonTextLen, offset); goto lexed; } + case '+': case '.': + if (!lexer->allowJson5) + goto invalid; case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { @@ -645,6 +653,7 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, /* hit error or eof, bail */ goto lexed; default: + invalid: lexer->error = yajl_lex_invalid_char; tok = yajl_tok_error; goto lexed; diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index fd26d9952..9a651c8cd 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -7,6 +7,53 @@ sub cases { my $VAR1 = [ + { + name => "doubles", + opts => [ + -5 + ], + input => [ + "[ .1e2, 10., +3.141569, -.1e4]", + "" + ], + gives => [ + "array open '['", + "double: 10", + "double: 10", + "double: 3.14157", + "double: -1000", + "array close ']'", + "memory leaks:\t0" + ] + }, + { + name => "integers", + opts => [ + -5 + ], + input => [ + "[ +1,+2,+3,+4,+5,+6,+7,", + " +123456789 , -123456789,", + " +2147483647, -2147483647 ]", + "" + ], + gives => [ + "array open '['", + "integer: 1", + "integer: 2", + "integer: 3", + "integer: 4", + "integer: 5", + "integer: 6", + "integer: 7", + "integer: 123456789", + "integer: -123456789", + "integer: 2147483647", + "integer: -2147483647", + "array close ']'", + "memory leaks:\t0" + ] + }, { name => "trailing_commas", opts => [ From 00ac15cec5b6b5bfc542830d89e10b37403b5af9 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 7 Jul 2020 00:36:11 -0500 Subject: [PATCH 05/29] Turning on JSON5 also enables comments Includes the simple test case. --- modules/libcom/src/yajl/yajl.c | 3 ++- modules/libcom/test/yajlTestCases.pm | 31 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/modules/libcom/src/yajl/yajl.c b/modules/libcom/src/yajl/yajl.c index 87795b173..c21e163dd 100644 --- a/modules/libcom/src/yajl/yajl.c +++ b/modules/libcom/src/yajl/yajl.c @@ -90,12 +90,13 @@ yajl_config(yajl_handle h, int option, ...) va_start(ap, option); switch(opt) { + case yajl_allow_json5: + opt |= yajl_allow_comments; /* JSON5 allows comments */ case yajl_allow_comments: case yajl_dont_validate_strings: case yajl_allow_trailing_garbage: case yajl_allow_multiple_values: case yajl_allow_partial_values: - case yajl_allow_json5: if (va_arg(ap, int)) h->flags |= opt; else h->flags &= ~opt; break; diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 9a651c8cd..9b5530338 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -54,6 +54,37 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "simple_with_comments", + opts => [ + -5 + ], + input => [ + "{", + " \"this\": \"is\", // ignore this", + " \"really\": \"simple\",", + " /* ignore", + "this", + "too * / ", + "** //", + "(/", + "******/", + " \"json\": \"right?\"", + "}", + "" + ], + gives => [ + "map open '{'", + "key: 'this'", + "string: 'is'", + "key: 'really'", + "string: 'simple'", + "key: 'json'", + "string: 'right?'", + "map close '}'", + "memory leaks:\t0" + ] + }, { name => "trailing_commas", opts => [ From 98a358437f37ddfdff56079b4729fe27aef46f14 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 7 Jul 2020 23:10:07 -0500 Subject: [PATCH 06/29] Add JSON5 support for hexadecimal integers With tests for YAJL. Note yajl_parse_integer still can't handle LLONG_MIN in base 10 or 16. --- modules/libcom/src/yajl/yajl_lex.c | 33 +++++++++++++++- modules/libcom/src/yajl/yajl_lex.h | 4 +- modules/libcom/src/yajl/yajl_parser.c | 43 ++++++++++++++++----- modules/libcom/test/yajlTestCases.pm | 54 +++++++++++++++++++++++++-- 4 files changed, 118 insertions(+), 16 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index 781d8def0..2439d46a7 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -384,6 +384,7 @@ yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, * _beyond_ in order to know that they are complete. There * is an ambiguous case for integers at EOF. */ + const char hexDigits[] = "0123456789abcdefABCDEF"; unsigned char c; int numRd = 0; @@ -398,11 +399,16 @@ yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, c = readChar(lexer, jsonText, offset); } - /* a single zero, or a series of integers */ + /* a single zero, hex number, or a series of decimal digits */ if (c == '0') { numRd++; RETURN_IF_EOF; c = readChar(lexer, jsonText, offset); + if (c == 'x' || c == 'X') { + if (lexer->allowJson5) goto got_hex; + lexer->error = yajl_lex_unallowed_hex_integer; + return yajl_tok_error; + } } else if (c >= '1' && c <= '9') { do { numRd++; @@ -419,7 +425,7 @@ yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, /* optional fraction (indicates this is floating point) */ if (c == '.') { - got_decimal: + got_decimal: RETURN_IF_EOF; c = readChar(lexer, jsonText, offset); if (!lexer->allowJson5) numRd = 0; @@ -462,6 +468,25 @@ yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, tok = yajl_tok_double; } + goto end_number; + + got_hex: + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + + if (strchr(hexDigits, c)) { + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } while (strchr(hexDigits, c)); + } + else { + unreadChar(lexer, offset); + lexer->error = yajl_lex_missing_hex_digit_after_0x; + return yajl_tok_error; + } + + end_number: /* we always go "one too far" */ unreadChar(lexer, offset); @@ -736,6 +761,10 @@ yajl_lex_error_to_string(yajl_lex_error error) case yajl_lex_unallowed_comment: return "probable comment found in input text, comments are " "not enabled."; + case yajl_lex_missing_hex_digit_after_0x: + return "malformed number, a hex digit is required after the 0x/0X."; + case yajl_lex_unallowed_hex_integer: + return "probable hex number found, JSON5 is not enabled."; } return "unknown error code"; } diff --git a/modules/libcom/src/yajl/yajl_lex.h b/modules/libcom/src/yajl/yajl_lex.h index 7112b8765..87b7069f3 100644 --- a/modules/libcom/src/yajl/yajl_lex.h +++ b/modules/libcom/src/yajl/yajl_lex.h @@ -100,7 +100,9 @@ typedef enum { yajl_lex_missing_integer_after_decimal, yajl_lex_missing_integer_after_exponent, yajl_lex_missing_integer_after_minus, - yajl_lex_unallowed_comment + yajl_lex_unallowed_comment, + yajl_lex_missing_hex_digit_after_0x, + yajl_lex_unallowed_hex_integer, } yajl_lex_error; const char * yajl_lex_error_to_string(yajl_lex_error error); diff --git a/modules/libcom/src/yajl/yajl_parser.c b/modules/libcom/src/yajl/yajl_parser.c index 795378704..3f62932b6 100644 --- a/modules/libcom/src/yajl/yajl_parser.c +++ b/modules/libcom/src/yajl/yajl_parser.c @@ -34,29 +34,52 @@ #define LLONG_MIN (-0x7FFFFFFFFFFFFFFFLL - 1) #endif -#define MAX_VALUE_TO_MULTIPLY ((LLONG_MAX / 10) + (LLONG_MAX % 10)) - - /* same semantics as strtol */ long long yajl_parse_integer(const unsigned char *number, size_t length) { long long ret = 0; long sign = 1; + long base = 10; + long long max = LLONG_MAX / base; const unsigned char *pos = number; - if (*pos == '-') { pos++; sign = -1; } - if (*pos == '+') { pos++; } + const unsigned char *end = number + length; - while (pos < number + length) { - if ( ret > MAX_VALUE_TO_MULTIPLY ) { + if (*pos == '-') { + pos++; + sign = -1; + } + else if (*pos == '+') { + pos++; + } + + if (*pos == '0' && + (pos[1] == 'x' || pos[1] == 'X')) { + base = 16; + max = LLONG_MAX / base; + pos += 2; + } + + while (pos < end) { + int digit; + + if (ret > max) { errno = ERANGE; return sign == 1 ? LLONG_MAX : LLONG_MIN; } - ret *= 10; - if (LLONG_MAX - ret < (*pos - '0')) { + + ret *= base; + digit = *pos++ - '0'; + /* Don't have to check for non-digit characters, + * the lexer has already rejected any bad digits. + */ + if (digit > 9) + digit = (digit - ('A' - '0') + 10) & 0xf; + + if (LLONG_MAX - ret < digit) { errno = ERANGE; return sign == 1 ? LLONG_MAX : LLONG_MIN; } - ret += (*pos++ - '0'); + ret += digit; } return sign * ret; diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 9b5530338..0a89c97dc 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -32,9 +32,16 @@ sub cases { -5 ], input => [ - "[ +1,+2,+3,+4,+5,+6,+7,", + "[ +1,+2,+3,+4,+5,+6,+7,+8,+9,", + " 0x1,0x2,0x3,0x4,0x5,0x6,0x7,0x8,0x9,", + " 0xa,0xb,0xc,0xd,0xe,0xf,", + " 0xA,0xB,0xC,0xD,0xE,0xF,", + " +0xfedcba98, -0x6789ABCD,", " +123456789 , -123456789,", - " +2147483647, -2147483647 ]", + " +2147483647, -2147483648,", + " 0x7fffFFFFffffFFFF, -0x7FFFffffFFFFffff,", + " 9223372036854775807, -9223372036854775807", + "]", "" ], gives => [ @@ -46,10 +53,39 @@ sub cases { "integer: 5", "integer: 6", "integer: 7", + "integer: 8", + "integer: 9", + "integer: 1", + "integer: 2", + "integer: 3", + "integer: 4", + "integer: 5", + "integer: 6", + "integer: 7", + "integer: 8", + "integer: 9", + "integer: 10", + "integer: 11", + "integer: 12", + "integer: 13", + "integer: 14", + "integer: 15", + "integer: 10", + "integer: 11", + "integer: 12", + "integer: 13", + "integer: 14", + "integer: 15", + "integer: 4275878552", + "integer: -1737075661", "integer: 123456789", "integer: -123456789", "integer: 2147483647", - "integer: -2147483647", + "integer: -2147483648", + "integer: 9223372036854775807", + "integer: -9223372036854775807", + "integer: 9223372036854775807", + "integer: -9223372036854775807", "array close ']'", "memory leaks:\t0" ] @@ -2739,6 +2775,18 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "hex", + opts => [], + input => [ + "0x1", + "" + ], + gives => [ + "lexical error: probable hex number found, JSON5 is not enabled.", + "memory leaks:\t0" + ] + }, { name => "high_overflow", opts => [], From 97b8df691292d62d2b13e883c21c473e88c8ac86 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Thu, 9 Jul 2020 23:57:05 -0500 Subject: [PATCH 07/29] Added JSON5 support for parsing special numbers NaN and both Infinities, with tests. Special handling was added to yajl_test since different OSs don't always generate the same output for special numbers (nan/NaN/...). --- modules/libcom/src/yajl/yajl_lex.c | 107 ++++++++++++++++----------- modules/libcom/src/yajl/yajl_lex.h | 1 + modules/libcom/test/yajlTestCases.pm | 44 ++++++++++- modules/libcom/test/yajl_test.c | 11 ++- 4 files changed, 117 insertions(+), 46 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index 2439d46a7..49dfb7544 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -399,6 +399,25 @@ yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, c = readChar(lexer, jsonText, offset); } + if (c == 'I') { + const char * want = "nfinity"; + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + if (c != *want) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_invalid_string; + return yajl_tok_error; + } + } while (*(++want)); + if (!lexer->allowJson5) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_unallowed_special_number; + return yajl_tok_error; + } + return yajl_tok_double; + } + /* a single zero, hex number, or a series of decimal digits */ if (c == '0') { numRd++; @@ -534,6 +553,23 @@ yajl_lex_comment(yajl_lexer lexer, const unsigned char * jsonText, return tok; } +/* Macro to reduce code duplication in yajl_lex_lex() */ +#define LEX_WANT(tring) \ + const char * want = tring; \ + do { \ + if (*offset >= jsonTextLen) { \ + tok = yajl_tok_eof; \ + goto lexed; \ + } \ + c = readChar(lexer, jsonText, offset); \ + if (c != *want) { \ + unreadChar(lexer, offset); \ + lexer->error = yajl_lex_invalid_string; \ + tok = yajl_tok_error; \ + goto lexed; \ + } \ + } while (*(++want)) + yajl_tok yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t * offset, @@ -579,59 +615,42 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, startOffset++; break; case 't': { - const char * want = "rue"; - do { - if (*offset >= jsonTextLen) { - tok = yajl_tok_eof; - goto lexed; - } - c = readChar(lexer, jsonText, offset); - if (c != *want) { - unreadChar(lexer, offset); - lexer->error = yajl_lex_invalid_string; - tok = yajl_tok_error; - goto lexed; - } - } while (*(++want)); + LEX_WANT("rue"); tok = yajl_tok_bool; goto lexed; } case 'f': { - const char * want = "alse"; - do { - if (*offset >= jsonTextLen) { - tok = yajl_tok_eof; - goto lexed; - } - c = readChar(lexer, jsonText, offset); - if (c != *want) { - unreadChar(lexer, offset); - lexer->error = yajl_lex_invalid_string; - tok = yajl_tok_error; - goto lexed; - } - } while (*(++want)); + LEX_WANT("alse"); tok = yajl_tok_bool; goto lexed; } case 'n': { - const char * want = "ull"; - do { - if (*offset >= jsonTextLen) { - tok = yajl_tok_eof; - goto lexed; - } - c = readChar(lexer, jsonText, offset); - if (c != *want) { - unreadChar(lexer, offset); - lexer->error = yajl_lex_invalid_string; - tok = yajl_tok_error; - goto lexed; - } - } while (*(++want)); + LEX_WANT("ull"); tok = yajl_tok_null; goto lexed; } + case 'I': { + LEX_WANT("nfinity"); + if (!lexer->allowJson5) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_unallowed_special_number; + tok = yajl_tok_error; + } else { + tok = yajl_tok_double; + } + goto lexed; + } + case 'N': { + LEX_WANT("aN"); + if (!lexer->allowJson5) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_unallowed_special_number; + tok = yajl_tok_error; + } else { + tok = yajl_tok_double; + } + goto lexed; + } case '"': { tok = yajl_lex_string(lexer, (const unsigned char *) jsonText, jsonTextLen, offset); @@ -757,7 +776,7 @@ yajl_lex_error_to_string(yajl_lex_error error) "decimal point."; case yajl_lex_missing_integer_after_minus: return "malformed number, a digit is required after the " - "minus sign."; + "plus/minus sign."; case yajl_lex_unallowed_comment: return "probable comment found in input text, comments are " "not enabled."; @@ -765,6 +784,8 @@ yajl_lex_error_to_string(yajl_lex_error error) return "malformed number, a hex digit is required after the 0x/0X."; case yajl_lex_unallowed_hex_integer: return "probable hex number found, JSON5 is not enabled."; + case yajl_lex_unallowed_special_number: + return "special number Infinity or NaN found, JSON5 is not enabled."; } return "unknown error code"; } diff --git a/modules/libcom/src/yajl/yajl_lex.h b/modules/libcom/src/yajl/yajl_lex.h index 87b7069f3..c5eec8f00 100644 --- a/modules/libcom/src/yajl/yajl_lex.h +++ b/modules/libcom/src/yajl/yajl_lex.h @@ -103,6 +103,7 @@ typedef enum { yajl_lex_unallowed_comment, yajl_lex_missing_hex_digit_after_0x, yajl_lex_unallowed_hex_integer, + yajl_lex_unallowed_special_number, } yajl_lex_error; const char * yajl_lex_error_to_string(yajl_lex_error error); diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 0a89c97dc..4d5b65d49 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -13,7 +13,7 @@ sub cases { -5 ], input => [ - "[ .1e2, 10., +3.141569, -.1e4]", + "[ .1e2, 10., +3.141569, -.1e4, NaN, Infinity, +Infinity, -Infinity ]", "" ], gives => [ @@ -22,6 +22,10 @@ sub cases { "double: 10", "double: 3.14157", "double: -1000", + "double: NaN", + "double: Infinity", + "double: Infinity", + "double: -Infinity", "array close ']'", "memory leaks:\t0" ] @@ -2798,6 +2802,18 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "infinity", + opts => [], + input => [ + "Infinity", + "" + ], + gives => [ + "lexical error: special number Infinity or NaN found, JSON5 is not enabled.", + "memory leaks:\t0" + ] + }, { name => "integers", opts => [], @@ -2884,7 +2900,7 @@ sub cases { "string: 'blue'", "string: 'baby where are you?'", "string: 'oh boo hoo!'", - "lexical error: malformed number, a digit is required after the minus sign.", + "lexical error: malformed number, a digit is required after the plus/minus sign.", "memory leaks:\t0" ] }, @@ -2922,6 +2938,18 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "minus_infinity", + opts => [], + input => [ + "-Infinity", + "" + ], + gives => [ + "lexical error: special number Infinity or NaN found, JSON5 is not enabled.", + "memory leaks:\t0" + ] + }, { name => "missing_integer_after_decimal_point", opts => [], @@ -2962,6 +2990,18 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "nan", + opts => [], + input => [ + "NaN", + "" + ], + gives => [ + "lexical error: special number Infinity or NaN found, JSON5 is not enabled.", + "memory leaks:\t0" + ] + }, { name => "non_utf8_char_in_string", opts => [], diff --git a/modules/libcom/test/yajl_test.c b/modules/libcom/test/yajl_test.c index 178ca6e04..079366680 100644 --- a/modules/libcom/test/yajl_test.c +++ b/modules/libcom/test/yajl_test.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -85,7 +86,15 @@ static int test_yajl_integer(void *ctx, long long integerVal) static int test_yajl_double(void *ctx, double doubleVal) { - printf("double: %g\n", doubleVal); + if (isnan(doubleVal)) { + printf("double: NaN\n"); + } + else if (isinf(doubleVal)) { + printf("double: %sInfinity\n", doubleVal > 0 ? "" : "-"); + } + else { + printf("double: %g\n", doubleVal); + } return 1; } From baaf50c6d2efd0efe9a795920aa800bc1dce643d Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 00:15:15 -0500 Subject: [PATCH 08/29] Add yajl_gen_json5 option and generator support for special numbers When this flag is set, the yajl_gen_double() routine can output the values NaN, -Infinity and +Infinity. --- modules/libcom/src/yajl/yajl_gen.c | 19 +++++++++++++++++-- modules/libcom/src/yajl/yajl_gen.h | 18 ++++++++++++++---- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_gen.c b/modules/libcom/src/yajl/yajl_gen.c index 80b32b3a6..3cffc9430 100644 --- a/modules/libcom/src/yajl/yajl_gen.c +++ b/modules/libcom/src/yajl/yajl_gen.c @@ -58,6 +58,7 @@ yajl_gen_config(yajl_gen g, int option, ...) switch(opt) { case yajl_gen_beautify: case yajl_gen_validate_utf8: + case yajl_gen_json5: if (va_arg(ap, int)) g->flags |= opt; else g->flags &= ~opt; break; @@ -213,10 +214,24 @@ yajl_gen_status yajl_gen_double(yajl_gen g, double number) { char i[32]; + int special = 1; ENSURE_VALID_STATE; ENSURE_NOT_KEY; - if (isnan(number) || isinf(number)) return yajl_gen_invalid_number; + if (isnan(number)) { + strcpy(i, "NaN"); + } + else if (isinf(number)) { + sprintf(i, "%cInfinity", number < 0 ? '-' : '+'); + } + else { + special = 0; + sprintf(i, "%.20g", number); + if (strspn(i, "0123456789-") == strlen(i)) { + strcat(i, ".0"); + } + } + if (special && !(g->flags & yajl_gen_json5)) + return yajl_gen_invalid_number; INSERT_SEP; INSERT_WHITESPACE; - sprintf(i, "%.20g", number); g->print(g->ctx, i, (unsigned int)strlen(i)); APPENDED_ATOM; FINAL_NEWLINE; diff --git a/modules/libcom/src/yajl/yajl_gen.h b/modules/libcom/src/yajl/yajl_gen.h index 9afa46267..c142eeedf 100644 --- a/modules/libcom/src/yajl/yajl_gen.h +++ b/modules/libcom/src/yajl/yajl_gen.h @@ -98,7 +98,14 @@ extern "C" { * iterest of saving bytes. Setting this flag will cause YAJL to * always escape '/' in generated JSON strings. */ - yajl_gen_escape_solidus = 0x10 + yajl_gen_escape_solidus = 0x10, + /** + * Special numbers such as NaN and Infinity cannot be represented in + * the original JSON, but are permitted in JSON5. Setting this flag + * allows yajl_gen_double() to output the JSON5 representation of these + * special numbers instead of returning with an error. + */ + yajl_gen_json5 = 0x20, } yajl_gen_option; /** Allow the modification of generator options subsequent to handle @@ -122,9 +129,12 @@ extern "C" { /** Generate an integer number. */ YAJL_API yajl_gen_status yajl_gen_integer(yajl_gen hand, long long int number); - /** Generate a floating point number. \p number may not be infinity or - * NaN, as these have no representation in JSON. In these cases the - * generator will return \ref yajl_gen_invalid_number */ + /** generate a floating point number + * \param number the value to output, which may only be \c Infinity or \c NaN + * if the \ref yajl_gen_json5 flag is set, as these values + * have no legal representation in JSON. In these cases the + * generator will return \ref yajl_gen_invalid_number + */ YAJL_API yajl_gen_status yajl_gen_double(yajl_gen hand, double number); /** Generate a number from the string given in \p num. */ YAJL_API yajl_gen_status yajl_gen_number(yajl_gen hand, From 88e0ced03eb315142ab729baee423566c4850b18 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 00:26:50 -0500 Subject: [PATCH 09/29] JSON5 support for generating unquoted map keys Added a new routine to yajl_encode.c that validates bare identifiers. Use this in yajl_gen_string() to avoid quoting keys we don't have to. --- modules/libcom/src/yajl/yajl_encode.c | 24 ++++++++++++++++++++++++ modules/libcom/src/yajl/yajl_encode.h | 2 ++ modules/libcom/src/yajl/yajl_gen.c | 15 ++++++++++++--- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_encode.c b/modules/libcom/src/yajl/yajl_encode.c index dfd4af6ac..082e28d33 100644 --- a/modules/libcom/src/yajl/yajl_encode.c +++ b/modules/libcom/src/yajl/yajl_encode.c @@ -218,3 +218,27 @@ int yajl_string_validate_utf8(const unsigned char * s, size_t len) return 1; } + +int yajl_string_validate_identifier(const unsigned char * str, size_t len) +{ + const unsigned char * s = str; + int c; + + if (!len || !str) return 0; + + c = *s++; /* First character [$_A-Za-z] */ + if ((c != '$' && c < 'A') || + (c > 'Z' && c != '_' && c < 'a') || + (c > 'z')) + return 0; + + while (--len) { + c = *s++; /* Remaining characters [$_A-Za-z0-9] */ + if ((c != '$' && c < '0') || + (c > '9' && c < 'A') || + (c > 'Z' && c != '_' && c < 'a') || + (c > 'z')) + return 0; + } + return 1; +} diff --git a/modules/libcom/src/yajl/yajl_encode.h b/modules/libcom/src/yajl/yajl_encode.h index 2c0559705..cb3895f9a 100644 --- a/modules/libcom/src/yajl/yajl_encode.h +++ b/modules/libcom/src/yajl/yajl_encode.h @@ -35,6 +35,8 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str, int yajl_string_validate_utf8(const unsigned char * s, size_t len); +int yajl_string_validate_identifier(const unsigned char * str, size_t len); + #ifdef __cplusplus } #endif diff --git a/modules/libcom/src/yajl/yajl_gen.c b/modules/libcom/src/yajl/yajl_gen.c index 3cffc9430..d5f5fdcd7 100644 --- a/modules/libcom/src/yajl/yajl_gen.c +++ b/modules/libcom/src/yajl/yajl_gen.c @@ -261,9 +261,18 @@ yajl_gen_string(yajl_gen g, const unsigned char * str, } } ENSURE_VALID_STATE; INSERT_SEP; INSERT_WHITESPACE; - g->print(g->ctx, "\"", 1); - yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus); - g->print(g->ctx, "\"", 1); + if (g->flags & yajl_gen_json5 && + (g->state[g->depth] == yajl_gen_map_key || + g->state[g->depth] == yajl_gen_map_start) && + yajl_string_validate_identifier(str, len)) { + /* No need to quote this key */ + g->print(g->ctx, (const char *) str, len); + } + else { + g->print(g->ctx, "\"", 1); + yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus); + g->print(g->ctx, "\"", 1); + } APPENDED_ATOM; FINAL_NEWLINE; return yajl_gen_status_ok; From e2256d06631e067d40710b5388b10b7eac4555ff Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 00:35:26 -0500 Subject: [PATCH 10/29] Accept unquoted identifiers as map keys Adds another lexer entry point for lexing map keys only, adjust parser to use this instead of the general lexer. Also defines another lexer token for internal use only. --- modules/libcom/src/yajl/yajl_lex.c | 169 +++++++++++++++++++++++--- modules/libcom/src/yajl/yajl_lex.h | 15 ++- modules/libcom/src/yajl/yajl_parser.c | 7 +- modules/libcom/test/yajlTestCases.pm | 43 +++++++ 4 files changed, 213 insertions(+), 21 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index 49dfb7544..ad5171488 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -130,19 +130,21 @@ yajl_lex_free(yajl_lexer lxr) return; } -/* a lookup table which lets us quickly determine three things: +/* a lookup table which lets us quickly determine various things: * VEC - valid escaped control char - * note. the solidus '/' may be escaped or not. + * Note: the solidus '/' may be escaped or not. * IJC - invalid json char * VHC - valid hex char * NFP - needs further processing (from a string scanning perspective) * NUC - needs utf8 checking when enabled (from a string scanning perspective) + * VIC - valid identifier char (after the first char) */ #define VEC 0x01 #define IJC 0x02 #define VHC 0x04 #define NFP 0x08 #define NUC 0x10 +#define VIC 0x20 static const char charLookupTable[256] = { @@ -151,20 +153,20 @@ static const char charLookupTable[256] = /*10*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , /*18*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , -/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , 0 , 0 , +/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , VIC , 0 , 0 , 0 , /*28*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , VEC , -/*30*/ VHC , VHC , VHC , VHC , VHC , VHC , VHC , VHC , -/*38*/ VHC , VHC , 0 , 0 , 0 , 0 , 0 , 0 , +/*30*/ VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, +/*38*/ VHC|VIC, VHC|VIC, 0 , 0 , 0 , 0 , 0 , 0 , -/*40*/ 0 , VHC , VHC , VHC , VHC , VHC , VHC , 0 , -/*48*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , -/*50*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , -/*58*/ 0 , 0 , 0 , 0 , NFP|VEC|IJC, 0 , 0 , 0 , +/*40*/ 0 , VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VIC , +/*48*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC , +/*50*/ VIC , VIC , VIC , VIC , VIC , VIC , VIC , VIC , +/*58*/ VIC , VIC , VIC , 0 , NFP|VEC|IJC, 0 , 0 , VIC , -/*60*/ 0 , VHC , VEC|VHC, VHC , VHC , VHC , VEC|VHC, 0 , -/*68*/ 0 , 0 , 0 , 0 , 0 , 0 , VEC , 0 , -/*70*/ 0 , 0 , VEC , 0 , VEC , 0 , 0 , 0 , -/*78*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , 0 , +/*60*/ 0 , VHC|VIC, VEC|VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VEC|VHC|VIC, VIC, +/*68*/ VIC , VIC , VIC , VIC , VIC , VIC , VEC|VIC, VIC , +/*70*/ VIC , VIC , VEC , VIC , VEC|VIC, VIC , VIC , VIC , +/*78*/ VIC , VIC , VIC , 0 , 0 , 0 , 0 , 0 , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , @@ -376,14 +378,31 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, #define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof; +/* For both identifiers and numbers, we always have to lex one + * character too many to know when they are complete. + */ + +static yajl_tok +yajl_lex_identifier(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset) +{ + unsigned char c; + + do { + RETURN_IF_EOF; + c = readChar(lexer, jsonText, offset); + } while (charLookupTable[c] & VIC); + + /* we always go "one too far" */ + unreadChar(lexer, offset); + + return yajl_tok_identifier; +} + static yajl_tok yajl_lex_number(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t * offset) { - /** XXX: numbers are the only entities in json that we must lex - * _beyond_ in order to know that they are complete. There - * is an ambiguous case for integers at EOF. */ - const char hexDigits[] = "0123456789abcdefABCDEF"; unsigned char c; int numRd = 0; @@ -732,6 +751,122 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, *outLen -= 2; } +#ifdef YAJL_LEXER_DEBUG + if (tok == yajl_tok_error) { + printf("lexical error: %s\n", + yajl_lex_error_to_string(yajl_lex_get_error(lexer))); + } else if (tok == yajl_tok_eof) { + printf("EOF hit\n"); + } else { + printf("lexed %s: '", tokToStr(tok)); + fwrite(*outBuf, 1, *outLen, stdout); + printf("'\n"); + } +#endif + + return tok; +} + +yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen) +{ + yajl_tok tok = yajl_tok_error; + unsigned char c; + size_t startOffset = *offset; + + *outBuf = NULL; + *outLen = 0; + + for (;;) { + assert(*offset <= jsonTextLen); + + if (*offset >= jsonTextLen) { + tok = yajl_tok_eof; + goto lexed; + } + + c = readChar(lexer, jsonText, offset); + + switch (c) { + case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + startOffset++; + break; + case '}': + tok = yajl_tok_right_brace; + goto lexed; + case '"': { + tok = yajl_lex_string(lexer, jsonText, jsonTextLen, offset); + goto lexed; + } + case '/': + /* If comments are disabled this is an error. */ + if (!lexer->allowComments) { + unreadChar(lexer, offset); + lexer->error = yajl_lex_unallowed_comment; + tok = yajl_tok_error; + goto lexed; + } + /* Comments are enabled, so lex it. + * Possible outcomes are: + * - successful lex (tok_comment, which means continue), + * - malformed comment opening (slash not followed by + * '*' or '/') (tok_error) + * - eof hit. (tok_eof) */ + tok = yajl_lex_comment(lexer, jsonText, jsonTextLen, offset); + if (tok == yajl_tok_comment) { + /* "error" is silly, but that's the initial + * state of tok. guilty until proven innocent. */ + tok = yajl_tok_error; + yajl_buf_clear(lexer->buf); + lexer->bufInUse = 0; + startOffset = *offset; + break; + } + /* hit error or eof, bail */ + goto lexed; + default: + if (lexer->allowJson5 && (c == '$' || c == '_' || + (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) { + tok = yajl_lex_identifier(lexer, jsonText, jsonTextLen, offset); + } + else { + lexer->error = yajl_lex_invalid_char; + tok = yajl_tok_error; + } + goto lexed; + } + } + + lexed: + /* need to append to buffer if the buffer is in use or + * if it's an EOF token */ + if (tok == yajl_tok_eof || lexer->bufInUse) { + if (!lexer->bufInUse) yajl_buf_clear(lexer->buf); + lexer->bufInUse = 1; + yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset); + lexer->bufOff = 0; + + if (tok != yajl_tok_eof) { + *outBuf = yajl_buf_data(lexer->buf); + *outLen = yajl_buf_len(lexer->buf); + lexer->bufInUse = 0; + } + } else if (tok != yajl_tok_error) { + *outBuf = jsonText + startOffset; + *outLen = *offset - startOffset; + } + + /* For strings skip the quotes. */ + if (tok == yajl_tok_string || + tok == yajl_tok_string_with_escapes) { + assert(*outLen >= 2); + (*outBuf)++; + *outLen -= 2; + } + else if (tok == yajl_tok_identifier) { + tok = yajl_tok_string; + } #ifdef YAJL_LEXER_DEBUG if (tok == yajl_tok_error) { diff --git a/modules/libcom/src/yajl/yajl_lex.h b/modules/libcom/src/yajl/yajl_lex.h index c5eec8f00..ebe647bd2 100644 --- a/modules/libcom/src/yajl/yajl_lex.h +++ b/modules/libcom/src/yajl/yajl_lex.h @@ -41,7 +41,12 @@ typedef enum { yajl_tok_string, yajl_tok_string_with_escapes, - /* comment tokens are not currently returned to the parser, ever */ + /* These tokens are used within the lexer and never seen by the parser: */ + + /* An unquoted map key, for JSON5 only, returned as yajl_tok_string */ + yajl_tok_identifier, + + /* A comment token, never returned */ yajl_tok_comment } yajl_tok; @@ -84,6 +89,14 @@ yajl_tok yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t * offset, const unsigned char ** outBuf, size_t * outLen); +/** + * A specialized version of yajl_lex_lex for use when the next token is + * a map key, which the parser knows. + */ +yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, + size_t jsonTextLen, size_t * offset, + const unsigned char ** outBuf, size_t * outLen); + /** have a peek at the next token, but don't move the lexer forward */ yajl_tok yajl_lex_peek(yajl_lexer lexer, const unsigned char * jsonText, size_t jsonTextLen, size_t offset); diff --git a/modules/libcom/src/yajl/yajl_parser.c b/modules/libcom/src/yajl/yajl_parser.c index 3f62932b6..b400709a6 100644 --- a/modules/libcom/src/yajl/yajl_parser.c +++ b/modules/libcom/src/yajl/yajl_parser.c @@ -401,8 +401,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, case yajl_state_map_need_key: { /* only difference between these two states is that in * start '}' is valid, whereas in need_key, we've parsed - * a comma, and a string key _must_ follow */ - tok = yajl_lex_lex(hand->lexer, jsonText, jsonTextLen, + * a comma, so unless this is JSON5 a key _must_ follow. */ + tok = yajl_lex_key(hand->lexer, jsonText, jsonTextLen, offset, &buf, &bufLen); switch (tok) { case yajl_tok_eof: @@ -439,7 +439,8 @@ yajl_do_parse(yajl_handle hand, const unsigned char * jsonText, } default: yajl_bs_set(hand->stateStack, yajl_state_parse_error); - hand->parseError = + hand->parseError = hand->flags & yajl_allow_json5 ? + "invalid object key (must be a string or identifier)" : "invalid object key (must be a string)"; goto around_again; } diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 4d5b65d49..03f00b206 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -94,6 +94,49 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "map_identifiers", + opts => [ + -5 + ], + input => [ + "{", + " \$:1,", + " _:2,", + " A:3,", + " Z:4,", + " a:5,", + " z:6,", + " \$1:7,", + " _zz:8,", + " ZZ9\$Zalpha:9", + "}", + "" + ], + gives => [ + "map open '{'", + "key: '\$'", + "integer: 1", + "key: '_'", + "integer: 2", + "key: 'A'", + "integer: 3", + "key: 'Z'", + "integer: 4", + "key: 'a'", + "integer: 5", + "key: 'z'", + "integer: 6", + "key: '\$1'", + "integer: 7", + "key: '_zz'", + "integer: 8", + "key: 'ZZ9\$Zalpha'", + "integer: 9", + "map close '}'", + "memory leaks:\t0" + ] + }, { name => "simple_with_comments", opts => [ From 55f4e55383080ed78f0d5000d0124646f04b5c3e Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 19:21:57 -0500 Subject: [PATCH 11/29] Support for JSON5 character escapes in strings Any character other than the digits 1-9 may be preceded by a reverse solidus '\', and unless the combination has an explicitly defined expansion the character is included without the solidus. JSON5 adds \', \0 and \v to the set of defined escapes, and an escaped newline is omitted from a string. In the test case Perl uses \13 instead of \v in the output but it is the correct character (13 octal = 11 decimal = '\v'). --- modules/libcom/src/yajl/yajl_encode.c | 20 ++++++++++++++--- modules/libcom/src/yajl/yajl_lex.c | 10 ++++++++- modules/libcom/test/yajlTestCases.pm | 31 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_encode.c b/modules/libcom/src/yajl/yajl_encode.c index 082e28d33..947dce1d7 100644 --- a/modules/libcom/src/yajl/yajl_encode.c +++ b/modules/libcom/src/yajl/yajl_encode.c @@ -128,8 +128,6 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str, case 'r': unescaped = "\r"; break; case 'n': unescaped = "\n"; break; case '\\': unescaped = "\\"; break; - case '/': unescaped = "/"; break; - case '"': unescaped = "\""; break; case 'f': unescaped = "\f"; break; case 'b': unescaped = "\b"; break; case 't': unescaped = "\t"; break; @@ -165,8 +163,24 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str, break; } + /* The following escapes are only valid when parsing JSON5. + * The lexer catches them when allowJson5 is not set. + */ + case '\n': beg = ++end; continue; + case '\r': + if (str[++end] == '\n') ++end; + beg = end; + continue; + case '0': + utf8Buf[0] = '\0'; + yajl_buf_append(buf, utf8Buf, 1); + beg = ++end; + continue; + case 'v': unescaped = "\v"; break; default: - assert("this should never happen" == NULL); + utf8Buf[0] = str[end]; + utf8Buf[1] = 0; + unescaped = utf8Buf; } yajl_buf_append(buf, unescaped, (unsigned int)strlen(unescaped)); beg = ++end; diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index ad5171488..dca39a55a 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -336,12 +336,20 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, goto finish_string_lex; } } - } else if (!(charLookupTable[curChar] & VEC)) { + } + else if (lexer->allowJson5 ? (curChar >= '1' && curChar <= '9') + : !(charLookupTable[curChar] & VEC)) { /* back up to offending char */ unreadChar(lexer, offset); lexer->error = yajl_lex_string_invalid_escaped_char; goto finish_string_lex; } + else if (lexer->allowJson5 && curChar == '\r') { + STR_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if (curChar != '\n') + unreadChar(lexer, offset); + } } /* when not validating UTF8 it's a simple table lookup to determine * if the present character is invalid */ diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 03f00b206..2eef474b1 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -168,6 +168,37 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "strings", + opts => [ + -5 + ], + input => [ + "[", + " \"Hello\\!\",", + " \"\\\"Evenin\\',\\\" said the barman.\",", + " // The following string has 3 different escaped line-endings,", + " // LF, CR, and CR+LF, which all disappear from the final string.", + " \"Well \\", + "hi \\\rthere \\\r", + "y'all!\",", + " \"\\b\\f\\n\\r\\t\\v\\\\\",", + " \"\\A\\C\\/\\D\\C\",", + "]", + "" + ], + gives => [ + "array open '['", + "string: 'Hello!'", + "string: '\"Evenin',\" said the barman.'", + "string: 'Well hi there y'all!'", + "string: '\b\f", + "\r\t\13\\'", + "string: 'AC/DC'", + "array close ']'", + "memory leaks:\t0" + ] + }, { name => "trailing_commas", opts => [ From 549d6f67e32e16705343882df04aa7a3f314efca Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 5 Aug 2020 00:06:53 -0500 Subject: [PATCH 12/29] YAJL support for JSON5 \xXX hex escapes in strings, with tests Teach the lexer/parser to recognize and decode them in JSON5 mode. Teach the encoder to use them in JSON5 mode. Add another error message for bad hex digits. Test cases to show they work, and that the bad-digit check fires. --- modules/libcom/src/yajl/yajl_encode.c | 45 ++++++++++++++++++++++----- modules/libcom/src/yajl/yajl_encode.h | 3 +- modules/libcom/src/yajl/yajl_gen.c | 3 +- modules/libcom/src/yajl/yajl_lex.c | 21 +++++++++++-- modules/libcom/src/yajl/yajl_lex.h | 3 +- modules/libcom/test/yajlTestCases.pm | 29 +++++++++++++++++ 6 files changed, 91 insertions(+), 13 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_encode.c b/modules/libcom/src/yajl/yajl_encode.c index 947dce1d7..732f451f9 100644 --- a/modules/libcom/src/yajl/yajl_encode.c +++ b/modules/libcom/src/yajl/yajl_encode.c @@ -33,13 +33,22 @@ yajl_string_encode(const yajl_print_t print, void * ctx, const unsigned char * str, size_t len, - int escape_solidus) + int escape_solidus, + int output_json5) { size_t beg = 0; size_t end = 0; char hexBuf[7]; - hexBuf[0] = '\\'; hexBuf[1] = 'u'; hexBuf[2] = '0'; hexBuf[3] = '0'; - hexBuf[6] = 0; + char *hexAt; + if (output_json5) { + hexBuf[0] = '\\'; hexBuf[1] = 'x'; + hexBuf[4] = 0; + hexAt = &hexBuf[2]; + } else { + hexBuf[0] = '\\'; hexBuf[1] = 'u'; hexBuf[2] = '0'; hexBuf[3] = '0'; + hexBuf[6] = 0; + hexAt = &hexBuf[4]; + } while (end < len) { const char * escaped = NULL; @@ -57,9 +66,20 @@ yajl_string_encode(const yajl_print_t print, case '\f': escaped = "\\f"; break; case '\b': escaped = "\\b"; break; case '\t': escaped = "\\t"; break; + case '\0': + if (output_json5) { + escaped = "\\0"; break; + } + goto ashex; + case '\v': + if (output_json5) { + escaped = "\\v"; break; + } + goto ashex; default: if ((unsigned char) str[end] < 32) { - CharToHex(str[end], hexBuf + 4); + ashex: + CharToHex(str[end], hexAt); escaped = hexBuf; } break; @@ -75,10 +95,10 @@ yajl_string_encode(const yajl_print_t print, print(ctx, (const char *) (str + beg), end - beg); } -static void hexToDigit(unsigned int * val, const unsigned char * hex) +static void hexToDigit(unsigned int * val, unsigned int len, const unsigned char * hex) { unsigned int i; - for (i=0;i<4;i++) { + for (i=0;i= 'A') c = (c & ~0x20) - 7; c -= '0'; @@ -133,14 +153,14 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str, case 't': unescaped = "\t"; break; case 'u': { unsigned int codepoint = 0; - hexToDigit(&codepoint, str + ++end); + hexToDigit(&codepoint, 4, str + ++end); end+=3; /* check if this is a surrogate */ if ((codepoint & 0xFC00) == 0xD800) { end++; if (str[end] == '\\' && str[end + 1] == 'u') { unsigned int surrogate = 0; - hexToDigit(&surrogate, str + end + 2); + hexToDigit(&surrogate, 4, str + end + 2); codepoint = (((codepoint & 0x3F) << 10) | ((((codepoint >> 6) & 0xF) + 1) << 16) | @@ -177,6 +197,15 @@ void yajl_string_decode(yajl_buf buf, const unsigned char * str, beg = ++end; continue; case 'v': unescaped = "\v"; break; + case 'x': { + unsigned int codepoint = 0; + hexToDigit(&codepoint, 2, str + ++end); + end++; + utf8Buf[0] = (char) codepoint; + yajl_buf_append(buf, utf8Buf, 1); + beg = ++end; + continue; + } default: utf8Buf[0] = str[end]; utf8Buf[1] = 0; diff --git a/modules/libcom/src/yajl/yajl_encode.h b/modules/libcom/src/yajl/yajl_encode.h index cb3895f9a..fd58dec9c 100644 --- a/modules/libcom/src/yajl/yajl_encode.h +++ b/modules/libcom/src/yajl/yajl_encode.h @@ -28,7 +28,8 @@ void yajl_string_encode(const yajl_print_t printer, void * ctx, const unsigned char * str, size_t length, - int escape_solidus); + int escape_solidus, + int output_json5); void yajl_string_decode(yajl_buf buf, const unsigned char * str, size_t length); diff --git a/modules/libcom/src/yajl/yajl_gen.c b/modules/libcom/src/yajl/yajl_gen.c index d5f5fdcd7..7d86ec8c7 100644 --- a/modules/libcom/src/yajl/yajl_gen.c +++ b/modules/libcom/src/yajl/yajl_gen.c @@ -270,7 +270,8 @@ yajl_gen_string(yajl_gen g, const unsigned char * str, } else { g->print(g->ctx, "\"", 1); - yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus); + yajl_string_encode(g->print, g->ctx, str, len, g->flags & yajl_gen_escape_solidus, + g->flags & yajl_gen_json5); g->print(g->ctx, "\"", 1); } APPENDED_ATOM; diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index dca39a55a..f780a3b0a 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -332,7 +332,21 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, if (!(charLookupTable[curChar] & VHC)) { /* back up to offending char */ unreadChar(lexer, offset); - lexer->error = yajl_lex_string_invalid_hex_char; + lexer->error = yajl_lex_string_invalid_hex_u_char; + goto finish_string_lex; + } + } + } + else if (lexer->allowJson5 && curChar == 'x') { + unsigned int i = 0; + + for (i=0;i<2;i++) { + STR_CHECK_EOF; + curChar = readChar(lexer, jsonText, offset); + if (!(charLookupTable[curChar] & VHC)) { + /* back up to offending char */ + unreadChar(lexer, offset); + lexer->error = yajl_lex_string_invalid_hex_x_char; goto finish_string_lex; } } @@ -905,9 +919,12 @@ yajl_lex_error_to_string(yajl_lex_error error) "which it may not."; case yajl_lex_string_invalid_json_char: return "invalid character inside string."; - case yajl_lex_string_invalid_hex_char: + case yajl_lex_string_invalid_hex_u_char: return "invalid (non-hex) character occurs after '\\u' inside " "string."; + case yajl_lex_string_invalid_hex_x_char: + return "invalid (non-hex) character occurs after '\\x' inside " + "string."; case yajl_lex_invalid_char: return "invalid char in json text."; case yajl_lex_invalid_string: diff --git a/modules/libcom/src/yajl/yajl_lex.h b/modules/libcom/src/yajl/yajl_lex.h index ebe647bd2..7c2a6b9d8 100644 --- a/modules/libcom/src/yajl/yajl_lex.h +++ b/modules/libcom/src/yajl/yajl_lex.h @@ -107,7 +107,8 @@ typedef enum { yajl_lex_string_invalid_utf8, yajl_lex_string_invalid_escaped_char, yajl_lex_string_invalid_json_char, - yajl_lex_string_invalid_hex_char, + yajl_lex_string_invalid_hex_u_char, + yajl_lex_string_invalid_hex_x_char, yajl_lex_invalid_char, yajl_lex_invalid_string, yajl_lex_missing_integer_after_decimal, diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index 2eef474b1..f2e7a8f93 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -7,6 +7,21 @@ sub cases { my $VAR1 = [ + { + name => "codepoints_from_hex", + opts => [ + -5 + ], + input => [ + "\"\\x0a\\x07\\x21\\x40\\x7c\"", + "" + ], + gives => [ + "string: '", + "\a!\@|'", + "memory leaks:\t0" + ] + }, { name => "doubles", opts => [ @@ -94,6 +109,20 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "invalid_hex_char", + opts => [ + -5 + ], + input => [ + "\"yabba dabba do \\x1g !!\"", + "" + ], + gives => [ + "lexical error: invalid (non-hex) character occurs after '\\x' inside string.", + "memory leaks:\t0" + ] + }, { name => "map_identifiers", opts => [ From a8e0de043cbd153ba94011d7ab30e29690a0fa97 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 19:29:12 -0500 Subject: [PATCH 13/29] Support for 'single-quoted strings' Also adds missing character flag VIC for 'r'. The a5_spec_example test was copied from the JSON5 spec. --- modules/libcom/src/yajl/yajl_lex.c | 25 +++++++------ modules/libcom/test/yajlTestCases.pm | 52 ++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index f780a3b0a..e59ce5488 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -153,7 +153,7 @@ static const char charLookupTable[256] = /*10*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , /*18*/ IJC , IJC , IJC , IJC , IJC , IJC , IJC , IJC , -/*20*/ 0 , 0 , NFP|VEC|IJC, 0 , VIC , 0 , 0 , 0 , +/*20*/ 0 , 0 , NFP|VEC, 0 , VIC , 0 , 0 , NFP|VEC, /*28*/ 0 , 0 , 0 , 0 , 0 , 0 , 0 , VEC , /*30*/ VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, /*38*/ VHC|VIC, VHC|VIC, 0 , 0 , 0 , 0 , 0 , 0 , @@ -165,7 +165,7 @@ static const char charLookupTable[256] = /*60*/ 0 , VHC|VIC, VEC|VHC|VIC, VHC|VIC, VHC|VIC, VHC|VIC, VEC|VHC|VIC, VIC, /*68*/ VIC , VIC , VIC , VIC , VIC , VIC , VEC|VIC, VIC , -/*70*/ VIC , VIC , VEC , VIC , VEC|VIC, VIC , VIC , VIC , +/*70*/ VIC , VIC , VEC|VIC, VIC , VEC|VIC, VIC , VIC , VIC , /*78*/ VIC , VIC , VIC , 0 , 0 , 0 , 0 , 0 , NUC , NUC , NUC , NUC , NUC , NUC , NUC , NUC , @@ -277,7 +277,7 @@ yajl_string_scan(const unsigned char * buf, size_t len, int utf8check) static yajl_tok yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, - size_t jsonTextLen, size_t * offset) + size_t jsonTextLen, size_t * offset, const char quote) { yajl_tok tok = yajl_tok_error; int hasEscapes = 0; @@ -312,7 +312,7 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, curChar = readChar(lexer, jsonText, offset); /* quote terminates */ - if (curChar == '"') { + if (curChar == quote) { tok = yajl_tok_string; break; } @@ -406,7 +406,7 @@ yajl_lex_string(yajl_lexer lexer, const unsigned char * jsonText, static yajl_tok yajl_lex_identifier(yajl_lexer lexer, const unsigned char * jsonText, - size_t jsonTextLen, size_t * offset) + size_t jsonTextLen, size_t * offset) { unsigned char c; @@ -692,9 +692,11 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, } goto lexed; } + case '\'': + if (!lexer->allowJson5) goto invalid; + /* Fall through... */ case '"': { - tok = yajl_lex_string(lexer, (const unsigned char *) jsonText, - jsonTextLen, offset); + tok = yajl_lex_string(lexer, jsonText, jsonTextLen, offset, c); goto lexed; } case '+': case '.': @@ -705,8 +707,7 @@ yajl_lex_lex(yajl_lexer lexer, const unsigned char * jsonText, case '5': case '6': case '7': case '8': case '9': { /* integer parsing wants to start from the beginning */ unreadChar(lexer, offset); - tok = yajl_lex_number(lexer, (const unsigned char *) jsonText, - jsonTextLen, offset); + tok = yajl_lex_number(lexer, jsonText, jsonTextLen, offset); goto lexed; } case '/': @@ -817,8 +818,11 @@ yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, case '}': tok = yajl_tok_right_brace; goto lexed; + case '\'': + if (!lexer->allowJson5) goto invalid; + /* Fall through... */ case '"': { - tok = yajl_lex_string(lexer, jsonText, jsonTextLen, offset); + tok = yajl_lex_string(lexer, jsonText, jsonTextLen, offset, c); goto lexed; } case '/': @@ -853,6 +857,7 @@ yajl_tok yajl_lex_key(yajl_lexer lexer, const unsigned char * jsonText, tok = yajl_lex_identifier(lexer, jsonText, jsonTextLen, offset); } else { + invalid: lexer->error = yajl_lex_invalid_char; tok = yajl_tok_error; } diff --git a/modules/libcom/test/yajlTestCases.pm b/modules/libcom/test/yajlTestCases.pm index f2e7a8f93..7d2aff130 100644 --- a/modules/libcom/test/yajlTestCases.pm +++ b/modules/libcom/test/yajlTestCases.pm @@ -197,6 +197,54 @@ sub cases { "memory leaks:\t0" ] }, + { + name => "spec_example", + opts => [ + -5 + ], + input => [ + "{", + " // comments", + " unquoted: 'and you can quote me on that',", + " singleQuotes: 'I can use \"double quotes\" here',", + " lineBreaks: \"Look, Mom! \\", + "No \\\\n's!\",", + " hexadecimal: 0xdecaf,", + " leadingDecimalPoint: .8675309, andTrailing: 8675309.,", + " positiveSign: +1,", + " trailingComma: 'in objects', andIn: ['arrays',],", + " \"backwardsCompatible\": \"with JSON\",", + "}", + "" + ], + gives => [ + "map open '{'", + "key: 'unquoted'", + "string: 'and you can quote me on that'", + "key: 'singleQuotes'", + "string: 'I can use \"double quotes\" here'", + "key: 'lineBreaks'", + "string: 'Look, Mom! No \\n's!'", + "key: 'hexadecimal'", + "integer: 912559", + "key: 'leadingDecimalPoint'", + "double: 0.867531", + "key: 'andTrailing'", + "double: 8.67531e+06", + "key: 'positiveSign'", + "integer: 1", + "key: 'trailingComma'", + "string: 'in objects'", + "key: 'andIn'", + "array open '['", + "string: 'arrays'", + "array close ']'", + "key: 'backwardsCompatible'", + "string: 'with JSON'", + "map close '}'", + "memory leaks:\t0" + ] + }, { name => "strings", opts => [ @@ -204,7 +252,7 @@ sub cases { ], input => [ "[", - " \"Hello\\!\",", + " 'Hello\\!',", " \"\\\"Evenin\\',\\\" said the barman.\",", " // The following string has 3 different escaped line-endings,", " // LF, CR, and CR+LF, which all disappear from the final string.", @@ -212,7 +260,7 @@ sub cases { "hi \\\rthere \\\r", "y'all!\",", " \"\\b\\f\\n\\r\\t\\v\\\\\",", - " \"\\A\\C\\/\\D\\C\",", + " '\\A\\C\\/\\D\\C',", "]", "" ], From de2de5e2fd154db9af27504e6f59a6483bd2d937 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 19:34:07 -0500 Subject: [PATCH 14/29] yajl: Clean up the formatting of some C macros --- modules/libcom/src/yajl/yajl_gen.c | 31 +++++++++++++++--------------- modules/libcom/src/yajl/yajl_lex.c | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_gen.c b/modules/libcom/src/yajl/yajl_gen.c index 7d86ec8c7..05183e247 100644 --- a/modules/libcom/src/yajl/yajl_gen.c +++ b/modules/libcom/src/yajl/yajl_gen.c @@ -134,17 +134,17 @@ yajl_gen_free(yajl_gen g) } #define INSERT_SEP \ - if (g->state[g->depth] == yajl_gen_map_key || \ - g->state[g->depth] == yajl_gen_in_array) { \ - g->print(g->ctx, ",", 1); \ - if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); \ - } else if (g->state[g->depth] == yajl_gen_map_val) { \ - g->print(g->ctx, ":", 1); \ - if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, " ", 1); \ - } + if (g->state[g->depth] == yajl_gen_map_key || \ + g->state[g->depth] == yajl_gen_in_array) { \ + g->print(g->ctx, ",", 1); \ + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, "\n", 1); \ + } else if (g->state[g->depth] == yajl_gen_map_val) { \ + g->print(g->ctx, ":", 1); \ + if ((g->flags & yajl_gen_beautify)) g->print(g->ctx, " ", 1); \ + } -#define INSERT_WHITESPACE \ - if ((g->flags & yajl_gen_beautify)) { \ +#define INSERT_WHITESPACE \ + if ((g->flags & yajl_gen_beautify)) { \ if (g->state[g->depth] != yajl_gen_map_val) { \ unsigned int _i; \ for (_i=0;_idepth;_i++) \ @@ -163,8 +163,8 @@ yajl_gen_free(yajl_gen g) /* check that we're not complete, or in error state. in a valid state * to be generating */ #define ENSURE_VALID_STATE \ - if (g->state[g->depth] == yajl_gen_error) { \ - return yajl_gen_in_error_state;\ + if (g->state[g->depth] == yajl_gen_error) { \ + return yajl_gen_in_error_state; \ } else if (g->state[g->depth] == yajl_gen_complete) { \ return yajl_gen_generation_complete; \ } @@ -194,8 +194,9 @@ yajl_gen_free(yajl_gen g) break; \ } \ -#define FINAL_NEWLINE \ - if ((g->flags & yajl_gen_beautify) && g->state[g->depth] == yajl_gen_complete) \ +#define FINAL_NEWLINE \ + if ((g->flags & yajl_gen_beautify) && \ + g->state[g->depth] == yajl_gen_complete) \ g->print(g->ctx, "\n", 1); yajl_gen_status @@ -224,7 +225,7 @@ yajl_gen_double(yajl_gen g, double number) } else { special = 0; - sprintf(i, "%.20g", number); + sprintf(i, "%.17g", number); if (strspn(i, "0123456789-") == strlen(i)) { strcat(i, ".0"); } diff --git a/modules/libcom/src/yajl/yajl_lex.c b/modules/libcom/src/yajl/yajl_lex.c index e59ce5488..8dbff43da 100644 --- a/modules/libcom/src/yajl/yajl_lex.c +++ b/modules/libcom/src/yajl/yajl_lex.c @@ -200,7 +200,7 @@ static const char charLookupTable[256] = * * NOTE: on error the offset will point to the first char of the * invalid utf8 */ -#define UTF8_CHECK_EOF if (*offset >= jsonTextLen) { return yajl_tok_eof; } +#define UTF8_CHECK_EOF if (*offset >= jsonTextLen) return yajl_tok_eof; static yajl_tok yajl_lex_utf8_char(yajl_lexer lexer, const unsigned char * jsonText, From 975e9ec553e940693df1ca33dfa9f9371d79c7ef Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 20:57:38 -0500 Subject: [PATCH 15/29] Doxygen text and markup fixes and updates --- modules/libcom/src/yajl/yajl_common.h | 16 ++- modules/libcom/src/yajl/yajl_gen.h | 124 ++++++++++------- modules/libcom/src/yajl/yajl_parse.h | 183 +++++++++++++++----------- 3 files changed, 194 insertions(+), 129 deletions(-) diff --git a/modules/libcom/src/yajl/yajl_common.h b/modules/libcom/src/yajl/yajl_common.h index 57e2d7eb5..d842aa5bf 100644 --- a/modules/libcom/src/yajl/yajl_common.h +++ b/modules/libcom/src/yajl/yajl_common.h @@ -45,10 +45,12 @@ extern "C" { */ #define EPICS_YAJL_VERSION VERSION_INT(2,1,0,0) -/** Maximum generation depth for YAJL's JSON generation routines */ +/** A limit used by the generator API, YAJL_MAX_DEPTH is the maximum + * depth to which arrays and maps may be nested. + */ #define YAJL_MAX_DEPTH 128 -/** Symbol decoration for Microsoft builds */ +/** Marks a yajl routine for export from the DLL/shared library. */ #define YAJL_API LIBCOM_API /** Pointer to a malloc() function, supporting client overriding memory @@ -62,18 +64,18 @@ typedef void (*yajl_free_func)(void *ctx, void * ptr); /** Pointer to a realloc() function which can resize an allocation. */ typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz); -/** A structure which can be passed to yajl_*_alloc routines to allow the +/** A structure which can be passed to yajl_*_alloc() routines to allow the * client to specify memory allocation functions to be used. */ typedef struct { - /** Pointer to a function that can allocate uninitialized memory */ + /** Pointer to a function that can allocate uninitialized memory. */ yajl_malloc_func malloc; - /** Pointer to a function that can resize memory allocations */ + /** Pointer to a function that can resize memory allocations. */ yajl_realloc_func realloc; /** Pointer to a function that can free memory allocated using - * reallocFunction or mallocFunction */ + * the above realloc or malloc functions. */ yajl_free_func free; - /** A context pointer that will be passed to above allocation routines */ + /** A context pointer that will be passed to above allocation routines. */ void * ctx; } yajl_alloc_funcs; diff --git a/modules/libcom/src/yajl/yajl_gen.h b/modules/libcom/src/yajl/yajl_gen.h index c142eeedf..35d3f2b4d 100644 --- a/modules/libcom/src/yajl/yajl_gen.h +++ b/modules/libcom/src/yajl/yajl_gen.h @@ -28,34 +28,34 @@ #ifdef __cplusplus extern "C" { #endif - /** Generator status codes */ + /** Generator status codes. */ typedef enum { - /** No error */ + /** No error. */ yajl_gen_status_ok = 0, /** At a point where a map key is generated, a function other than - * yajl_gen_string() was called */ + * yajl_gen_string() was called. */ yajl_gen_keys_must_be_strings, - /** YAJL's maximum generation depth was exceeded. see + /** YAJL's maximum generation depth was exceeded. See * \ref YAJL_MAX_DEPTH */ yajl_max_depth_exceeded, - /** A generator function (yajl_gen_XXX) was called while in an error - * state */ + /** A yajl_gen_XXX() generator function was called while in an error + * state. */ yajl_gen_in_error_state, - /** A complete JSON document has been generated */ + /** A complete JSON document has been generated. */ yajl_gen_generation_complete, /** yajl_gen_double() was passed an invalid floating point value - * (infinity or NaN). */ + * (infinity or NaN) without \ref yajl_gen_json5 set. */ yajl_gen_invalid_number, /** A print callback was passed in, so there is no internal - * buffer to get from */ + * buffer to get from. */ yajl_gen_no_buf, - /** Returned from yajl_gen_string() when the yajl_gen_validate_utf8() - * option is enabled and an invalid was passed by client code. + /** Returned from yajl_gen_string() when the \ref yajl_gen_validate_utf8 + * option is enabled and an invalid UTF8 code was passed by client. */ yajl_gen_invalid_string } yajl_gen_status; - /** An opaque handle to a generator */ + /** An opaque handle to a generator. */ typedef struct yajl_gen_t * yajl_gen; /** A callback used for "printing" the results. */ @@ -64,76 +64,110 @@ extern "C" { size_t len); /** Configuration parameters for the parser, these may be passed to - * yajl_gen_config() along with option specific argument(s). In general, + * yajl_gen_config() along with option-specific argument(s). In general, * all configuration parameters default to *off*. */ typedef enum { - /** Generate indented (beautiful) output */ + /** Generate indented (beautiful) output. + * + * yajl_gen_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_gen_config(g, yajl_gen_beautify, 1); // Human format please + * \endcode + */ yajl_gen_beautify = 0x01, /** - * Set an indent string which is used when yajl_gen_beautify() - * is enabled. Maybe something like \\t or some number of - * spaces. The default is four spaces ' '. + * Set the indent string which is used when \ref yajl_gen_beautify + * is enabled, which may only contain whitespace characters such as + * \c \\t or some number of spaces. The default is four spaces ' '. + * + * yajl_gen_config() argument type: const char * + * + * Example: \code{.cpp} + * yajl_gen_config(g, yajl_gen_indent_string, " "); // 2 spaces + * \endcode */ yajl_gen_indent_string = 0x02, /** * Set a function and context argument that should be used to - * output generated json. the function should conform to the - * yajl_print_t() prototype while the context argument is a + * output the generated json. The function should conform to the + * \ref yajl_print_t prototype while the context argument may be any * void * of your choosing. * + * yajl_gen_config() arguments: \ref yajl_print_t, void * + * * Example: \code{.cpp} - * yajl_gen_config(g, yajl_gen_print_callback, myFunc, myVoidPtr); - * \endcode + * yajl_gen_config(g, yajl_gen_print_callback, myFunc, myVoidPtr); + * \endcode */ yajl_gen_print_callback = 0x04, /** * Normally the generator does not validate that strings you - * pass to it via yajl_gen_string() are valid UTF8. Enabling + * pass to it via yajl_gen_string() are valid UTF8. Enabling * this option will cause it to do so. + * + * yajl_gen_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_gen_config(g, yajl_gen_validate_utf8, 1); // Check UTF8 + * \endcode */ yajl_gen_validate_utf8 = 0x08, /** * The forward solidus (slash or '/' in human) is not required to be - * escaped in json text. By default, YAJL will not escape it in the - * iterest of saving bytes. Setting this flag will cause YAJL to + * escaped in JSON text. By default, YAJL will not escape it in the + * interest of saving bytes. Setting this flag will cause YAJL to * always escape '/' in generated JSON strings. + * + * yajl_gen_config() argument type: int (boolean) */ yajl_gen_escape_solidus = 0x10, /** + * JSON5 is an updated version of JSON with additional capabilities. * Special numbers such as NaN and Infinity cannot be represented in * the original JSON, but are permitted in JSON5. Setting this flag - * allows yajl_gen_double() to output the JSON5 representation of these - * special numbers instead of returning with an error. + * allows YAJL to output the JSON5 representation of these special + * numbers instead of returning with an error, and to emit map keys + * that are valid javascript identifiers without quotes. + * + * yajl_gen_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_gen_config(g, yajl_gen_json5, 1); // Output JSON5 + * \endcode */ yajl_gen_json5 = 0x20, } yajl_gen_option; - /** Allow the modification of generator options subsequent to handle - * allocation (via yajl_alloc()) - * \returns zero in case of errors, non-zero otherwise + /** Set generator options associated with a generator handle. See the + * \ref yajl_gen_option documentation for details of the available + * options and their arguments. + * \returns Zero in case of error, non-zero otherwise. */ YAJL_API int yajl_gen_config(yajl_gen hand, int option, ...); - /** Allocate a generator handle - * \param allocFuncs An optional pointer to a structure which allows - * the client to overide the memory allocation - * used by yajl. May be NULL, in which case - * malloc(), free() and realloc() will be used. + /** Allocate a generator handle. + * \param allocFuncs An optional pointer to a structure which allows the + * client to provide memory allocation functions for + * use by yajl. May be \c NULL to use the C runtime + * library's malloc(), free() and realloc(). * - * \returns an allocated handle on success, NULL on failure (bad params) + * \returns An allocated handle on success, \c NULL on failure (bad params) */ YAJL_API yajl_gen yajl_gen_alloc(const yajl_alloc_funcs * allocFuncs); - /** Free a generator handle */ + /** Free a generator handle. */ YAJL_API void yajl_gen_free(yajl_gen handle); /** Generate an integer number. */ YAJL_API yajl_gen_status yajl_gen_integer(yajl_gen hand, long long int number); - /** generate a floating point number - * \param number the value to output, which may only be \c Infinity or \c NaN - * if the \ref yajl_gen_json5 flag is set, as these values - * have no legal representation in JSON. In these cases the - * generator will return \ref yajl_gen_invalid_number + /** Generate a floating point number. + * \param hand The generator handle. + * \param number The value to output. The values \c Infinity or \c NaN are + * only accepted if the \ref yajl_gen_json5 option is set, + * as these values have no legal representation in JSON; + * the generator will return \ref yajl_gen_invalid_number + * otherwise. */ YAJL_API yajl_gen_status yajl_gen_double(yajl_gen hand, double number); /** Generate a number from the string given in \p num. */ @@ -159,16 +193,16 @@ extern "C" { /** Finish generating a JSON array. */ YAJL_API yajl_gen_status yajl_gen_array_close(yajl_gen hand); - /** Access the null terminated generator buffer. If incrementally + /** Access the zero-terminated generator buffer. If incrementally * outputing JSON, one should call yajl_gen_clear() to clear the - * buffer. This allows stream generation. */ + * buffer. This allows stream generation. */ YAJL_API yajl_gen_status yajl_gen_get_buf(yajl_gen hand, const unsigned char ** buf, size_t * len); /** Clear yajl's output buffer, but maintain all internal generation - * state. This function will not "reset" the generator state, and is - * intended to enable incremental JSON outputing. */ + * state. This function will not reset the generator state, and is + * intended to enable incremental JSON output. */ YAJL_API void yajl_gen_clear(yajl_gen hand); #ifdef __cplusplus diff --git a/modules/libcom/src/yajl/yajl_parse.h b/modules/libcom/src/yajl/yajl_parse.h index c9a1ee216..09ab546ff 100644 --- a/modules/libcom/src/yajl/yajl_parse.h +++ b/modules/libcom/src/yajl/yajl_parse.h @@ -28,42 +28,42 @@ #ifdef __cplusplus extern "C" { #endif - /** Error codes returned from this interface */ + /** Error codes returned from this interface. */ typedef enum { - /** No error was encountered */ + /** No error was encountered. */ yajl_status_ok, - /** A client callback returned zero, stopping the parse */ + /** A client callback returned zero, stopping the parse. */ yajl_status_client_canceled, - /** An error occured during the parse. Call yajl_get_error() for - * more information about the encountered error */ + /** An error occured during the parse. Call yajl_get_error() for + * more information about the encountered error. */ yajl_status_error } yajl_status; - /** Attain a human readable, english, string for an error */ + /** Return a human readable, english, string for an error. */ YAJL_API const char * yajl_status_to_string(yajl_status code); - /** An opaque handle to a parser */ + /** An opaque handle to a parser. */ typedef struct yajl_handle_t * yajl_handle; - /** YAJL is an event driven parser. This means as json elements are - * parsed, you are called back to do something with the data. The + /** YAJL is an event driven parser. This means as json elements are + * parsed, you are called back to do something with the data. The * functions in this table indicate the various events for which * you will be called back. Each callback accepts a "context" - * pointer, this is a void * that is passed into the yajl_parse() + * pointer, this is a \c void \c * that is passed into the yajl_parse() * function which the client code may use to pass around context. * - * All callbacks return an integer. If non-zero, the parse will - * continue. If zero, the parse will be canceled and - * yajl_status_client_canceled() will be returned from the parse. + * All callbacks return an integer. If non-zero, the parse will + * continue. If zero, the parse will be canceled and + * \c yajl_status_client_canceled will be returned from the parse. * * \attention * A note about the handling of numbers: - * yajl will only convert numbers that can be represented in a + * YAJL will only convert numbers that can be represented in a * double or a 64 bit (long long) int. All other numbers will * be passed to the client in string form using the yajl_number() * callback. Furthermore, if yajl_number() is not NULL, it will * always be used to return numbers, that is yajl_integer() and - * yajl_double() will be ignored. If yajl_number() is NULL but one + * yajl_double() will be ignored. If yajl_number() is NULL but one * of yajl_integer() or yajl_double() are defined, parsing of a * number larger than is representable in a double or 64 bit * integer will result in a parse error. @@ -74,12 +74,12 @@ extern "C" { int (* yajl_integer)(void * ctx, long long integerVal); int (* yajl_double)(void * ctx, double doubleVal); /** A callback which passes the string representation of the number - * back to the client. Will be used for all numbers when present */ + * back to the client. Will be used for all numbers when present. */ int (* yajl_number)(void * ctx, const char * numberVal, size_t numberLen); - /** Strings are returned as pointers into the JSON text when, - * possible, as a result, they are _not_ null padded */ + /** Strings are returned as pointers into the JSON text when + * possible. As a result they are _not_ zero-terminated. */ int (* yajl_string)(void * ctx, const unsigned char * stringVal, size_t stringLen); @@ -92,123 +92,152 @@ extern "C" { int (* yajl_end_array)(void * ctx); } yajl_callbacks; - /** Allocate a parser handle - * \param callbacks A yajl callbacks structure specifying the + /** Allocate a parser handle. + * \param callbacks A \c yajl_callbacks structure specifying the * functions to call when different JSON entities - * are encountered in the input text. May be NULL, + * are encountered in the input text. May be \c NULL, * which is only useful for validation. - * \param afs memory allocation functions, may be NULL for to use - * C runtime library routines (malloc and friends) - * \param ctx a context pointer that will be passed to callbacks. + * \param afs Memory allocation functions, pass \c NULL to use the + * C runtime library routines (malloc() and friends). + * \param ctx A context pointer that will be passed to callbacks. */ YAJL_API yajl_handle yajl_alloc(const yajl_callbacks * callbacks, yajl_alloc_funcs * afs, void * ctx); - /** Configuration parameters for the parser, these may be passed to - * yajl_config() along with option specific argument(s). In general, - * all configuration parameters default to *off*. */ + /** Configuration parameters for the parser. These should be passed to + * yajl_config() followed by any option specific argument(s). In general, + * all boolean configuration parameters default to *off*. */ typedef enum { - /** Ignore javascript style comments present in - * JSON input. Non-standard, but rather fun - * arguments: toggled off with integer zero, on otherwise. + /** + * Ignore javascript style comments present in the input. These are + * not standard in JSON but can be enabled with this. Turning on the + * \ref yajl_allow_json5 option enables this option automatically. * - * Example: \code{.cpp} - * yajl_config(h, yajl_allow_comments, 1); // turn comment support on - * \endcode + * yajl_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_config(h, yajl_allow_comments, 1); // turn comment support on + * \endcode */ yajl_allow_comments = 0x01, /** * When set the parser will verify that all strings in JSON input are - * valid UTF8 and will emit a parse error if this is not so. When set, - * this option makes parsing slightly more expensive (~7% depending - * on processor and compiler in use) + * valid UTF8, and will emit a parse error if this is not so. When set, + * this option makes parsing slightly more expensive (~7% depending on + * the processor and compiler in use) + * + * yajl_config() argument type: int (boolean) * * Example: \code{.cpp} - * yajl_config(h, yajl_dont_validate_strings, 1); // disable utf8 checking - * \endcode + * yajl_config(h, yajl_dont_validate_strings, 1); // disable utf8 checking + * \endcode */ yajl_dont_validate_strings = 0x02, /** - * By default, upon calls to yajl_complete_parse(), yajl will - * ensure the entire input text was consumed and will raise an error - * otherwise. Enabling this flag will cause yajl to disable this - * check. This can be useful when parsing json out of a that contains more - * than a single JSON document. + * By default, upon calls to yajl_complete_parse(), yajl will ensure + * the entire input text was consumed and will raise an error + * otherwise. Turning this flag on causes YAJL to disable the garbage + * check. This can be useful when parsing JSON out of an input stream + * that contains more than a single JSON document. + * + * yajl_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_config(h, yajl_allow_trailing_garbage, 1); // non-JSON follows + * \endcode */ yajl_allow_trailing_garbage = 0x04, /** - * Allow multiple values to be parsed by a single handle. The - * entire text must be valid JSON, and values can be seperated - * by any kind of whitespace. This flag will change the - * behavior of the parser, and cause it continue parsing after - * a value is parsed, rather than transitioning into a - * complete state. This option can be useful when parsing multiple - * values from an input stream. + * Allow multiple values to be parsed by a single handle. The entire + * text must be valid JSON, and values can be seperated by any kind of + * whitespace. This flag will change the behavior of the parser, and + * cause it to continue parsing after a value is parsed, rather than + * transitioning into a complete state. This option can be useful when + * parsing multiple values from an input stream. + * + * yajl_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_config(h, yajl_allow_multiple_values, 1); // multi-doc stream + * \endcode */ yajl_allow_multiple_values = 0x08, /** - * When yajl_complete_parse() is called the parser will - * check that the top level value was completely consumed. I.E., - * if called whilst in the middle of parsing a value - * yajl will enter an error state (premature EOF). Setting this - * flag suppresses that check and the corresponding error. + * When yajl_complete_parse() is called the parser will check that the + * top level value was completely consumed. If called whilst in the + * middle of parsing a value, yajl will enter an error state (premature + * EOF). Setting this flag suppresses that check and the corresponding + * error. + * + * yajl_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_config(h, yajl_allow_partial_values, 1); // might stop early + * \endcode */ yajl_allow_partial_values = 0x10, /** * The JSON5 standard allows additional formats for numbers, strings - * and object keys which are not permitted in the JSON standard. - * Setting this flag enables JSON5 formats in the lexer and parser. + * and object keys which are not permitted by the JSON standard. + * Setting this flag tells yajl to accept JSON5 standard input. + * This flag also enables \c yajl_allow_comments since comments are + * part of the JSON5 standard. + * + * yajl_config() argument type: int (boolean) + * + * Example: \code{.cpp} + * yajl_config(h, yajl_allow_json5, 1); // We accept JSON5! + * \endcode */ yajl_allow_json5 = 0x20, } yajl_option; - /** Allow the modification of parser options subsequent to handle - * allocation (via yajl_alloc()) - * \returns zero in case of errors, non-zero otherwise + /** Set parser options associated with a parser handle. See the + * \ref yajl_option documentation for details of the available options + * and their arguments. + * \returns Zero in case of error, non-zero otherwise. */ - YAJL_API int yajl_config(yajl_handle h, int option, ...); + YAJL_API int yajl_config(yajl_handle hand, int option, ...); - /** Free a parser handle */ - YAJL_API void yajl_free(yajl_handle handle); + /** Free a parser handle. */ + YAJL_API void yajl_free(yajl_handle hand); /** Parse some json! - * \param hand - a handle to the json parser allocated with yajl_alloc() - * \param jsonText - a pointer to the UTF8 json text to be parsed - * \param jsonTextLength - the length, in bytes, of input text + * \param hand A handle to the json parser allocated with yajl_alloc(). + * \param jsonText A pointer to the UTF8 json text to be parsed. + * \param jsonTextLength The length, in bytes, of input text. */ YAJL_API yajl_status yajl_parse(yajl_handle hand, const unsigned char * jsonText, size_t jsonTextLength); /** Parse any remaining buffered json. + * * Since yajl is a stream-based parser, without an explicit end of * input, yajl sometimes can't decide if content at the end of the - * stream is valid or not. For example, if "1" has been fed in, + * stream is valid or not. For example, if "1" has been received, * yajl can't know whether another digit is next or some character * that would terminate the integer token. * - * \param hand - a handle to the json parser allocated with yajl_alloc() + * \param hand a handle to the json parser allocated with yajl_alloc(). */ YAJL_API yajl_status yajl_complete_parse(yajl_handle hand); - /** Get an error string describing the state of the - * parse. + /** Get an error string describing the state of the parse. * - * If verbose is non-zero, the message will include the JSON - * text where the error occured, along with an arrow pointing to - * the specific char. + * If verbose is non-zero, the message will include the JSON text where + * the error occured, along with an arrow pointing to the specific char. * * \returns A dynamically allocated string will be returned which should - * be freed with yajl_free_error() + * be freed with yajl_free_error(). */ YAJL_API unsigned char * yajl_get_error(yajl_handle hand, int verbose, const unsigned char * jsonText, size_t jsonTextLength); - /** - * Get the amount of data consumed from the last chunk passed to YAJL. + /** Get the amount of data consumed from the last chunk passed to YAJL. * * In the case of a successful parse this can help you understand if * the entire buffer was consumed (which will allow you to handle @@ -221,7 +250,7 @@ extern "C" { */ YAJL_API size_t yajl_get_bytes_consumed(yajl_handle hand); - /** Free an error returned from yajl_get_error() */ + /** Free an error string returned from yajl_get_error(). */ YAJL_API void yajl_free_error(yajl_handle hand, unsigned char * str); #ifdef __cplusplus From c00f638f7e4ad36196454954cdc517d9cc11f01a Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Fri, 10 Jul 2020 21:00:26 -0500 Subject: [PATCH 16/29] yajl: Add yajl_gen_reset() routine For some reason this never got added with the rest of the generator. --- modules/libcom/src/yajl/yajl_gen.c | 8 ++++++++ modules/libcom/src/yajl/yajl_gen.h | 13 +++++++++++++ 2 files changed, 21 insertions(+) diff --git a/modules/libcom/src/yajl/yajl_gen.c b/modules/libcom/src/yajl/yajl_gen.c index 05183e247..6d93784f5 100644 --- a/modules/libcom/src/yajl/yajl_gen.c +++ b/modules/libcom/src/yajl/yajl_gen.c @@ -126,6 +126,14 @@ yajl_gen_alloc(const yajl_alloc_funcs * afs) return g; } +void +yajl_gen_reset(yajl_gen g, const char * sep) +{ + g->depth = 0; + memset((void *) &(g->state), 0, sizeof(g->state)); + if (sep != NULL) g->print(g->ctx, sep, strlen(sep)); +} + void yajl_gen_free(yajl_gen g) { diff --git a/modules/libcom/src/yajl/yajl_gen.h b/modules/libcom/src/yajl/yajl_gen.h index 35d3f2b4d..aba643a5a 100644 --- a/modules/libcom/src/yajl/yajl_gen.h +++ b/modules/libcom/src/yajl/yajl_gen.h @@ -205,6 +205,19 @@ extern "C" { * intended to enable incremental JSON output. */ YAJL_API void yajl_gen_clear(yajl_gen hand); + /** Reset the generator state. Allows a client to generate multiple + * JSON entities in a stream. + * \param hand The generator handle. + * \param sep This string will be inserted to separate the previously + * generated output from the following; passing \c NULL means + * *no separation* of entites (beware that generating + * multiple JSON numbers without a separator creates + * ambiguous output). + * + * Note: This call does not clear yajl's output buffer, which must be + * accomplished explicitly by calling yajl_gen_clear(). */ + YAJL_API void yajl_gen_reset(yajl_gen hand, const char * sep); + #ifdef __cplusplus } #endif From b0d78921fd43ef5f45db09c59e7bc3ac229be7db Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sun, 12 Jul 2020 23:50:01 -0500 Subject: [PATCH 17/29] Final changes to YAJL for use in Base * Bump the YAJL version number * Define a HAS_JSON5 macro * Set yajl_allow_json5 by default, fixing yajl_test and API comments --- modules/libcom/src/yajl/yajl.c | 2 +- modules/libcom/src/yajl/yajl_common.h | 20 +++++++++++++++++--- modules/libcom/src/yajl/yajl_parse.h | 3 ++- modules/libcom/test/yajl_test.c | 3 +++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/modules/libcom/src/yajl/yajl.c b/modules/libcom/src/yajl/yajl.c index c21e163dd..fdad3f68e 100644 --- a/modules/libcom/src/yajl/yajl.c +++ b/modules/libcom/src/yajl/yajl.c @@ -74,7 +74,7 @@ yajl_alloc(const yajl_callbacks * callbacks, hand->lexer = NULL; hand->bytesConsumed = 0; hand->decodeBuf = yajl_buf_alloc(&(hand->alloc)); - hand->flags = 0; + hand->flags = yajl_allow_json5 | yajl_allow_comments; yajl_bs_init(hand->stateStack, &(hand->alloc)); yajl_bs_push(hand->stateStack, yajl_state_start); diff --git a/modules/libcom/src/yajl/yajl_common.h b/modules/libcom/src/yajl/yajl_common.h index d842aa5bf..fca8454a8 100644 --- a/modules/libcom/src/yajl/yajl_common.h +++ b/modules/libcom/src/yajl/yajl_common.h @@ -37,13 +37,27 @@ extern "C" { * YAJL 1.0.12 version that was * bundled with EPICS Base 3.15.0.1 * - * YAJL 2.1.0 + * ***YAJL 2.1.0*** * \li Changes argument type for yajl_integer() from \c int to \c long \c long - * \li Changes argument type for yajl_string() and yajl_map_key() from \c unsigned to \c size_t + * \li Changes argument type for yajl_string() and yajl_map_key() from + * \c unsigned to \c size_t * \li Replacement of struct yajl_parser_config with yajl_config() * \li Replacement of yajl_parse_complete() with yajl_complete_parse() + * + * ***YAJL 2.2.0*** + * This version adds support for JSON5 parsing and generation. + * The upstream YAJL repository is no longer being maintained and the + * original author no longer responds to messsages, so while this code + * has been offered for merging into the upstream it is unlikely that + * will ever happen. The version number here is thus an EPICS-specific + * construct. */ -#define EPICS_YAJL_VERSION VERSION_INT(2,1,0,0) +#define EPICS_YAJL_VERSION VERSION_INT(2,2,0,0) + +/** A macro to make it easy to conditionally compile code that supports + * older releases. + */ +#define HAS_JSON5 /** A limit used by the generator API, YAJL_MAX_DEPTH is the maximum * depth to which arrays and maps may be nested. diff --git a/modules/libcom/src/yajl/yajl_parse.h b/modules/libcom/src/yajl/yajl_parse.h index 09ab546ff..90367d978 100644 --- a/modules/libcom/src/yajl/yajl_parse.h +++ b/modules/libcom/src/yajl/yajl_parse.h @@ -183,7 +183,8 @@ extern "C" { * and object keys which are not permitted by the JSON standard. * Setting this flag tells yajl to accept JSON5 standard input. * This flag also enables \c yajl_allow_comments since comments are - * part of the JSON5 standard. + * part of the JSON5 standard. In the EPICS build this option is + * enabled by default, it must be turned off to disable JSON5. * * yajl_config() argument type: int (boolean) * diff --git a/modules/libcom/test/yajl_test.c b/modules/libcom/test/yajl_test.c index 079366680..be92aecfd 100644 --- a/modules/libcom/test/yajl_test.c +++ b/modules/libcom/test/yajl_test.c @@ -205,6 +205,9 @@ main(int argc, char ** argv) /* allocate the parser */ hand = yajl_alloc(&callbacks, &allocFuncs, NULL); + /* turn off JSON5 (the EPICS default) */ + yajl_config(hand, yajl_allow_json5, 0); + /* check arguments. We expect exactly one! */ for (i=1;i Date: Sun, 19 Jul 2020 20:32:41 -0500 Subject: [PATCH 18/29] Drop TODOs from regression tests, hex now works in array values --- modules/database/test/std/rec/regressTest.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/database/test/std/rec/regressTest.c b/modules/database/test/std/rec/regressTest.c index 7e9cb2f73..cc7ee2986 100644 --- a/modules/database/test/std/rec/regressTest.c +++ b/modules/database/test/std/rec/regressTest.c @@ -71,7 +71,7 @@ void testArrayLength1(void) } /* - * https://bugs.launchpad.net/epics-base/+bug/1699445 + * https://bugs.launchpad.net/epics-base/+bug/1699445 and 1887981 */ static void testHexConstantLinks(void) @@ -81,7 +81,6 @@ void testHexConstantLinks(void) testdbGetFieldEqual("ai1", DBR_LONG, 0x10); testdbGetFieldEqual("li1", DBR_LONG, 0x10); testdbGetFieldEqual("mi1", DBR_LONG, 0x10); - testTodoBegin("Needs JSON5 for hex arrays"); testdbGetFieldEqual("as1.A", DBR_LONG, 0x10); testdbGetFieldEqual("as1.B", DBR_LONG, 0x10); testdbGetFieldEqual("as1.C", DBR_LONG, 0x10); @@ -90,7 +89,6 @@ void testHexConstantLinks(void) testdbGetFieldEqual("as1.F", DBR_LONG, 0x10); testdbGetFieldEqual("as1.G", DBR_LONG, 0x10); testdbGetFieldEqual("as1.H", DBR_LONG, 0x10); - testTodoEnd(); testIocShutdownOk(); testdbCleanup(); From 980711589a4a4864952c77492fddac251bb25718 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 21 Jul 2020 23:21:51 -0500 Subject: [PATCH 19/29] database/test: Give xRecord fields of all numeric types Extend dbStaticTest to check dbVerify() with all types. --- modules/database/test/ioc/db/dbStaticTest.c | 111 +++++++++++++++++++- modules/database/test/ioc/db/xRecord.dbd | 30 ++++++ 2 files changed, 136 insertions(+), 5 deletions(-) diff --git a/modules/database/test/ioc/db/dbStaticTest.c b/modules/database/test/ioc/db/dbStaticTest.c index 2598ba73d..5dabc7555 100644 --- a/modules/database/test/ioc/db/dbStaticTest.c +++ b/modules/database/test/ioc/db/dbStaticTest.c @@ -148,27 +148,128 @@ static void testDbVerify(const char *record) if (dbFindRecord(&entry, record) != 0) testAbort("Can't find record '%s'", record); - dbFindField(&entry, "UDF"); + dbFindField(&entry, "C8"); verify(&entry, "0", NULL); + verify(&entry, "-128", NULL); + verify(&entry, "127", NULL); + verify(&entry, "128", "Number too large for field type"); + verify(&entry, "0x7f", NULL); + verify(&entry, "0x80", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2345", "Extraneous characters after number"); + + dbFindField(&entry, "U8"); + verify(&entry, "0", NULL); + verify(&entry, "128", NULL); verify(&entry, "255", NULL); verify(&entry, "256", "Number too large for field type"); + verify(&entry, "0xff", NULL); verify(&entry, "0x100", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2345", "Extraneous characters after number"); - dbFindField(&entry, "PHAS"); + dbFindField(&entry, "I16"); verify(&entry, "0", NULL); verify(&entry, "-32768", NULL); verify(&entry, "-32769", "Number too large for field type"); - verify(&entry, "0x7fff", NULL); verify(&entry, "32768", "Number too large for field type"); + verify(&entry, "-0x8000", NULL); + verify(&entry, "0x7fff", NULL); + verify(&entry, "0x8000", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2345", "Extraneous characters after number"); - dbFindField(&entry, "VAL"); + dbFindField(&entry, "U16"); + verify(&entry, "0", NULL); + verify(&entry, "-32768", NULL); + verify(&entry, "-65535", NULL); + verify(&entry, "-65536", "Number too large for field type"); + verify(&entry, "65535", NULL); + verify(&entry, "0xffff", NULL); + verify(&entry, "0x10000", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2345", "Extraneous characters after number"); + + dbFindField(&entry, "I32"); verify(&entry, "0", NULL); verify(&entry, "-123456789", NULL); verify(&entry, "123456789", NULL); + verify(&entry, "-0x80000000", NULL); + verify(&entry, "-0x80000001", "Number too large for field type"); verify(&entry, "0x1234FEDC", NULL); + verify(&entry, "0x7fffffff", NULL); verify(&entry, "0x100000000", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); verify(&entry, "1.2345", "Extraneous characters after number"); + dbFindField(&entry, "U32"); + verify(&entry, "0", NULL); + verify(&entry, "-123456789", NULL); + verify(&entry, "123456789", NULL); + verify(&entry, "-0xffffffff", NULL); + verify(&entry, "-0x100000000", "Number too large for field type"); + verify(&entry, "0x1234FEDC", NULL); + verify(&entry, "0xffffffff", NULL); + verify(&entry, "0x100000000", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2345", "Extraneous characters after number"); + + dbFindField(&entry, "I64"); + verify(&entry, "0", NULL); + verify(&entry, "-1234567890123456789", NULL); + verify(&entry, "1234567890123456789", NULL); + verify(&entry, "-0x8000000000000000", NULL); + verify(&entry, "-0x8000000000000001", "Number too large for field type"); + verify(&entry, "0x123456780FEDCBA9", NULL); + verify(&entry, "0x7fffffffffffffff", NULL); + verify(&entry, "0x10000000000000000", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2345", "Extraneous characters after number"); + + dbFindField(&entry, "U64"); + verify(&entry, "0", NULL); + verify(&entry, "-1234567890123456789", NULL); + verify(&entry, "1234567890123456789", NULL); + verify(&entry, "-0xffffffffffffffff", NULL); + verify(&entry, "-0x10000000000000000", "Number too large for field type"); + verify(&entry, "0x123456780FEDCBA9", NULL); + verify(&entry, "0x7fffffffffffffff", NULL); + verify(&entry, "0x10000000000000000", "Number too large for field type"); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2345", "Extraneous characters after number"); + + dbFindField(&entry, "F32"); + verify(&entry, "0", NULL); + verify(&entry, "1.2345", NULL); + verify(&entry, ".12345", NULL); + verify(&entry, "-.12345", NULL); + verify(&entry, "+.12345", NULL); + verify(&entry, "1.e23", NULL); + verify(&entry, "1.e-23", NULL); + verify(&entry, "Infinity", NULL); + verify(&entry, "-Infinity", NULL); + verify(&entry, "+Infinity", NULL); + verify(&entry, "Nan", NULL); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2e-345", "Number too small for field type"); + verify(&entry, "1.2e345", "Number too large for field type"); + + dbFindField(&entry, "F64"); + verify(&entry, "0", NULL); + verify(&entry, "1.2345", NULL); + verify(&entry, ".12345", NULL); + verify(&entry, "-.12345", NULL); + verify(&entry, "+.12345", NULL); + verify(&entry, "1.e234", NULL); + verify(&entry, "1.e-234", NULL); + verify(&entry, "Infinity", NULL); + verify(&entry, "-Infinity", NULL); + verify(&entry, "+Infinity", NULL); + verify(&entry, "Nan", NULL); + verify(&entry, "None", "Not a valid integer"); + verify(&entry, "1.2e-345", "Number too small for field type"); + verify(&entry, "1.2e345", "Number too large for field type"); + dbFindField(&entry, "DESC"); verify(&entry, "", NULL); verify(&entry, "abcdefghijklmnopqrstuvwxyz", NULL); @@ -193,7 +294,7 @@ void dbTestIoc_registerRecordDeviceDriver(struct dbBase *); MAIN(dbStaticTest) { - testPlan(223); + testPlan(310); testdbPrepare(); testdbReadDatabase("dbTestIoc.dbd", NULL, NULL); diff --git a/modules/database/test/ioc/db/xRecord.dbd b/modules/database/test/ioc/db/xRecord.dbd index 915746a25..25013c042 100644 --- a/modules/database/test/ioc/db/xRecord.dbd +++ b/modules/database/test/ioc/db/xRecord.dbd @@ -5,6 +5,36 @@ recordtype(x) { field(VAL, DBF_LONG) { prompt("Value") } + field(C8, DBF_CHAR) { + prompt("Char") + } + field(U8, DBF_UCHAR) { + prompt("Byte") + } + field(I16, DBF_SHORT) { + prompt("Short") + } + field(U16, DBF_USHORT) { + prompt("UShort") + } + field(I32, DBF_LONG) { + prompt("Integer") + } + field(U32, DBF_ULONG) { + prompt("Unsigned") + } + field(I64, DBF_INT64) { + prompt("Long") + } + field(U64, DBF_UINT64) { + prompt("ULong") + } + field(F32, DBF_FLOAT) { + prompt("Float") + } + field(F64, DBF_DOUBLE) { + prompt("Double") + } field(LNK, DBF_INLINK) { prompt("Link") } From fa4af8b27d316075df049cc5d8a33eb886eb4527 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 21 Jul 2020 23:33:37 -0500 Subject: [PATCH 20/29] JSON5 in dbStatic: Modify lexer's number support dbLex.l acceps a leading or trailing decimal point with float/double values and an explicit leading + on all numbers. Tested in dbStaticTest.db but only passing tests. --- modules/database/src/ioc/dbStatic/dbLex.l | 11 ++++-- modules/database/test/ioc/db/dbStaticTest.db | 40 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/modules/database/src/ioc/dbStatic/dbLex.l b/modules/database/src/ioc/dbStatic/dbLex.l index 1dbd23d99..5c28044f8 100644 --- a/modules/database/src/ioc/dbStatic/dbLex.l +++ b/modules/database/src/ioc/dbStatic/dbLex.l @@ -25,10 +25,15 @@ hexdigit [0-9a-fA-F] unicodechar ({backslash}"u"{hexdigit}{4}) jsonchar ({normalchar}|{escapedchar}|{unicodechar}) jsondqstr ({doublequote}{jsonchar}*{doublequote}) -int ("-"?([0-9]|[1-9][0-9]+)) + +sign ([+-]?) +int ({sign}([0-9]|[1-9][0-9]+)) frac ("."[0-9]+) -exp ([eE][+-]?[0-9]+) -number ({int}{frac}?{exp}?) +exp ([eE]{sign}[0-9]+) +jsonnum ({int}{frac}?{exp}?) +intexp ({int}"."{exp}?) +fracexp ({sign}{frac}{exp}?) +number ({jsonnum}|{intexp}|{fracexp}) %{ #undef YY_INPUT diff --git a/modules/database/test/ioc/db/dbStaticTest.db b/modules/database/test/ioc/db/dbStaticTest.db index 63036727f..03ffaf4b9 100644 --- a/modules/database/test/ioc/db/dbStaticTest.db +++ b/modules/database/test/ioc/db/dbStaticTest.db @@ -6,3 +6,43 @@ record(x, "testrec") { alias("testrec", "testalias2") alias("testalias2", "testalias3") + +# New number formats allowed in JSON5, show dbLex/dbYacc parsing +record(x, "t1") { + field( C8, +12) + field( U8, +123) + field(I16, +1234) + field(U16, +12345) + field(I32, +123456) + field(U32, +1234567) + field(I64, +12345678) + field(U64, +123456789) + field(F32, .123) + field(F64, .123) + field(F32, 123.) + field(F64, 123.) + field(F32, +.123) + field(F64, +.123) + field(F32, +123.) + field(F64, +123.) + field(F32, -.123) + field(F64, -.123) + field(F32, -123.) + field(F64, -123.) + field(F32, .123e4) + field(F64, .123e4) + field(F32, 123.e4) + field(F64, 123.e4) + field(F32, 123.e+4) + field(F64, 123.e+4) + field(F32, 123.e-4) + field(F64, 123.e-4) + field(F32, +.123e4) + field(F64, +.123e4) + field(F32, +123.e4) + field(F64, +123.e4) + field(F32, -.123e4) + field(F64, -.123e4) + field(F32, -123.e4) + field(F64, -123.e4) +} From fe177e40fd8821953ebaac34360a97f91c12c728 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 21 Jul 2020 23:55:00 -0500 Subject: [PATCH 21/29] JSON5 in dbStatic: Add lexer support for hex integers dbLex.l accepts hex notation in JSON numbers. Tested in dbStaticTest.db as before. --- modules/database/src/ioc/dbStatic/dbLex.l | 5 +++- modules/database/test/ioc/db/dbStaticTest.db | 28 ++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/modules/database/src/ioc/dbStatic/dbLex.l b/modules/database/src/ioc/dbStatic/dbLex.l index 5c28044f8..542959231 100644 --- a/modules/database/src/ioc/dbStatic/dbLex.l +++ b/modules/database/src/ioc/dbStatic/dbLex.l @@ -33,7 +33,10 @@ exp ([eE]{sign}[0-9]+) jsonnum ({int}{frac}?{exp}?) intexp ({int}"."{exp}?) fracexp ({sign}{frac}{exp}?) -number ({jsonnum}|{intexp}|{fracexp}) + +zerox ("0x"|"0X") +hexint ({sign}{zerox}{hexdigit}+) +number ({jsonnum}|{intexp}|{fracexp}|{hexint}) %{ #undef YY_INPUT diff --git a/modules/database/test/ioc/db/dbStaticTest.db b/modules/database/test/ioc/db/dbStaticTest.db index 03ffaf4b9..546cb0bbb 100644 --- a/modules/database/test/ioc/db/dbStaticTest.db +++ b/modules/database/test/ioc/db/dbStaticTest.db @@ -17,6 +17,34 @@ record(x, "t1") { field(U32, +1234567) field(I64, +12345678) field(U64, +123456789) + field( C8, 0x1) + field( U8, 0x12) + field(I16, 0x123) + field(U16, 0x1234) + field(I32, 0x12345) + field(U32, 0x123456) + field(I64, 0x1234567) + field(U64, 0x12345678) + field( C8, +0x1) + field( U8, -0x12) + field(I16, +0x123) + field(U16, -0x1234) + field(I32, +0x12345) + field(U32, -0x123456) + field(I64, +0x1234567) + field(U64, -0x12345678) + field( C8, 0x7f) + field( U8, 0xff) + field(I16, 0x7fff) + field(I16, -0x8000) + field(U16, 0xffff) + field(I32, 0x7fffffff) + field(I32, -0x80000000) + field(U32, 0xffffffff) + field(I64, 0x7fffffffffffffff) + field(I64, -0x8000000000000000) + field(U64, 0xffffffffffffffff) + field(F32, .123) field(F64, .123) field(F32, 123.) From 0fca5fc8a9449379a274bfdba411e2a8580a83e0 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Wed, 22 Jul 2020 00:23:50 -0500 Subject: [PATCH 22/29] JSON5 in dbStatic: Lexer support for NaN and Infinity The JSON5 spec requires exact capitalization of these strings. Other numeric parsers such as strtod() are usually more lenient. --- modules/database/src/ioc/dbStatic/dbLex.l | 3 ++- modules/database/test/ioc/db/dbStaticTest.db | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/modules/database/src/ioc/dbStatic/dbLex.l b/modules/database/src/ioc/dbStatic/dbLex.l index 542959231..dd9dc8987 100644 --- a/modules/database/src/ioc/dbStatic/dbLex.l +++ b/modules/database/src/ioc/dbStatic/dbLex.l @@ -33,10 +33,11 @@ exp ([eE]{sign}[0-9]+) jsonnum ({int}{frac}?{exp}?) intexp ({int}"."{exp}?) fracexp ({sign}{frac}{exp}?) +specialnum ("NaN"|{sign}"Infinity") zerox ("0x"|"0X") hexint ({sign}{zerox}{hexdigit}+) -number ({jsonnum}|{intexp}|{fracexp}|{hexint}) +number ({jsonnum}|{intexp}|{fracexp}|{specialnum}|{hexint}) %{ #undef YY_INPUT diff --git a/modules/database/test/ioc/db/dbStaticTest.db b/modules/database/test/ioc/db/dbStaticTest.db index 546cb0bbb..75e981f1a 100644 --- a/modules/database/test/ioc/db/dbStaticTest.db +++ b/modules/database/test/ioc/db/dbStaticTest.db @@ -73,4 +73,12 @@ record(x, "t1") { field(F64, -.123e4) field(F32, -123.e4) field(F64, -123.e4) + field(F32, Infinity) + field(F64, Infinity) + field(F32, +Infinity) + field(F64, +Infinity) + field(F32, -Infinity) + field(F64, -Infinity) + field(F32, Nan) + field(F64, Nan) } From 0c800d44284c53fb32751c0adf85f9da64f50b93 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Thu, 23 Jul 2020 00:24:48 -0500 Subject: [PATCH 23/29] JSON5 in dbStatic: Update bare-word JSON keys Our bare-word character set is wider than JSON5's. Quote any keys containing the extra characters so YAJL can parse them, but don't quote keys unnecessarily. Tests for this behavior are in dbStaticTest.db Adjust the other tests that read links parsed by the dbStatic parser that used bareword keys, which are no longer quoted. --- modules/database/src/ioc/dbStatic/dbYacc.y | 17 ++++++++++++++--- modules/database/test/ioc/db/dbPutLinkTest.c | 4 ++-- modules/database/test/ioc/db/dbStaticTest.db | 3 +++ .../test/std/rec/linkRetargetLinkTest.c | 6 +++--- 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/modules/database/src/ioc/dbStatic/dbYacc.y b/modules/database/src/ioc/dbStatic/dbYacc.y index 80b031fac..cc563bcfd 100644 --- a/modules/database/src/ioc/dbStatic/dbYacc.y +++ b/modules/database/src/ioc/dbStatic/dbYacc.y @@ -32,8 +32,8 @@ static int yyAbort = 0; %token jsonNULL jsonTRUE jsonFALSE %token jsonNUMBER jsonSTRING jsonBARE -%type json_value json_object json_array -%type json_members json_pair json_elements json_string +%type json_value json_string json_object json_array +%type json_members json_pair json_key json_elements %% @@ -299,13 +299,24 @@ json_members: json_pair if (dbStaticDebug>2) printf("json %s\n", $$); }; -json_pair: json_string ':' json_value +json_pair: json_key ':' json_value { $$ = dbmfStrcat3($1, ":", $3); dbmfFree($1); dbmfFree($3); if (dbStaticDebug>2) printf("json %s\n", $$); }; +json_key: jsonSTRING + | jsonBARE +{ + /* A key containing any of these characters must be quoted for YAJL */ + if (strcspn($1, "+-.") < strlen($1)) { + $$ = dbmfStrcat3("\"", $1, "\""); + dbmfFree($1); + } + if (dbStaticDebug>2) printf("json %s\n", $$); +}; + json_string: jsonSTRING | jsonBARE { diff --git a/modules/database/test/ioc/db/dbPutLinkTest.c b/modules/database/test/ioc/db/dbPutLinkTest.c index 5158e3c91..830d37f6c 100644 --- a/modules/database/test/ioc/db/dbPutLinkTest.c +++ b/modules/database/test/ioc/db/dbPutLinkTest.c @@ -255,7 +255,7 @@ typedef struct { } testHWDataT; static const testHWDataT testHWData[] = { - {"rJSON_LINK", JSON_LINK, "{\"x\":true}", {0}, "{\"x\":true}"}, + {"rJSON_LINK", JSON_LINK, "{x:true}", {0}, "{x:true}"}, {"rVME_IO", VME_IO, "#C100 S101 @parm VME_IO", {100, 101}, "parm VME_IO"}, {"rCAMAC_IO", CAMAC_IO, "#B11 C12 N13 A14 F15 @parm CAMAC_IO", {11, 12, 13, 14, 15}, "parm CAMAC_IO"}, {"rAB_IO", AB_IO, "#L21 A22 C23 S24 @parm AB_IO", {21, 22, 23, 24}, "parm AB_IO"}, @@ -585,7 +585,7 @@ void testJLink(void) testdbPutFieldOk("j2.PROC", DBF_LONG, 1); testdbPutFieldOk("j3.PROC", DBF_LONG, 1); - testdbGetFieldEqual("j1.INP", DBF_STRING, "{\"z\":{\"good\":1}}"); + testdbGetFieldEqual("j1.INP", DBF_STRING, "{z:{good:1}}"); testdbGetFieldEqual("j1.VAL", DBF_LONG, 1); testdbGetFieldEqual("j2.VAL", DBF_LONG, 2); testdbGetFieldEqual("j3.VAL", DBF_LONG, 3); diff --git a/modules/database/test/ioc/db/dbStaticTest.db b/modules/database/test/ioc/db/dbStaticTest.db index 75e981f1a..bc25d1a23 100644 --- a/modules/database/test/ioc/db/dbStaticTest.db +++ b/modules/database/test/ioc/db/dbStaticTest.db @@ -81,4 +81,7 @@ record(x, "t1") { field(F64, -Infinity) field(F32, Nan) field(F64, Nan) + + info(i1, {x:0, +x:1, -x:2, .x:3}) + info(i2, Bare-word_string) } diff --git a/modules/database/test/std/rec/linkRetargetLinkTest.c b/modules/database/test/std/rec/linkRetargetLinkTest.c index bf98bc1f9..dab000167 100644 --- a/modules/database/test/std/rec/linkRetargetLinkTest.c +++ b/modules/database/test/std/rec/linkRetargetLinkTest.c @@ -75,18 +75,18 @@ static void testRetargetJLink(void) testdbGetFieldEqual("rec:j1", DBF_DOUBLE, 10.0); /* minimal args */ - testLongStrEq("rec:j1.INP$", "{\"calc\":{\"expr\":\"A+5\",\"args\":5}}"); + testLongStrEq("rec:j1.INP$", "{calc:{expr:\"A+5\",args:5}}"); /* with [] */ testPutLongStr("rec:j1.INP$", "{\"calc\":{\"expr\":\"A+5\",\"args\":[7]}}"); testdbPutFieldOk("rec:j1.PROC", DBF_LONG, 1); /* with const */ - testPutLongStr("rec:j1.INP$", "{\"calc\":{\"expr\":\"A+5\",\"args\":[{\"const\":7}]}}"); + testPutLongStr("rec:j1.INP$", "{calc:{expr:\"A+5\",args:[{const:7}]}}"); testdbPutFieldOk("rec:j1.PROC", DBF_LONG, 1); testdbGetFieldEqual("rec:j1", DBF_DOUBLE, 12.0); - testLongStrEq("rec:j1.INP$", "{\"calc\":{\"expr\":\"A+5\",\"args\":[{\"const\":7}]}}"); + testLongStrEq("rec:j1.INP$", "{calc:{expr:\"A+5\",args:[{const:7}]}}"); } MAIN(linkRetargetLinkTest) From 8c9e42d15ed181b4a1bea2a49637d5e1bb769625 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 28 Jul 2020 00:10:35 -0500 Subject: [PATCH 24/29] Fixed the response of epicsStrnRawFromEscaped() to numeric overflows \x only takes 2 hex digits now, and the octal parser ignores a 3rd digit if it would take the value over 0xff: "\400" => ' ' then '0' "\x088" => '\b' then '8' With additional tests. --- modules/libcom/src/misc/epicsString.c | 34 ++++++++++++++------------- modules/libcom/test/epicsStringTest.c | 26 ++++++++++++++++---- 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/modules/libcom/src/misc/epicsString.c b/modules/libcom/src/misc/epicsString.c index 951c65606..6df10c3a6 100644 --- a/modules/libcom/src/misc/epicsString.c +++ b/modules/libcom/src/misc/epicsString.c @@ -83,38 +83,40 @@ int epicsStrnRawFromEscaped(char *dst, size_t dstlen, const char *src, if (!srclen-- || !(c = *src++)) { OUT(u); goto done; } - if (c < '0' || c > '7') { + if (c < '0' || c > '7' || u > 037) { OUT(u); goto input; } u = u << 3 | (c - '0'); - - if (u > 0377) { - /* Undefined behaviour! */ - } OUT(u); } break; case 'x' : - { /* \xXXX... */ + { /* \xXX */ unsigned int u = 0; if (!srclen-- || !(c = *src++ & 0xff)) goto done; - while (isxdigit(c)) { - u = u << 4 | ((c > '9') ? toupper(c) - 'A' + 10 : c - '0'); - if (u > 0xff) { - /* Undefined behaviour! */ - } - if (!srclen-- || !(c = *src++ & 0xff)) { - OUT(u); - goto done; - } + if (!isxdigit(c)) + goto input; + + u = u << 4 | ((c > '9') ? toupper(c) - 'A' + 10 : c - '0'); + + if (!srclen-- || !(c = *src++ & 0xff)) { + OUT(u); + goto done; } + + if (!isxdigit(c)) { + OUT(u); + goto input; + } + + u = u << 4 | ((c > '9') ? toupper(c) - 'A' + 10 : c - '0'); OUT(u); - goto input; } + break; default: OUT(c); diff --git a/modules/libcom/test/epicsStringTest.c b/modules/libcom/test/epicsStringTest.c index 8bd1d5baa..0f1537f97 100644 --- a/modules/libcom/test/epicsStringTest.c +++ b/modules/libcom/test/epicsStringTest.c @@ -88,7 +88,7 @@ MAIN(epicsStringTest) char *s; int status; - testPlan(406); + testPlan(416); testChars(); @@ -257,6 +257,19 @@ MAIN(epicsStringTest) testOk(result[0] == 0123, " Octal escape (got \\%03o)", result[0]); testOk(result[status] == 0, " 0-terminated"); + memset(result, 'x', sizeof(result)); + status = epicsStrnRawFromEscaped(result, 4, "\\377", 4); + testOk(status == 1, "raw(\"\\377\", 4) -> %d (exp. 1)", status); + testOk((result[0] & 0xff) == 0377, " Octal escape (got \\%03o)", result[0]); + testOk(result[status] == 0, " 0-terminated"); + + memset(result, 'x', sizeof(result)); + status = epicsStrnRawFromEscaped(result, 4, "\\400", 4); + testOk(status == 2, "raw(\"\\400\", 4) -> %d (exp. 2)", status); + testOk(result[0] == 040, " Octal escape (got \\%03o)", result[0]); + testOk(result[1] == '0', " Terminator char got '%c'", result[1]); + testOk(result[status] == 0, " 0-terminated"); + memset(result, 'x', sizeof(result)); status = epicsStrnRawFromEscaped(result, 4, "\\812", 2); testOk(status == 1, "raw(\"\\812\", 2) -> %d (exp. 1)", status); @@ -307,14 +320,17 @@ MAIN(epicsStringTest) memset(result, 'x', sizeof(result)); status = epicsStrnRawFromEscaped(result, 4, "\\x012", 5); - testOk(status == 1, "raw(\"\\x012\", 5) -> %d (exp. 1)", status); - testOk(result[0] == 0x12," Hex escape (got \\x%x)", result[0]); + testOk(status == 2, "raw(\"\\x012\", 5) -> %d (exp. 2)", status); + testOk(result[0] == 0x1," Hex escape (got \\x%x)", result[0]); + testOk(result[1] == '2', " Terminator char got '%c'", result[1]); testOk(result[status] == 0, " 0-terminated"); memset(result, 'x', sizeof(result)); status = epicsStrnRawFromEscaped(result, 4, "\\x0012", 6); - testOk(status == 1, "raw(\"\\x0012\", 6) -> %d (exp. 1)", status); - testOk(result[0] == 0x12," Hex escape (got \\x%x)", result[0]); + testOk(status == 3, "raw(\"\\x0012\", 6) -> %d (exp. 3)", status); + testOk(result[0] == 0," Hex escape (got \\x%x)", result[0]); + testOk(result[1] == '1', " Terminator char got '%c'", result[1]); + testOk(result[2] == '2', " Following char got '%c'", result[2]); testOk(result[status] == 0, " 0-terminated"); memset(result, 'x', sizeof(result)); From c1152f94fd81d19836ce37773209212f9e657379 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 4 Aug 2020 23:18:01 -0500 Subject: [PATCH 25/29] epicsString: Remove support for Octal character escapes \nnn These changes to the functions epicsStrnRawFromEscaped() and epicsStrnEscapedFromRaw() were prompted by the addition of JSON5 support to the dbStatic parser. * \0 now represents a zero byte * Unprintable characters are now escaped in hex Tests for the octal escapes have been removed. --- modules/libcom/src/misc/epicsString.c | 38 ++++------------ modules/libcom/test/epicsStringTest.c | 62 ++------------------------- 2 files changed, 11 insertions(+), 89 deletions(-) diff --git a/modules/libcom/src/misc/epicsString.c b/modules/libcom/src/misc/epicsString.c index 6df10c3a6..766af20b6 100644 --- a/modules/libcom/src/misc/epicsString.c +++ b/modules/libcom/src/misc/epicsString.c @@ -66,30 +66,7 @@ int epicsStrnRawFromEscaped(char *dst, size_t dstlen, const char *src, case '\\': OUT('\\'); break; case '\'': OUT('\''); break; case '\"': OUT('\"'); break; - - case '0' :case '1' :case '2' :case '3' : - case '4' :case '5' :case '6' :case '7' : - { /* \ooo */ - unsigned int u = c - '0'; - - if (!srclen-- || !(c = *src++)) { - OUT(u); goto done; - } - if (c < '0' || c > '7') { - OUT(u); goto input; - } - u = u << 3 | (c - '0'); - - if (!srclen-- || !(c = *src++)) { - OUT(u); goto done; - } - if (c < '0' || c > '7' || u > 037) { - OUT(u); goto input; - } - u = u << 3 | (c - '0'); - OUT(u); - } - break; + case '0': OUT('\0'); break; case 'x' : { /* \xXX */ @@ -139,6 +116,7 @@ int epicsStrnEscapedFromRaw(char *dst, size_t dstlen, const char *src, return -1; while (srclen--) { + static const char hex[] = "0123456789abcdef"; int c = *src++; #define OUT(chr) ndst++; if (--rem > 0) *dst++ = chr @@ -153,15 +131,15 @@ int epicsStrnEscapedFromRaw(char *dst, size_t dstlen, const char *src, case '\\': OUT('\\'); OUT('\\'); break; case '\'': OUT('\\'); OUT('\''); break; case '\"': OUT('\\'); OUT('\"'); break; + case '\0': OUT('\\'); OUT('0'); break; default: if (isprint(c & 0xff)) { OUT(c); break; } - OUT('\\'); - OUT('0' + ((c & 0300) >> 6)); - OUT('0' + ((c & 0070) >> 3)); - OUT('0' + (c & 0007)); + OUT('\\'); OUT('x'); + OUT(hex[(c >> 4) & 0x0f]); + OUT(hex[ c & 0x0f]); } #undef OUT } @@ -180,7 +158,7 @@ size_t epicsStrnEscapedFromRawSize(const char *src, size_t srclen) switch (c) { case '\a': case '\b': case '\f': case '\n': case '\r': case '\t': case '\v': case '\\': - case '\'': case '\"': + case '\'': case '\"': case '\0': ndst++; break; default: @@ -260,7 +238,7 @@ int epicsStrPrintEscaped(FILE *fp, const char *s, size_t len) if (isprint(0xff & (int)c)) nout += fprintf(fp, "%c", c); else - nout += fprintf(fp, "\\%03o", (unsigned char)c); + nout += fprintf(fp, "\\x%02x", (unsigned char)c); break; } } diff --git a/modules/libcom/test/epicsStringTest.c b/modules/libcom/test/epicsStringTest.c index 0f1537f97..054f0c043 100644 --- a/modules/libcom/test/epicsStringTest.c +++ b/modules/libcom/test/epicsStringTest.c @@ -88,7 +88,7 @@ MAIN(epicsStringTest) char *s; int status; - testPlan(416); + testPlan(387); testChars(); @@ -159,7 +159,7 @@ MAIN(epicsStringTest) status = epicsStrnEscapedFromRawSize(ABCD, 4); testOk(status == 4, "size(\"ABCD\", 4) -> %d (exp. 4)", status); status = epicsStrnEscapedFromRawSize(ABCD, 5); - testOk(status == 8, "size(\"ABCD\", 5) -> %d (exp. 8)", status); + testOk(status == 6, "size(\"ABCD\", 5) -> %d (exp. 8)", status); testDiag("Testing esc = epicsStrnEscapedFromRaw(out, 4, ...)"); @@ -177,7 +177,7 @@ MAIN(epicsStringTest) memset(result, 'x', sizeof(result)); status = epicsStrnEscapedFromRaw(result, 4, ABCD, 5); - testOk(status == 8, "esc(\"ABCD\", 5) -> %d (exp. 8)", status); + testOk(status == 6, "esc(\"ABCD\", 5) -> %d (exp. 8)", status); testOk(result[3] == 0, " 0-terminated"); testOk(result[4] == 'x', " No overrun"); @@ -234,62 +234,6 @@ MAIN(epicsStringTest) testOk(result[0] == 'A', " Char '%c' (exp. 'A')", result[0]); testOk(result[status] == 0, " 0-terminated"); - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\123", 1); - testOk(status == 0, "raw(\"\\123\", 1) -> %d (exp. 0)", status); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\123", 2); - testOk(status == 1, "raw(\"\\123\", 2) -> %d (exp. 1)", status); - testOk(result[0] == 1, " Octal escape (got \\%03o)", result[0]); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\123", 3); - testOk(status == 1, "raw(\"\\123\", 3) -> %d (exp. 1)", status); - testOk(result[0] == 012, " Octal escape (got \\%03o)", result[0]); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\123", 4); - testOk(status == 1, "raw(\"\\123\", 4) -> %d (exp. 1)", status); - testOk(result[0] == 0123, " Octal escape (got \\%03o)", result[0]); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\377", 4); - testOk(status == 1, "raw(\"\\377\", 4) -> %d (exp. 1)", status); - testOk((result[0] & 0xff) == 0377, " Octal escape (got \\%03o)", result[0]); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\400", 4); - testOk(status == 2, "raw(\"\\400\", 4) -> %d (exp. 2)", status); - testOk(result[0] == 040, " Octal escape (got \\%03o)", result[0]); - testOk(result[1] == '0', " Terminator char got '%c'", result[1]); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\812", 2); - testOk(status == 1, "raw(\"\\812\", 2) -> %d (exp. 1)", status); - testOk(result[0] == '8', " Escaped '%c')", result[0]); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\182", 3); - testOk(status == 2, "raw(\"\\182\", 3) -> %d (exp. 2)", status); - testOk(result[0] == 1, " Octal escape (got \\%03o)", result[0]); - testOk(result[1] == '8', " Terminated with '%c'", result[1]); - testOk(result[status] == 0, " 0-terminated"); - - memset(result, 'x', sizeof(result)); - status = epicsStrnRawFromEscaped(result, 4, "\\128", 4); - testOk(status == 2, "raw(\"\\128\", 4) -> %d (exp. 2)", status); - testOk(result[0] == 012, " Octal escape (got \\%03o)", result[0]); - testOk(result[1] == '8', " Terminator char got '%c'", result[1]); - testOk(result[status] == 0, " 0-terminated"); - memset(result, 'x', sizeof(result)); status = epicsStrnRawFromEscaped(result, 4, "\\x12", 1); testOk(status == 0, "raw(\"\\x12\", 1) -> %d (exp. 0)", status); From b34d3c83fcdc869eb3f879241be831b6caef78b3 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Tue, 4 Aug 2020 23:31:11 -0500 Subject: [PATCH 26/29] Add support for hex escapes to the dbStatic lexer Only translate escaped chars that are inside a jsonSTRING value. --- modules/database/src/ioc/dbStatic/dbLex.l | 5 +++-- modules/database/src/ioc/dbStatic/dbLexRoutines.c | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modules/database/src/ioc/dbStatic/dbLex.l b/modules/database/src/ioc/dbStatic/dbLex.l index dd9dc8987..4d961975b 100644 --- a/modules/database/src/ioc/dbStatic/dbLex.l +++ b/modules/database/src/ioc/dbStatic/dbLex.l @@ -20,10 +20,11 @@ bareword [a-zA-Z0-9_\-+:.\[\]<>;] punctuation [:,\[\]{}] normalchar [^"\\\0-\x1f] barechar [a-zA-Z0-9_\-+.] -escapedchar ({backslash}["\\/bfnrt]) +escapedchar ({backslash}[^ux1-9]) hexdigit [0-9a-fA-F] +latinchar ({backslash}"x"{hexdigit}{2}) unicodechar ({backslash}"u"{hexdigit}{4}) -jsonchar ({normalchar}|{escapedchar}|{unicodechar}) +jsonchar ({normalchar}|{escapedchar}|{latinchar}|{unicodechar}) jsondqstr ({doublequote}{jsonchar}*{doublequote}) sign ([+-]?) diff --git a/modules/database/src/ioc/dbStatic/dbLexRoutines.c b/modules/database/src/ioc/dbStatic/dbLexRoutines.c index 2bc429479..1ec804441 100644 --- a/modules/database/src/ioc/dbStatic/dbLexRoutines.c +++ b/modules/database/src/ioc/dbStatic/dbLexRoutines.c @@ -1172,12 +1172,14 @@ static void dbRecordField(char *name,char *value) yyerror(NULL); return; } + if (*value == '"') { /* jsonSTRING values still have their quotes */ value++; value[strlen(value) - 1] = 0; + dbTranslateEscape(value, value); /* in-place; safe & legal */ } - dbTranslateEscape(value, value); /* in-place; safe & legal */ + status = dbPutString(pdbentry,value); if (status) { char msg[128]; @@ -1203,12 +1205,14 @@ static void dbRecordInfo(char *name, char *value) if (duplicate) return; ptempListNode = (tempListNode *)ellFirst(&tempList); pdbentry = ptempListNode->item; + if (*value == '"') { /* jsonSTRING values still have their quotes */ value++; value[strlen(value) - 1] = 0; + dbTranslateEscape(value, value); /* in-place; safe & legal */ } - dbTranslateEscape(value, value); /* yuck: in-place, but safe */ + status = dbPutInfo(pdbentry,name,value); if (status) { epicsPrintf("Can't set \"%s\" info \"%s\" to \"%s\"\n", From 75b89b40bff8fe66951fb561b577dd30a2d3ce7b Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 8 Aug 2020 21:51:45 -0500 Subject: [PATCH 27/29] Support single-quoted strings in dbStatic JSON5 values Teach lexer to recognize them. Strip leading & trailing quotes from string values. Add some tests. --- modules/database/src/ioc/dbStatic/dbLex.l | 12 ++++++++---- modules/database/src/ioc/dbStatic/dbLexRoutines.c | 4 ++-- modules/database/test/ioc/db/dbPutLinkTest.c | 11 +++++++---- modules/database/test/ioc/db/dbPutLinkTestJ.db | 4 ++-- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/modules/database/src/ioc/dbStatic/dbLex.l b/modules/database/src/ioc/dbStatic/dbLex.l index 4d961975b..33185133c 100644 --- a/modules/database/src/ioc/dbStatic/dbLex.l +++ b/modules/database/src/ioc/dbStatic/dbLex.l @@ -10,6 +10,7 @@ newline "\n" backslash "\\" +singlequote "'" doublequote "\"" comment "#" whitespace [ \t\r\n] @@ -18,14 +19,17 @@ stringchar [^"\n\\] bareword [a-zA-Z0-9_\-+:.\[\]<>;] punctuation [:,\[\]{}] -normalchar [^"\\\0-\x1f] +normalchar [^"'\\\0-\x1f] barechar [a-zA-Z0-9_\-+.] escapedchar ({backslash}[^ux1-9]) hexdigit [0-9a-fA-F] latinchar ({backslash}"x"{hexdigit}{2}) unicodechar ({backslash}"u"{hexdigit}{4}) -jsonchar ({normalchar}|{escapedchar}|{latinchar}|{unicodechar}) -jsondqstr ({doublequote}{jsonchar}*{doublequote}) +jsondqchar ({normalchar}|{singlequote}|{escapedchar}|{latinchar}|{unicodechar}) +jsondqstr ({doublequote}{jsondqchar}*{doublequote}) +jsonsqchar ({normalchar}|{doublequote}|{escapedchar}|{latinchar}|{unicodechar}) +jsonsqstr ({singlequote}{jsonsqchar}*{singlequote}) +jsonstr ({jsondqstr}|{jsonsqstr}) sign ([+-]?) int ({sign}([0-9]|[1-9][0-9]+)) @@ -107,7 +111,7 @@ static int yyreset(void) {punctuation} return yytext[0]; -{jsondqstr} { +{jsonstr} { yylval.Str = dbmfStrdup((char *) yytext); return jsonSTRING; } diff --git a/modules/database/src/ioc/dbStatic/dbLexRoutines.c b/modules/database/src/ioc/dbStatic/dbLexRoutines.c index 1ec804441..ade66e347 100644 --- a/modules/database/src/ioc/dbStatic/dbLexRoutines.c +++ b/modules/database/src/ioc/dbStatic/dbLexRoutines.c @@ -1173,7 +1173,7 @@ static void dbRecordField(char *name,char *value) return; } - if (*value == '"') { + if (*value == '"' || *value == '\'') { /* jsonSTRING values still have their quotes */ value++; value[strlen(value) - 1] = 0; @@ -1206,7 +1206,7 @@ static void dbRecordInfo(char *name, char *value) ptempListNode = (tempListNode *)ellFirst(&tempList); pdbentry = ptempListNode->item; - if (*value == '"') { + if (*value == '"' || *value == '\'') { /* jsonSTRING values still have their quotes */ value++; value[strlen(value) - 1] = 0; diff --git a/modules/database/test/ioc/db/dbPutLinkTest.c b/modules/database/test/ioc/db/dbPutLinkTest.c index 830d37f6c..a1b45c8fc 100644 --- a/modules/database/test/ioc/db/dbPutLinkTest.c +++ b/modules/database/test/ioc/db/dbPutLinkTest.c @@ -67,6 +67,7 @@ static const struct testParseDataT { {" #B111 C112 N113 @cparam", {CAMAC_IO, "cparam", 0, "BCN", {111, 112, 113}}}, {" @hello world ", {INST_IO, "hello world", 0, "", /*{}*/}}, {" {\"x\":true} ", {JSON_LINK, "{\"x\":true}", 0, "", /*{}*/}}, + {" {'x':true} ", {JSON_LINK, "{'x':true}", 0, "", /*{}*/}}, {NULL} }; @@ -587,7 +588,9 @@ void testJLink(void) testdbGetFieldEqual("j1.INP", DBF_STRING, "{z:{good:1}}"); testdbGetFieldEqual("j1.VAL", DBF_LONG, 1); + testdbGetFieldEqual("j2.INP", DBF_STRING, "{\"z\":{'good':2}}"); testdbGetFieldEqual("j2.VAL", DBF_LONG, 2); + testdbGetFieldEqual("j2.TSEL", DBF_STRING, "j1.TIME NPP NMS"); testdbGetFieldEqual("j3.VAL", DBF_LONG, 3); testNumZ(6); @@ -596,7 +599,7 @@ void testJLink(void) testdbPutFieldOk("j1.PROC", DBF_LONG, 1); testdbGetFieldEqual("j1.VAL", DBF_LONG, 4); - testdbPutFieldOk("j2.TSEL", DBF_STRING, "{\"z\":{\"good\":0}}"); + testdbPutFieldOk("j2.TSEL", DBF_STRING, "{'z':{good:0}}"); testdbPutFieldOk("j2.PROC", DBF_LONG, 1); testNumZ(7); @@ -611,8 +614,8 @@ void testJLink(void) testNumZ(7); /* Check SDIS using a JSON link prevents processing */ - testdbPutFieldOk("j1.SDIS", DBF_STRING, "{\"z\":{\"good\":1}}"); - testdbPutFieldOk("j1.INP", DBF_STRING, "{\"z\":{\"good\":1}}"); + testdbPutFieldOk("j1.SDIS", DBF_STRING, "{z:{good:1}}"); + testdbPutFieldOk("j1.INP", DBF_STRING, "{z:{good:1}}"); testdbPutFieldOk("j1.PROC", DBF_LONG, 1); testdbGetFieldEqual("j1.VAL", DBF_LONG, 4); @@ -697,7 +700,7 @@ void testTSEL(void) MAIN(dbPutLinkTest) { - testPlan(337); + testPlan(342); testLinkParse(); testLinkFailParse(); testCADBSet(); diff --git a/modules/database/test/ioc/db/dbPutLinkTestJ.db b/modules/database/test/ioc/db/dbPutLinkTestJ.db index 621557c2c..4f518c50f 100644 --- a/modules/database/test/ioc/db/dbPutLinkTestJ.db +++ b/modules/database/test/ioc/db/dbPutLinkTestJ.db @@ -7,8 +7,8 @@ record(x, "j1") { } record(x, "j2") { - field(INP, {z:{good:2}}) - field(TSEL, "j1.TIME") + field(INP, {"z":{'good':2}}) + field(TSEL, 'j1.TIME') } record(x, "j3") { From 7cc246afc115fff58d6753f07cdf1e1a2fa76383 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sat, 8 Aug 2020 22:37:49 -0500 Subject: [PATCH 28/29] Modify database tests to use JSON5 (except for the tests that check parsing). Remove escaped double-quotes in map keys where possible. Replace escaped double-quotes with single quotes. --- modules/database/test/ioc/db/chfPluginTest.c | 84 +++++++++---------- modules/database/test/ioc/db/dbChannelTest.c | 22 ++--- modules/database/test/std/filters/arrTest.cpp | 14 ++-- modules/database/test/std/filters/dbndTest.c | 8 +- modules/database/test/std/filters/decTest.c | 16 ++-- modules/database/test/std/filters/syncTest.c | 18 ++-- modules/database/test/std/filters/tsTest.c | 2 +- modules/database/test/std/link/lnkCalcTest.c | 28 +++---- modules/database/test/std/link/lnkStateTest.c | 8 +- .../database/test/std/rec/analogMonitorTest.c | 2 +- .../database/test/std/rec/linkRetargetLink.db | 2 +- .../test/std/rec/linkRetargetLinkTest.c | 8 +- 12 files changed, 106 insertions(+), 106 deletions(-) diff --git a/modules/database/test/ioc/db/chfPluginTest.c b/modules/database/test/ioc/db/chfPluginTest.c index a382bfbd9..4a1667cf2 100644 --- a/modules/database/test/ioc/db/chfPluginTest.c +++ b/modules/database/test/ioc/db/chfPluginTest.c @@ -612,7 +612,7 @@ MAIN(chfPluginTest) /* tag i */ e1 = e_alloc; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"alloc-fail\":{\"i\":1}}")), + "x.{'alloc-fail':{i:1}}")), "create channel for alloc-fail: allocPvt returning NULL"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) @@ -627,7 +627,7 @@ MAIN(chfPluginTest) /* tag D (t and d) and f */ e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"strict-tagged\":{\"D\":1.2e15,\"f\":false}}")), + "x.{'strict-tagged':{D:1.2e15,f:false}}")), "create channel for strict-tagged parsing: D (t and d) and f"); testOk(checkValues(puser1, 3, 12, 0, 1.2e15, "hello", 0, 4), "guards intact, values correct"); @@ -641,7 +641,7 @@ MAIN(chfPluginTest) /* tag D2 (t and d) and f */ e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"strict-tagged\":{\"D2\":1.2e15,\"f\":false}}")), + "x.{'strict-tagged':{D2:1.2e15,f:false}}")), "create channel for strict-tagged parsing: D2 (t and d) and f"); testOk(checkValues(puser1, 4, 12, 0, 1.2e15, "hello", 0, 4), "guards intact, values correct"); @@ -655,7 +655,7 @@ MAIN(chfPluginTest) /* tag F: (t and f), d missing) */ e1 = e_alloc | e_error | e_free; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"strict-tagged\":{\"F\":false}}")), + "x.{'strict-tagged':{F:false}}")), "create channel for strict-tagged parsing: F (t and f), d missing"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) @@ -663,7 +663,7 @@ MAIN(chfPluginTest) /* tag I: (t and i) and f, d missing) */ e1 = e_alloc | e_error | e_free; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"strict-tagged\":{\"I\":1,\"f\":false}}")), + "x.{'strict-tagged':{I:1,f:false}}")), "create channel for strict-tagged parsing: I (t and i) and f, d missing"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) @@ -676,7 +676,7 @@ MAIN(chfPluginTest) /* tag i */ e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"sloppy-tagged\":{\"I\":1}}")), + "x.{'sloppy-tagged':{I:1}}")), "create channel for sloppy-tagged parsing: I"); testOk(checkValues(puser1, 1, 1, 1, 1.234e5, "hello", 0, 4), "guards intact, values correct"); @@ -690,7 +690,7 @@ MAIN(chfPluginTest) /* tag f */ e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"sloppy-tagged\":{\"F\":false}}")), + "x.{'sloppy-tagged':{F:false}}")), "create channel for sloppy-tagged parsing: F"); testOk(checkValues(puser1, 2, 12, 0, 1.234e5, "hello", 0, 4), "guards intact, values correct"); @@ -704,7 +704,7 @@ MAIN(chfPluginTest) /* tag d */ e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"sloppy-tagged\":{\"D\":1.2e15}}")), + "x.{'sloppy-tagged':{D:1.2e15}}")), "create channel for sloppy-tagged parsing: D"); testOk(checkValues(puser1, 3, 12, 1, 1.2e15, "hello", 0, 4), "guards intact, values correct"); @@ -718,7 +718,7 @@ MAIN(chfPluginTest) /* tag s */ e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"sloppy-tagged\":{\"S\":\"bar\"}}")), + "x.{'sloppy-tagged':{S:'bar'}}")), "create channel for sloppy-tagged parsing: S"); testOk(checkValues(puser1, 4, 12, 1, 1.234e5, "bar", 0, 4), "guards intact, values correct"); @@ -732,7 +732,7 @@ MAIN(chfPluginTest) /* tag c */ e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"sloppy-tagged\":{\"C\":\"R\"}}")), + "x.{'sloppy-tagged':{C:'R'}}")), "create channel for sloppy-tagged parsing: C"); testOk(checkValues(puser1, 5, 12, 1, 1.234e5, "hello", 0, 1), "guards intact, values correct"); @@ -749,7 +749,7 @@ MAIN(chfPluginTest) /* All perfect */ testHead("STRICT parsing: all ok"); e1 = e_alloc | e_ok; c1 = 0; - testOk(!!(pch = dbChannelCreate("x.{\"strict\":{\"i\":1,\"f\":false,\"d\":1.2e15,\"s\":\"bar\",\"c\":\"R\"}}")), + testOk(!!(pch = dbChannelCreate("x.{strict:{i:1,f:false,d:1.2e15,s:'bar',c:'R'}}")), "create channel for strict parsing: JSON correct"); testOk(checkValues(puser1, 99, 1, 0, 1.2e15, "bar", 0, 1), "guards intact, values correct"); @@ -765,35 +765,35 @@ MAIN(chfPluginTest) testHead("STRICT parsing: any missing parameter must fail"); e1 = e_alloc | e_error | e_free; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"strict\":{\"i\":1,\"f\":false,\"d\":1.2e15,\"s\":\"bar\"}}")), + "x.{strict:{i:1,f:false,d:1.2e15,s:'bar'}}")), "create channel for strict parsing: c missing"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) testDiag("expected %#x - called %#x", e1, c1); e1 = e_alloc | e_error | e_free; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"strict\":{\"f\":false,\"i\":1,\"d\":1.2e15,\"c\":\"R\"}}")), + "x.{strict:{f:false,i:1,d:1.2e15,c:'R'}}")), "create channel for strict parsing: s missing"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) testDiag("expected %#x - called %#x", e1, c1); e1 = e_alloc | e_error | e_free; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"strict\":{\"i\":1,\"c\":\"R\",\"f\":false,\"s\":\"bar\"}}")), + "x.{strict:{i:1,c:'R',f:false,s:'bar'}}")), "create channel for strict parsing: d missing"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) testDiag("expected %#x - called %#x", e1, c1); e1 = e_alloc | e_error | e_free; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"strict\":{\"d\":1.2e15,\"c\":\"R\",\"i\":1,\"s\":\"bar\"}}")), + "x.{strict:{d:1.2e15,c:'R',i:1,s:'bar'}}")), "create channel for strict parsing: f missing"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) testDiag("expected %#x - called %#x", e1, c1); e1 = e_alloc | e_error | e_free; c1 = 0; testOk(!(pch = dbChannelCreate( - "x.{\"strict\":{\"c\":\"R\",\"s\":\"bar\",\"f\":false,\"d\":1.2e15}}")), + "x.{strict:{c:'R',s:'bar',f:false,d:1.2e15}}")), "create channel for strict parsing: i missing"); testOk(!puser1, "user part cleaned up"); if (!testOk(c1 == e1, "all expected calls happened")) @@ -805,7 +805,7 @@ MAIN(chfPluginTest) testHead("NOCONV parsing: missing parameters get default value"); e1 = e_alloc | e_ok; c1 = 0; testOk(!!(pch = dbChannelCreate( - "x.{\"noconv\":{\"i\":1,\"f\":false,\"d\":1.2e15,\"s\":\"bar\"}}")), + "x.{noconv:{i:1,f:false,d:1.2e15,s:'bar'}}")), "create channel for noconv parsing: c missing"); testOk(checkValues(puser1, 99, 1, 0, 1.2e15, "bar", 0, 4), "guards intact, values correct"); @@ -819,28 +819,28 @@ MAIN(chfPluginTest) e1 = e_any; testOk(!!(pch = dbChannelCreate( - "x.{\"noconv\":{\"i\":1,\"f\":false,\"d\":1.2e15,\"c\":\"R\"}}")), + "x.{noconv:{i:1,f:false,d:1.2e15,c:'R'}}")), "create channel for noconv parsing: s missing"); testOk(checkValues(puser1, 99, 1, 0, 1.2e15, "hello", 0, 1), "guards intact, values correct"); if (pch) dbChannelDelete(pch); testOk(!!(pch = dbChannelCreate( - "x.{\"noconv\":{\"i\":1,\"f\":false,\"s\":\"bar\",\"c\":\"R\"}}")), + "x.{noconv:{i:1,f:false,s:'bar',c:'R'}}")), "create channel for noconv parsing: d missing"); testOk(checkValues(puser1, 99, 1, 0, 1.234e5, "bar", 0, 1), "guards intact, values correct"); if (pch) dbChannelDelete(pch); testOk(!!(pch = dbChannelCreate( - "x.{\"noconv\":{\"i\":1,\"d\":1.2e15,\"s\":\"bar\",\"c\":\"R\"}}")), + "x.{noconv:{i:1,d:1.2e15,s:'bar',c:'R'}}")), "create channel for noconv parsing: f missing"); testOk(checkValues(puser1, 99, 1, 1, 1.2e15, "bar", 0, 1), "guards intact, values correct"); if (pch) dbChannelDelete(pch); testOk(!!(pch = dbChannelCreate( - "x.{\"noconv\":{\"f\":false,\"d\":1.2e15,\"s\":\"bar\",\"c\":\"R\"}}")), + "x.{noconv:{f:false,d:1.2e15,s:'bar',c:'R'}}")), "create channel for noconv parsing: i missing"); testOk(checkValues(puser1, 99, 12, 0, 1.2e15, "bar", 0, 1), "guards intact, values correct"); @@ -849,7 +849,7 @@ MAIN(chfPluginTest) /* Reject wrong types */ #define WRONGTYPETEST(Var, Val, Typ) \ e1 = e_alloc | e_error | e_free; c1 = 0; \ - testOk(!(pch = dbChannelCreate("x.{\"noconv\":{\""#Var"\":"#Val"}}")), \ + testOk(!(pch = dbChannelCreate("x.{noconv:{'"#Var"':"#Val"}}")), \ "create channel for noconv parsing: wrong type "#Typ" for "#Var); \ testOk(!puser1, "user part cleaned up"); \ if (!testOk(c1 == e1, "all expected calls happened")) \ @@ -877,8 +877,8 @@ MAIN(chfPluginTest) #define CONVTESTGOOD(Var, Val, Typ, Ival, Fval, Dval, Sval1, Sval2, Cval) \ e1 = e_alloc | e_ok; c1 = 0; \ - testDiag("Calling dbChannelCreate x.{\"sloppy\":{\""#Var"\":"#Val"}}"); \ - testOk(!!(pch = dbChannelCreate("x.{\"sloppy\":{\""#Var"\":"#Val"}}")), \ + testDiag("Calling dbChannelCreate x.{sloppy:{"#Var":"#Val"}}"); \ + testOk(!!(pch = dbChannelCreate("x.{sloppy:{"#Var":"#Val"}}")), \ "create channel for sloppy parsing: "#Typ" (good) for "#Var); \ testOk(checkValues(puser1, 99, Ival, Fval, Dval, Sval1, Sval2, Cval), \ "guards intact, values correct"); \ @@ -892,8 +892,8 @@ MAIN(chfPluginTest) #define CONVTESTBAD(Var, Val, Typ) \ e1 = e_alloc | e_error | e_free; c1 = 0; \ - testDiag("Calling dbChannelCreate x.{\"sloppy\":{\""#Var"\":"#Val"}}"); \ - testOk(!(pch = dbChannelCreate("x.{\"sloppy\":{\""#Var"\":"#Val"}}")), \ + testDiag("Calling dbChannelCreate x.{sloppy:{"#Var":"#Val"}}"); \ + testOk(!(pch = dbChannelCreate("x.{sloppy:{"#Var":"#Val"}}")), \ "create channel for sloppy parsing: "#Typ" (bad) for "#Var); \ testOk(!puser1, "user part cleaned up"); \ if (!testOk(c1 == e1, "create channel: all expected calls happened")) \ @@ -1018,36 +1018,36 @@ MAIN(chfPluginTest) if (!testOk(c1 == e1, "delete channel (1): all expected calls happened")) testDiag("expected %#x - called %#x", e1, c1); \ if (!testOk(c2 == e2, "delete channel (2): all expected calls happened")) testDiag("expected %#x - called %#x", e2, c2); - CHAINTEST1("1 pre", "{\"pre\":{}}", e_reg_pre, e_pre | e_dtor, 1); /* One filter, pre chain */ - CHAINTEST1("1 post", "{\"post\":{}}", e_reg_post, e_post | e_dtor, 1); /* One filter, post chain */ - CHAINTEST1("1 both", "{\"sloppy\":{}}", e_reg_pre | e_reg_post, e_pre | e_post | e_dtor, 2); /* One, both chains */ - CHAINTEST2("2 pre", "{\"pre\":{},\"pre\":{}}", e_reg_pre, e_pre | e_dtor, e_reg_pre, e_pre, 2); /* Two filters, pre chain */ - CHAINTEST2("2 post", "{\"post\":{},\"post\":{}}", e_reg_post, e_post | e_dtor, e_reg_post, e_post, 2); /* Two filters, post chain */ - CHAINTEST2("2 both", "{\"sloppy\":{},\"sloppy\":{}}", /* Two, both chains */ + CHAINTEST1("1 pre", "{pre:{}}", e_reg_pre, e_pre | e_dtor, 1); /* One filter, pre chain */ + CHAINTEST1("1 post", "{post:{}}", e_reg_post, e_post | e_dtor, 1); /* One filter, post chain */ + CHAINTEST1("1 both", "{sloppy:{}}", e_reg_pre | e_reg_post, e_pre | e_post | e_dtor, 2); /* One, both chains */ + CHAINTEST2("2 pre", "{pre:{},pre:{}}", e_reg_pre, e_pre | e_dtor, e_reg_pre, e_pre, 2); /* Two filters, pre chain */ + CHAINTEST2("2 post", "{post:{},post:{}}", e_reg_post, e_post | e_dtor, e_reg_post, e_post, 2); /* Two filters, post chain */ + CHAINTEST2("2 both", "{sloppy:{},sloppy:{}}", /* Two, both chains */ e_reg_pre | e_reg_post, e_pre | e_post | e_dtor, e_reg_pre | e_reg_post, e_pre | e_post, 4); - CHAINTEST2("1 pre, 1 post", "{\"pre\":{},\"post\":{}}", e_reg_pre, e_pre | e_dtor, e_reg_post, e_post, 2); /* Two, pre then post */ - CHAINTEST2("1 post, 1 pre", "{\"post\":{},\"pre\":{}}", e_reg_post, e_post, e_reg_pre, e_pre | e_dtor, 2); /* Two, post then pre */ - CHAINTEST2("1 pre, 1 both", "{\"pre\":{},\"sloppy\":{}}", /* Two, pre then both */ + CHAINTEST2("1 pre, 1 post", "{pre:{},post:{}}", e_reg_pre, e_pre | e_dtor, e_reg_post, e_post, 2); /* Two, pre then post */ + CHAINTEST2("1 post, 1 pre", "{post:{},pre:{}}", e_reg_post, e_post, e_reg_pre, e_pre | e_dtor, 2); /* Two, post then pre */ + CHAINTEST2("1 pre, 1 both", "{pre:{},sloppy:{}}", /* Two, pre then both */ e_reg_pre, e_pre | e_dtor, e_reg_pre | e_reg_post, e_pre | e_post, 3); - CHAINTEST2("1 both, 1 pre", "{\"sloppy\":{},\"pre\":{}}", /* Two, both then pre */ + CHAINTEST2("1 both, 1 pre", "{sloppy:{},pre:{}}", /* Two, both then pre */ e_reg_pre | e_reg_post, e_pre | e_post | e_dtor, e_reg_pre, e_pre, 3); - CHAINTEST2("1 post, 1 both", "{\"post\":{},\"sloppy\":{}}", /* Two, post then both */ + CHAINTEST2("1 post, 1 both", "{post:{},sloppy:{}}", /* Two, post then both */ e_reg_post, e_post, e_reg_pre | e_reg_post, e_pre | e_post | e_dtor, 3); - CHAINTEST2("1 both, 1 post", "{\"sloppy\":{},\"post\":{}}", /* Two, both then post */ + CHAINTEST2("1 both, 1 post", "{sloppy:{},post:{}}", /* Two, both then post */ e_reg_pre | e_reg_post, e_pre | e_post | e_dtor, e_reg_post, e_post, 3); /* Plugins dropping updates */ drop = 0; - CHAINTEST2("2 both (drop at 0)", "{\"sloppy\":{},\"sloppy\":{}}", /* Two, both chains, drop at filter 0 */ + CHAINTEST2("2 both (drop at 0)", "{sloppy:{},sloppy:{}}", /* Two, both chains, drop at filter 0 */ e_reg_pre | e_reg_post, e_pre, e_reg_pre | e_reg_post, 0, -1); drop = 1; - CHAINTEST2("2 both (drop at 1)", "{\"sloppy\":{},\"sloppy\":{}}", /* Two, both chains, drop at filter 1 */ + CHAINTEST2("2 both (drop at 1)", "{sloppy:{},sloppy:{}}", /* Two, both chains, drop at filter 1 */ e_reg_pre | e_reg_post, e_pre, e_reg_pre | e_reg_post, e_pre, -1); drop = 2; - CHAINTEST2("2 both (drop at 2)", "{\"sloppy\":{},\"sloppy\":{}}", /* Two, both chains, drop at filter 2 */ + CHAINTEST2("2 both (drop at 2)", "{sloppy:{},sloppy:{}}", /* Two, both chains, drop at filter 2 */ e_reg_pre | e_reg_post, e_pre | e_post, e_reg_pre | e_reg_post, e_pre, -1); drop = 3; - CHAINTEST2("2 both (drop at 3)", "{\"sloppy\":{},\"sloppy\":{}}", /* Two, both chains, drop at filter 3 */ + CHAINTEST2("2 both (drop at 3)", "{sloppy:{},sloppy:{}}", /* Two, both chains, drop at filter 3 */ e_reg_pre | e_reg_post, e_pre | e_post, e_reg_pre | e_reg_post, e_pre | e_post, -1); drop = -1; diff --git a/modules/database/test/ioc/db/dbChannelTest.c b/modules/database/test/ioc/db/dbChannelTest.c index 32a2243ee..3b90d8187 100644 --- a/modules/database/test/ioc/db/dbChannelTest.c +++ b/modules/database/test/ioc/db/dbChannelTest.c @@ -183,7 +183,7 @@ MAIN(testDbChannel) /* dbChannelTest is an API routine... */ /* dbChannelTest() allows but ignores field modifiers */ testOk1(!dbChannelTest("x.NAME$")); testOk1(!dbChannelTest("x.{}")); - testOk1(!dbChannelTest("x.VAL{\"json\":true}")); + testOk1(!dbChannelTest("x.VAL{json:true}")); /* dbChannelCreate() accepts field modifiers */ testOk1(!!(pch = dbChannelCreate("x.{}"))); @@ -212,34 +212,34 @@ MAIN(testDbChannel) /* dbChannelTest is an API routine... */ testOk(!dbChannelCreate("x.NOFIELD"), "Create, bad field"); testOk(!dbChannelCreate("x.{not-json}"), "Create, bad JSON"); eltc(0); - testOk(!dbChannelCreate("x.{\"none\":null}"), "Create, bad filter"); + testOk(!dbChannelCreate("x.{none:null}"), "Create, bad filter"); eltc(1); dbRegisterFilter("any", &testIf, NULL); /* Parser event rejection by filter */ e = e_start; - testOk1(!dbChannelCreate("x.{\"any\":null}")); + testOk1(!dbChannelCreate("x.{any:null}")); r = e_start; e = e_start | e_null | e_abort; - testOk1(!dbChannelCreate("x.{\"any\":null}")); + testOk1(!dbChannelCreate("x.{any:null}")); r = e_start | e_null; e = e_start | e_null | e_end; - testOk1(!dbChannelCreate("x.{\"any\":null}")); + testOk1(!dbChannelCreate("x.{any:null}")); /* Successful parsing... */ r = r_any; e = e_start | e_null | e_end; - testOk1(!!(pch = dbChannelCreate("x.{\"any\":null}"))); + testOk1(!!(pch = dbChannelCreate("x.{any:null}"))); e = e_close; if (pch) dbChannelDelete(pch); dbRegisterFilter("scalar", &testIf, NULL); e = e_start | e_null | e_end; - testOk1(!!(pch = dbChannelCreate("x.{\"scalar\":null}"))); + testOk1(!!(pch = dbChannelCreate("x.{scalar:null}"))); e = e_report; dbChannelShow(pch, 2, 2); @@ -249,23 +249,23 @@ MAIN(testDbChannel) /* dbChannelTest is an API routine... */ e = e_start | e_start_array | e_boolean | e_integer | e_end_array | e_end; - testOk1(!!(pch = dbChannelCreate("x.{\"any\":[true,1]}"))); + testOk1(!!(pch = dbChannelCreate("x.{any:[true,1]}"))); e = e_close; if (pch) dbChannelDelete(pch); e = e_start | e_start_map | e_map_key | e_double | e_string | e_end_map | e_end; - testOk1(!!(pch = dbChannelCreate("x.{\"any\":{\"a\":2.7183,\"b\":\"c\"}}"))); + testOk1(!!(pch = dbChannelCreate("x.{any:{a:2.7183,b:'c'}}"))); e = e_close; if (pch) dbChannelDelete(pch); /* More event rejection */ r = r_scalar; e = e_start | e_start_array | e_abort; - testOk1(!dbChannelCreate("x.{\"scalar\":[null]}")); + testOk1(!dbChannelCreate("x.{scalar:[null]}")); e = e_start | e_start_map | e_abort; - testOk1(!dbChannelCreate("x.{\"scalar\":{}}")); + testOk1(!dbChannelCreate("x.{scalar:{}}")); testIocShutdownOk(); testdbCleanup(); diff --git a/modules/database/test/std/filters/arrTest.cpp b/modules/database/test/std/filters/arrTest.cpp index 10d234cd7..bd83bd8ab 100644 --- a/modules/database/test/std/filters/arrTest.cpp +++ b/modules/database/test/std/filters/arrTest.cpp @@ -178,7 +178,7 @@ static void check(short dbr_type) { /* Default: should not change anything */ testHead("Ten %s elements from rec, increment 1, full size (default)", typname); - createAndOpen(valname, "{\"arr\":{}}", "(default)", &pch, 1); + createAndOpen(valname, "{arr:{}}", "(default)", &pch, 1); testOk(pch->final_type == valaddr.field_type, "final type unchanged (%d->%d)", valaddr.field_type, pch->final_type); testOk(pch->final_no_elements == valaddr.no_elements, @@ -188,7 +188,7 @@ static void check(short dbr_type) { dbChannelDelete(pch); testHead("Ten %s elements from rec, increment 1, out-of-bound start parameter", typname); - createAndOpen(valname, "{\"arr\":{\"s\":-500}}", "out-of-bound start", &pch, 1); + createAndOpen(valname, "{arr:{s:-500}}", "out-of-bound start", &pch, 1); testOk(pch->final_type == valaddr.field_type, "final type unchanged (%d->%d)", valaddr.field_type, pch->final_type); testOk(pch->final_no_elements == valaddr.no_elements, @@ -197,7 +197,7 @@ static void check(short dbr_type) { dbChannelDelete(pch); testHead("Ten %s elements from rec, increment 1, out-of-bound end parameter", typname); - createAndOpen(valname, "{\"arr\":{\"e\":500}}", "out-of-bound end", &pch, 1); + createAndOpen(valname, "{arr:{e:500}}", "out-of-bound end", &pch, 1); testOk(pch->final_type == valaddr.field_type, "final type unchanged (%d->%d)", valaddr.field_type, pch->final_type); testOk(pch->final_no_elements == valaddr.no_elements, @@ -206,7 +206,7 @@ static void check(short dbr_type) { dbChannelDelete(pch); testHead("Ten %s elements from rec, increment 1, zero increment parameter", typname); - createAndOpen(valname, "{\"arr\":{\"i\":0}}", "zero increment", &pch, 1); + createAndOpen(valname, "{arr:{i:0}}", "zero increment", &pch, 1); testOk(pch->final_type == valaddr.field_type, "final type unchanged (%d->%d)", valaddr.field_type, pch->final_type); testOk(pch->final_no_elements == valaddr.no_elements, @@ -215,7 +215,7 @@ static void check(short dbr_type) { dbChannelDelete(pch); testHead("Ten %s elements from rec, increment 1, invalid increment parameter", typname); - createAndOpen(valname, "{\"arr\":{\"i\":-30}}", "invalid increment", &pch, 1); + createAndOpen(valname, "{arr:{i:-30}}", "invalid increment", &pch, 1); testOk(pch->final_type == valaddr.field_type, "final type unchanged (%d->%d)", valaddr.field_type, pch->final_type); testOk(pch->final_no_elements == valaddr.no_elements, @@ -225,7 +225,7 @@ static void check(short dbr_type) { #define TEST5(Incr, Left, Right, Type) \ testHead("Five %s elements from rec, increment " #Incr ", " Type " addressing", typname); \ - createAndOpen(valname, "{\"arr\":{\"s\":" #Left ",\"e\":" #Right ",\"i\":" #Incr "}}", \ + createAndOpen(valname, "{arr:{s:" #Left ",e:" #Right ",i:" #Incr "}}", \ "(" #Left ":" #Incr ":" #Right ")", &pch, 1); \ testOk(pch->final_type == valaddr.field_type, \ "final type unchanged (%d->%d)", valaddr.field_type, pch->final_type); \ @@ -262,7 +262,7 @@ static void check(short dbr_type) { #define TEST5B(Incr, Left, Right, Type) \ testHead("Five %s elements from buffer, increment " #Incr ", " Type " addressing", typname); \ - createAndOpen(valname, "{\"arr\":{},\"arr\":{\"s\":" #Left ",\"e\":" #Right ",\"i\":" #Incr "}}", \ + createAndOpen(valname, "{arr:{},arr:{s:" #Left ",e:" #Right ",i:" #Incr "}}", \ "(" #Left ":" #Incr ":" #Right ")", &pch, 2); \ testOk(pch->final_type == valaddr.field_type, \ "final type unchanged (%d->%d)", valaddr.field_type, pch->final_type); \ diff --git a/modules/database/test/std/filters/dbndTest.c b/modules/database/test/std/filters/dbndTest.c index 358788299..e7897e28a 100644 --- a/modules/database/test/std/filters/dbndTest.c +++ b/modules/database/test/std/filters/dbndTest.c @@ -148,7 +148,7 @@ MAIN(dbndTest) testOk(!!(plug = dbFindFilter(dbnd, strlen(dbnd))), "plugin dbnd registered correctly"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dbnd\":{}}")), "dbChannel with plugin dbnd (delta=0) created"); + testOk(!!(pch = dbChannelCreate("x.VAL{dbnd:{}}")), "dbChannel with plugin dbnd (delta=0) created"); testOk((ellCount(&pch->filters) == 1), "channel has one plugin"); /* Start the free-list */ @@ -202,7 +202,7 @@ MAIN(dbndTest) /* Delta = -1: pass any update */ testHead("Delta = -1: pass any update"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dbnd\":{\"d\":-1.0}}")), "dbChannel with plugin dbnd (delta=-1) created"); + testOk(!!(pch = dbChannelCreate("x.VAL{dbnd:{d:-1.0}}")), "dbChannel with plugin dbnd (delta=-1) created"); testOk(!(dbChannelOpen(pch)), "dbChannel with plugin dbnd opened"); pfl2 = db_create_read_log(pch); @@ -220,7 +220,7 @@ MAIN(dbndTest) /* Delta = absolute */ testHead("Delta = absolute"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dbnd\":{\"d\":3}}")), "dbChannel with plugin dbnd (delta=3) created"); + testOk(!!(pch = dbChannelCreate("x.VAL{dbnd:{d:3}}")), "dbChannel with plugin dbnd (delta=3) created"); testOk(!(dbChannelOpen(pch)), "dbChannel with plugin dbnd opened"); pfl2 = db_create_read_log(pch); @@ -254,7 +254,7 @@ MAIN(dbndTest) /* Delta = relative */ testHead("Delta = relative"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dbnd\":{\"m\":\"rel\",\"d\":50}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{dbnd:{m:'rel',d:50}}")), "dbChannel with plugin dbnd (mode=rel, delta=50) created"); testOk(!(dbChannelOpen(pch)), "dbChannel with plugin dbnd opened"); diff --git a/modules/database/test/std/filters/decTest.c b/modules/database/test/std/filters/decTest.c index b9d753dc4..e0961af10 100644 --- a/modules/database/test/std/filters/decTest.c +++ b/modules/database/test/std/filters/decTest.c @@ -149,20 +149,20 @@ MAIN(decTest) "plugin '%s' registered correctly", myname); /* N < 1 */ - testOk(!(pch = dbChannelCreate("x.VAL{\"dec\":{\"n\":-1}}")), + testOk(!(pch = dbChannelCreate("x.VAL{dec:{n:-1}}")), "dbChannel with dec (n=-1) failed"); - testOk(!(pch = dbChannelCreate("x.VAL{\"dec\":{\"n\":0}}")), + testOk(!(pch = dbChannelCreate("x.VAL{dec:{n:0}}")), "dbChannel with dec (n=0) failed"); /* Bad parms */ - testOk(!(pch = dbChannelCreate("x.VAL{\"dec\":{}}")), + testOk(!(pch = dbChannelCreate("x.VAL{dec:{}}")), "dbChannel with dec (no parm) failed"); - testOk(!(pch = dbChannelCreate("x.VAL{\"dec\":{\"x\":true}}")), + testOk(!(pch = dbChannelCreate("x.VAL{dec:{x:true}}")), "dbChannel with dec (x=true) failed"); /* No Decimation (N=1) */ testHead("No Decimation (n=1)"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dec\":{\"n\":1}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{dec:{n:1}}")), "dbChannel with plugin dec (n=1) created"); /* Start the free-list */ @@ -192,7 +192,7 @@ MAIN(decTest) /* Decimation (N=2) */ testHead("Decimation (n=2)"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dec\":{\"n\":2}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{dec:{n:2}}")), "dbChannel with plugin dec (n=2) created"); checkAndOpenChannel(pch, plug); @@ -222,7 +222,7 @@ MAIN(decTest) /* Decimation (N=3) */ testHead("Decimation (n=3)"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dec\":{\"n\":3}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{dec:{n:3}}")), "dbChannel with plugin dec (n=3) created"); checkAndOpenChannel(pch, plug); @@ -252,7 +252,7 @@ MAIN(decTest) /* Decimation (N=4) */ testHead("Decimation (n=4)"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"dec\":{\"n\":4}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{dec:{n:4}}")), "dbChannel with plugin dec (n=4) created"); checkAndOpenChannel(pch, plug); diff --git a/modules/database/test/std/filters/syncTest.c b/modules/database/test/std/filters/syncTest.c index 7d7441572..6527d2c96 100644 --- a/modules/database/test/std/filters/syncTest.c +++ b/modules/database/test/std/filters/syncTest.c @@ -225,19 +225,19 @@ MAIN(syncTest) testOk(!!(red = dbStateCreate("red")), "state 'red' created successfully"); /* nonexisting state */ - testOk(!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"while\",\"s\":\"blue\"}}")), + testOk(!(pch = dbChannelCreate("x.VAL{sync:{m:'while',s:'blue'}}")), "dbChannel with sync (m='while' s='blue') (nonex state) failed"); /* missing state */ - testOk(!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"while\"}}")), + testOk(!(pch = dbChannelCreate("x.VAL{sync:{m:'while'}}")), "dbChannel with sync (m='while') (no state) failed"); /* missing mode */ - testOk(!(pch = dbChannelCreate("x.VAL{\"sync\":{\"s\":\"red\"}}")), + testOk(!(pch = dbChannelCreate("x.VAL{sync:{s:'red'}}")), "dbChannel with sync (s='red') (no mode) failed"); /* mode WHILE */ testHead("Mode WHILE (m='while', s='red')"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"while\",\"s\":\"red\"}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{sync:{m:'while',s:'red'}}")), "dbChannel with plugin sync (m='while' s='red') created"); /* Start the free-list */ @@ -274,7 +274,7 @@ MAIN(syncTest) /* mode UNLESS */ testHead("Mode UNLESS (m='unless', s='red')"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"unless\",\"s\":\"red\"}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{sync:{m:'unless',s:'red'}}")), "dbChannel with plugin sync (m='unless' s='red') created"); checkAndOpenChannel(pch, plug); @@ -306,7 +306,7 @@ MAIN(syncTest) /* mode BEFORE */ testHead("Mode BEFORE (m='before', s='red')"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"before\",\"s\":\"red\"}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{sync:{m:'before',s:'red'}}")), "dbChannel with plugin sync (m='before' s='red') created"); checkAndOpenChannel(pch, plug); @@ -340,7 +340,7 @@ MAIN(syncTest) /* mode FIRST */ testHead("Mode FIRST (m='first', s='red')"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"first\",\"s\":\"red\"}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{sync:{m:'first',s:'red'}}")), "dbChannel with plugin sync (m='first' s='red') created"); checkAndOpenChannel(pch, plug); @@ -373,7 +373,7 @@ MAIN(syncTest) /* mode LAST */ testHead("Mode LAST (m='last', s='red')"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"last\",\"s\":\"red\"}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{sync:{m:'last',s:'red'}}")), "dbChannel with plugin sync (m='last' s='red') created"); checkAndOpenChannel(pch, plug); @@ -407,7 +407,7 @@ MAIN(syncTest) /* mode AFTER */ testHead("Mode AFTER (m='after', s='red')"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"sync\":{\"m\":\"after\",\"s\":\"red\"}}")), + testOk(!!(pch = dbChannelCreate("x.VAL{sync:{m:'after',s:'red'}}")), "dbChannel with plugin sync (m='after' s='red') created"); checkAndOpenChannel(pch, plug); diff --git a/modules/database/test/std/filters/tsTest.c b/modules/database/test/std/filters/tsTest.c index 4ebd6fcc2..bd0b799e9 100644 --- a/modules/database/test/std/filters/tsTest.c +++ b/modules/database/test/std/filters/tsTest.c @@ -74,7 +74,7 @@ MAIN(tsTest) testOk(!!(plug = dbFindFilter(ts, strlen(ts))), "plugin ts registered correctly"); - testOk(!!(pch = dbChannelCreate("x.VAL{\"ts\":{}}")), "dbChannel with plugin ts created"); + testOk(!!(pch = dbChannelCreate("x.VAL{ts:{}}")), "dbChannel with plugin ts created"); testOk((ellCount(&pch->filters) == 1), "channel has one plugin"); memset(&fl, PATTERN, sizeof(fl)); diff --git a/modules/database/test/std/link/lnkCalcTest.c b/modules/database/test/std/link/lnkCalcTest.c index 7f108a4ba..5297a53c5 100644 --- a/modules/database/test/std/link/lnkCalcTest.c +++ b/modules/database/test/std/link/lnkCalcTest.c @@ -53,9 +53,9 @@ static void testCalc() { dbStateId red; - testPutLongStr("io.INPUT", "{\"calc\":{" - "\"expr\":\"a\"," - "\"args\":[{\"state\":\"red\"}]" + testPutLongStr("io.INPUT", "{calc:{" + "expr:'a'," + "args:[{state:'red'}]" "}}"); if (testOk1(pinp->type == JSON_LINK)) testDiag("Link was set to '%s'", pinp->value.json.string); @@ -78,11 +78,11 @@ static void testCalc() dbStateId minor = dbStateCreate("minor"); epicsEnum16 stat, sevr; - testPutLongStr("io.INPUT", "{\"calc\":{" - "\"expr\":\"0\"," - "\"major\":\"A\"," - "\"minor\":\"B\"," - "\"args\":[{\"state\":\"major\"},{\"state\":\"minor\"}]" + testPutLongStr("io.INPUT", "{calc:{" + "expr:'0'," + "major:'A'," + "minor:'B'," + "args:[{state:'major'},{state:'minor'}]" "}}"); if (testOk1(pinp->type == JSON_LINK)) testDiag("Link was set to '%s'", pinp->value.json.string); @@ -114,12 +114,12 @@ static void testCalc() dbStateId red = dbStateFind("red"); dbStateId out = dbStateCreate("out"); - testPutLongStr("io.OUTPUT", "{\"calc\":{" - "\"expr\":\"!a\"," - "\"out\":{\"state\":\"out\"}," - "\"args\":[{\"state\":\"red\"}]," - "\"units\":\"things\"," - "\"prec\":3" + testPutLongStr("io.OUTPUT", "{calc:{" + "expr:'!a'," + "out:{state:'out'}," + "args:[{state:'red'}]," + "units:'things'," + "prec:3" "}}"); if (testOk1(pout->type == JSON_LINK)) testDiag("Link was set to '%s'", pout->value.json.string); diff --git a/modules/database/test/std/link/lnkStateTest.c b/modules/database/test/std/link/lnkStateTest.c index 69dba38bb..406ea46e3 100644 --- a/modules/database/test/std/link/lnkStateTest.c +++ b/modules/database/test/std/link/lnkStateTest.c @@ -50,7 +50,7 @@ static void testState() red = dbStateFind("red"); testOk(!red, "No state red exists"); - testdbPutFieldOk("io.INPUT", DBF_STRING, "{\"state\":\"red\"}"); + testdbPutFieldOk("io.INPUT", DBF_STRING, "{state:'red'}"); if (testOk1(pinp->type == JSON_LINK)) testDiag("Link was set to '%s'", pinp->value.json.string); red = dbStateFind("red"); @@ -64,7 +64,7 @@ static void testState() testOk(!status, "dbGetLink succeeded (status = %ld)", status); testOk(i16, "Got TRUE"); - testdbPutFieldOk("io.INPUT", DBF_STRING, "{\"state\":\"!red\"}"); + testdbPutFieldOk("io.INPUT", DBF_STRING, "{state:'!red'}"); if (testOk1(pinp->type == JSON_LINK)) testDiag("Link was set to '%s'", pinp->value.json.string); @@ -72,7 +72,7 @@ static void testState() testOk(!status, "dbGetLink succeeded (status = %ld)", status); testOk(!i16, "Got FALSE"); - testdbPutFieldOk("io.OUTPUT", DBF_STRING, "{\"state\":\"red\"}"); + testdbPutFieldOk("io.OUTPUT", DBF_STRING, "{state:'red'}"); if (testOk1(pout->type == JSON_LINK)) testDiag("Link was set to '%s'", pout->value.json.string); @@ -106,7 +106,7 @@ static void testState() testOk(!status, "dbPutLink %g succeeded (status = %ld)", f64, status); testOk(dbStateGet(red), "state was set"); - testdbPutFieldOk("io.OUTPUT", DBF_STRING, "{\"state\":\"!red\"}"); + testdbPutFieldOk("io.OUTPUT", DBF_STRING, "{state:'!red'}"); if (testOk1(pout->type == JSON_LINK)) testDiag("Link was set to '%s'", pout->value.json.string); diff --git a/modules/database/test/std/rec/analogMonitorTest.c b/modules/database/test/std/rec/analogMonitorTest.c index bf79bdd83..b0f134787 100644 --- a/modules/database/test/std/rec/analogMonitorTest.c +++ b/modules/database/test/std/rec/analogMonitorTest.c @@ -180,7 +180,7 @@ MAIN(analogMonitorTest) for (irec = 0; irec < NO_OF_RECORD_TYPES; irec++) { strcpy(cval, t_Record[irec]); strcpy(chan, cval); - strcat(chan, ".VAL{\"test\":{}}"); + strcat(chan, ".VAL{test:{}}"); if ((strcmp(t_Record[irec], "sel") == 0) || (strcmp(t_Record[irec], "calc") == 0) || (strcmp(t_Record[irec], "calcout") == 0)) { diff --git a/modules/database/test/std/rec/linkRetargetLink.db b/modules/database/test/std/rec/linkRetargetLink.db index 148e2d50b..759ddaf22 100644 --- a/modules/database/test/std/rec/linkRetargetLink.db +++ b/modules/database/test/std/rec/linkRetargetLink.db @@ -18,7 +18,7 @@ record(stringout, "rec:link2") { record(ai, "rec:j1") { field(INP, {calc:{ - expr:"A+5", + expr:'A+5', args:5 }}) field(PINI, "YES") diff --git a/modules/database/test/std/rec/linkRetargetLinkTest.c b/modules/database/test/std/rec/linkRetargetLinkTest.c index dab000167..a99471bb6 100644 --- a/modules/database/test/std/rec/linkRetargetLinkTest.c +++ b/modules/database/test/std/rec/linkRetargetLinkTest.c @@ -75,18 +75,18 @@ static void testRetargetJLink(void) testdbGetFieldEqual("rec:j1", DBF_DOUBLE, 10.0); /* minimal args */ - testLongStrEq("rec:j1.INP$", "{calc:{expr:\"A+5\",args:5}}"); + testLongStrEq("rec:j1.INP$", "{calc:{expr:'A+5',args:5}}"); /* with [] */ - testPutLongStr("rec:j1.INP$", "{\"calc\":{\"expr\":\"A+5\",\"args\":[7]}}"); + testPutLongStr("rec:j1.INP$", "{calc:{expr:'A+5',args:[7]}}"); testdbPutFieldOk("rec:j1.PROC", DBF_LONG, 1); /* with const */ - testPutLongStr("rec:j1.INP$", "{calc:{expr:\"A+5\",args:[{const:7}]}}"); + testPutLongStr("rec:j1.INP$", "{calc:{expr:'A+5',args:[{const:7}]}}"); testdbPutFieldOk("rec:j1.PROC", DBF_LONG, 1); testdbGetFieldEqual("rec:j1", DBF_DOUBLE, 12.0); - testLongStrEq("rec:j1.INP$", "{calc:{expr:\"A+5\",args:[{const:7}]}}"); + testLongStrEq("rec:j1.INP$", "{calc:{expr:'A+5',args:[{const:7}]}}"); } MAIN(linkRetargetLinkTest) From 5f5cc85e28e7fee4c6e290294033d6a34d9d8237 Mon Sep 17 00:00:00 2001 From: Andrew Johnson Date: Sun, 9 Aug 2020 00:06:31 -0500 Subject: [PATCH 29/29] Release Notes for json5 changes --- documentation/RELEASE_NOTES.md | 54 ++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/documentation/RELEASE_NOTES.md b/documentation/RELEASE_NOTES.md index d54b84571..f6b965a31 100644 --- a/documentation/RELEASE_NOTES.md +++ b/documentation/RELEASE_NOTES.md @@ -17,6 +17,60 @@ should also be read to understand what has changed since earlier releases. +### Support for JSON5 + +The YAJL parser and generator routines in libcom and in the IOC's dbStatic +parser now support the JSON5 standard. This adds various features to JSON +without altering the API for the code other than adding a new option to the +YAJL parser which can be used to disable JSON5 support if desired. The new +features include: + +- The ability to handle numeric values `Infinity`, `-Infinity` and `NaN`. +- String values and map keys may be enclosed in single quotes `'`, inside which + the double-quote character `"` doesn't have to be escaped with a back-slash + `\`, although a single-quote character `'` (or apostrophy) must be escaped + inside a single-quoted string. +- Numbers may start with a plus sign, `+`. +- Integers may be expressed in hexadecimal with a leading `0x` or `0X`. +- Floating-point numbers may start or end with their decimal point `.` + (after the sign or before the exponent respectively if present). +- Map keys that match the regex `[A-Za-z_][A-Za-z_0-9]*` don't have to be + enclosed in quotes at all. The dbStatic parser adds `.+-` to the characters + allowed but will add quotes around such keys before passing them to YAJL. +- Arrays and maps allow a comma before the closing bracket/brace character. +- The YAJL parser will elide a backslash followed by a newline characters from + a string value. The dbStatic parser doesn't allow that however. + +Code that must also compile against the older API can use the new C macro +`HAS_JSON5` to detect the new version. This macro is defined on including +either the `yajl_parse.h` or `yajl_gen.h` headers, which also provide the +new configuration options to turn on JSON5 support. + +All APIs in the IOC that previously accepted JSON will now accept JSON5. +This includes JSON field modifiers (channel filters), JSON link addresses, +constant input link array values and database info-tag values. JSON values +that get parsed by the dbLoadRecords() routine are still more liberal than +the other uses as the ability to use unquoted strings that was called +"relaxed JSON" is still supported, whereas the JSON5 standard and the YAJL +parser only allow unquoted strings to be used for keys in a JSON map. + +This change fixes [lauchpad bug #1714455](https://bugs.launchpad.net/bugs/1714455). + + +### Character escape changes + +- The libCom routines `epicsStrnRawFromEscaped()` and `dbTranslateEscape()` + declared in epicsString.h no longer accept octal escaped characters such as + `\123` or `\41`. +- The routine `epicsStrnEscapedFromRaw()` now generates hex + excaped characters for unprintable characters such as `\x1f`. +- Hex escape character sequences `\xXX` must now contain exactly 2 hex digits. +- An escape sequence `\0` now generates a zero byte in the raw string, but the + other digits `1-9` should not appear after a back-slash. + +These changes are to more closely follow the JSON5 standard, which doesn't +support octal character escapes or the `\a` (Bel, `\x07`) escape sequence. + ## EPICS Release 7.0.4.1 ### Bug fixes