From 2b99e3c6ee13cd9a4a0846a5e1395f94f22b9681 Mon Sep 17 00:00:00 2001 From: Michael Davidsaver Date: Sun, 19 Nov 2023 10:55:35 -0800 Subject: [PATCH] JSON parsing YAJL Added in Base 3.15 Omit when building with 3.14 --- src/Makefile | 4 + src/json.cpp | 534 ++++++++++++++++++++++++++++++++++++++++++++++ src/pvxs/json.h | 119 +++++++++++ test/Makefile | 6 + test/testjson.cpp | 330 ++++++++++++++++++++++++++++ 5 files changed, 993 insertions(+) create mode 100644 src/json.cpp create mode 100644 src/pvxs/json.h create mode 100644 test/testjson.cpp diff --git a/src/Makefile b/src/Makefile index 89d246f..97fc223 100644 --- a/src/Makefile +++ b/src/Makefile @@ -52,6 +52,7 @@ INC += pvxs/unittest.h INC += pvxs/util.h INC += pvxs/sharedArray.h INC += pvxs/data.h +INC += pvxs/json.h INC += pvxs/nt.h INC += pvxs/netcommon.h INC += pvxs/server.h @@ -72,6 +73,9 @@ LIB_SRCS += bitmask.cpp LIB_SRCS += type.cpp LIB_SRCS += data.cpp LIB_SRCS += datafmt.cpp +ifdef BASE_3_15 +LIB_SRCS += json.cpp +endif LIB_SRCS += pvrequest.cpp LIB_SRCS += dataencode.cpp LIB_SRCS += nt.cpp diff --git a/src/json.cpp b/src/json.cpp new file mode 100644 index 0000000..3f15e18 --- /dev/null +++ b/src/json.cpp @@ -0,0 +1,534 @@ +/** + * Copyright - See the COPYRIGHT that is included with this distribution. + * pvxs is distributed subject to a Software License Agreement found + * in file LICENSE that is included with this distribution. + */ + +#include +#include +#include + +#include +#include + +#include + +#include "utilpvt.h" + +namespace pvxs { +namespace json { + +namespace { +// undef implies API version 0 +#ifndef EPICS_YAJL_VERSION +typedef long integer_arg; +typedef unsigned size_arg; +#else +typedef long long integer_arg; +typedef size_t size_arg; +#endif + +struct JAny; +// use std::list so that insertion does not invalidate references +typedef std::list > JMap; +typedef std::list JList; + +struct JAny { + aligned_union<8, bool, double, int64_t, std::string, JMap, JList>::type store; + enum type_t { + Placeholder, + Null, + Bool, + Double, + Int64, + String, + List, + Map, + } type = Null; + + template + T& as() { return *reinterpret_cast(&store); } + template + const T& as() const { return *reinterpret_cast(&store); } + + JAny() : store({}), type(Placeholder) {} + JAny(const JAny&) = delete; + JAny& operator=(const JAny&) = delete; + + ~JAny() { + switch(type) { + case Placeholder: + case Null: + case Bool: + case Double: + case Int64: + break; // nothing to do for POD + case String: + as().~basic_string(); + break; + case List: + as().~JList(); + break; + case Map: + as().~JMap(); + break; + } + } +}; + +constexpr size_t stkLimit = 10; + +struct JContext { + std::vector stk; + std::ostringstream msg; + + void exc(const std::exception& e) noexcept { + try { + msg<<"\nError: "<(); + list.emplace_back(); + auto& newtop = list.back(); + stk.push_back(&newtop); + return newtop; + } else { + throw std::logic_error("invalid stack state for array"); + } + } + void consume_top() { + assert(!stk.empty()); + stk.pop_back(); + } +}; + +struct YHandle { + yajl_handle handle; + explicit + YHandle(yajl_handle handle) + :handle(handle) + { + if(!handle) + throw std::runtime_error("yajl_alloc fails"); + } + ~YHandle() { + yajl_free(handle); + } + operator yajl_handle() { return handle; } +}; + +#define TRY \ + auto ctx = static_cast(raw); \ + try + +#define CATCH() \ + catch(std::exception& e){ \ + ctx->exc(e); \ + return 0; \ + } + +int jvalue_null(void * raw) noexcept { + TRY { + auto& top = ctx->setup_top(); + top.type = JAny::Null; + ctx->consume_top(); + + return 1; + }CATCH() +} + +int jvalue_boolean(void * raw, int val) noexcept { + TRY { + auto& top = ctx->setup_top(); + top.type = JAny::Bool; + top.as() = val; + ctx->consume_top(); + + return 1; + }CATCH() +} + +int jvalue_integer(void * raw, integer_arg val) noexcept { + TRY { + auto& top = ctx->setup_top(); + top.type = JAny::Int64; + top.as() = val; + ctx->consume_top(); + + return 1; + }CATCH() +} + +int jvalue_real(void * raw, double val) noexcept { + TRY { + auto& top = ctx->setup_top(); + top.type = JAny::Double; + top.as() = val; + ctx->consume_top(); + + return 1; + }CATCH() +} + +int jvalue_string(void * raw, const unsigned char * val, size_arg len) { + TRY { + auto& top = ctx->setup_top(); + new (&top.store) std::string((const char*)val, len); + top.type = JAny::String; + ctx->consume_top(); + + return 1; + }CATCH() +} + +int jvalue_start_map(void * raw) noexcept { + TRY { + if(ctx->stk.size() >= stkLimit) + throw std::runtime_error("JSON structure too deep!"); + + auto& top = ctx->setup_top(); + new (&top.store) JMap(); + top.type = JAny::Map; + + return 1; + }CATCH() +} + +int jvalue_map_key(void * raw, const unsigned char * val, size_arg len) { + TRY { + assert(!ctx->stk.empty()); + auto& top = *ctx->stk.back(); + assert(top.type==JAny::Map); + + auto& map = top.as(); + map.emplace_back(std::piecewise_construct, + std::forward_as_tuple((const char*)val, len), + std::forward_as_tuple()); + ctx->stk.push_back(&map.back().second); + + return 1; + }CATCH() +} + +int jvalue_end_map(void * raw) noexcept { + TRY { + assert(!ctx->stk.empty()); + const auto& top = ctx->stk.back(); + assert(top->type==JAny::Map); + + ctx->consume_top(); + + return 1; + }CATCH() +} + +int jvalue_start_array(void * raw) noexcept { + TRY { + if(ctx->stk.size() >= stkLimit) + throw std::runtime_error("JSON structure too deep!"); + + auto& top = ctx->stk.back(); + assert(top->type==JAny::Placeholder); + new (&top->store) JList(); + top->type = JAny::List; + + return 1; + }CATCH() +} + +int jvalue_end_array(void * raw) noexcept { + TRY { + assert(!ctx->stk.empty()); + const auto& top = ctx->stk.back(); + assert(top->type==JAny::List); + + ctx->consume_top(); + + return 1; + }CATCH() +} + +const yajl_callbacks jvalue_cbs = { + jvalue_null, + jvalue_boolean, + jvalue_integer, + jvalue_real, + nullptr, + jvalue_string, + jvalue_start_map, + jvalue_map_key, + jvalue_end_map, + jvalue_start_array, + jvalue_end_array, +}; + +static +TypeDef type_from_ast(const JAny& src) { + switch(src.type) { + case JAny::Placeholder: + throw std::logic_error("placeholder in JSON AST"); + case JAny::Null: + return TypeDef(TypeCode::Any); + case JAny::Bool: + return TypeDef(TypeCode::Bool); + case JAny::Double: + return TypeDef(TypeCode::Float64); + case JAny::Int64: + return TypeDef(TypeCode::Int64); + case JAny::String: + return TypeDef(TypeCode::String); + case JAny::Map: { + std::vector members; + auto& map = src.as(); + for(auto& pair : map) { + members.emplace_back(type_from_ast(pair.second).as(pair.first)); + } + // TODO: infer struct_t name? + // TODO: infer Union? + return TypeDef(TypeCode::Struct, std::string(), members); + } + case JAny::List: { + const auto& list = src.as(); + if(list.empty()) + return TypeDef(TypeCode::Float64A); + TypeCode ltype = TypeCode::Null; + for(const auto& item : list) { + TypeCode itype; + switch(item.type) { + case JAny::Placeholder: + throw std::logic_error("placeholder in JSON AST"); + case JAny::Null: + throw std::runtime_error("Unable to infer list with null"); + case JAny::Int64: + itype = TypeCode::Int64A; + break; + case JAny::Double: + itype = TypeCode::Float64A; + break; + case JAny::Bool: + itype = TypeCode::BoolA; + break; + case JAny::String: + itype = TypeCode::StringA; + break; + case JAny::List: + case JAny::Map: + throw std::runtime_error("Infer array from JSON list of list/map not yet implemented"); + } + + if(ltype==TypeCode::Null) { // first item + ltype = itype; + } else if(ltype==itype) { + // same, no-op + } else if(ltype.kind()==Kind::Real && (itype.kind()==Kind::Integer || ltype.kind()==Kind::Bool)) { + // silently promote value + } else if(itype.kind()==Kind::Real && (ltype.kind()==Kind::Integer || ltype.kind()==Kind::Bool)) { + // silently promote list type int/bool to real + ltype = itype; + } else if(ltype.kind()==Kind::Integer && itype.kind()==Kind::Bool) { + // silently promote value + } else if(itype.kind()==Kind::Integer && ltype.kind()==Kind::Bool) { + // silently promote bool to int + ltype = itype; + } else { + throw std::runtime_error(SB()<<"Unable to infer array from JSON list containing both "<()); + break; + case JAny::Double: + dest.from(src.as()); + break; + case JAny::Int64: + dest.from(src.as()); + break; + case JAny::String: + dest.from(src.as()); + break; + case JAny::Map: { + auto& map = src.as(); + for(auto& pair : map) { + auto node(dest.lookup(pair.first)); + apply_ast(node, pair.second); + } + } + break; + case JAny::List: { + auto& list = src.as(); + auto dtype(dest.type()); + + if(dtype==TypeCode::StructA || dtype==TypeCode::UnionA) { + shared_array elems(list.size()); + size_t i=0; + for(auto& elem : list) { + if(elem.type!=JAny::Null) { + auto& eval = elems[i] = dest.allocMember(); + apply_ast(eval, elem); + } + i++; + } + dest.from(elems.freeze()); + + } else if(dtype==TypeCode::AnyA) { + shared_array elems(list.size()); + size_t i=0; + for(auto& elem : list) { + if(elem.type!=JAny::Null) { + auto& eval = elems[i] = infer_from_ast(elem); + apply_ast(eval, elem); + } + i++; + } + dest.from(elems.freeze()); + + } else { // array of scalar type + auto arr(allocArray(dtype.arrayType(), list.size())); + auto etype(arr.original_type()); + auto esize(elementSize(etype)); + auto cur(arr.data()); + + for(auto& elem : list) { + ArrayType stype; + switch(elem.type) { + case JAny::Bool: stype = ArrayType::Bool; break; + case JAny::Double: stype = ArrayType::Float64; break; + case JAny::Int64: stype = ArrayType::Int64; break; + case JAny::String: stype = ArrayType::String; break; + default: + throw std::runtime_error(SB()<<"Can't assign "<() = parseTo(num); + } else { // integer + top.type = JAny::Int64; + top.as() = parseTo(num); + } + + return; + } + + // parse into AST + JContext ctx; + ctx.stk.push_back(&top); + +#ifndef EPICS_YAJL_VERSION + yajl_parser_config conf; + memset(&conf, 0, sizeof(conf)); + conf.allowComments = 1; + conf.checkUTF8 = 1; + YHandle handle(yajl_alloc(&jvalue_cbs, &conf, NULL, &ctx)); +#else + YHandle handle(yajl_alloc(&jvalue_cbs, NULL, &ctx)); + + yajl_config(handle, yajl_allow_comments, 1); +#endif + + auto sts(yajl_parse(handle, (const unsigned char*)cur, len)); + auto consumed(yajl_get_bytes_consumed(handle)); + + switch(sts) { + case yajl_status_ok: + for(; consumed < len; consumed++) { + if(!isspace(cur[consumed])) + throw std::runtime_error("Trailing junk after JSON"); + } + // success + break; + case yajl_status_client_canceled: + throw std::runtime_error(ctx.msg.str()); +#ifndef EPICS_YAJL_VERSION + case yajl_status_insufficient_data: + throw std::runtime_error("JSON incomplete"); + break; +#endif + case yajl_status_error: { + auto raw(yajl_get_error(handle, 1, (const unsigned char*)cur, len)); + try { + ctx.msg<<"\nJSON Syntax error: "< + +#include +#include + +#include + +#if EPICS_VERSION_INT < VERSION_INT(3, 15, 0, 0) +# error JSON support requires Base 3.15 or later +#endif + +#ifdef PVXS_EXPERT_API_ENABLED + +namespace pvxs { +class Value; + +namespace json { + +/** Parse JSON string + * + * String must be complete JSON value, object, or sequence. + * Leading and trailing whitespace is allowed and ignored. + * + * A character array must remain valid and unmodified for + * the lifetime of the Parse object referencing it. + * + * @section jsonassign Assignment + * + * When assigning JSON to Value, Value::from() is used. + * So the same range of implicit type conversions are attempted. + * eg. + * + * @code + * Value top = TypeDef(TypeCode::Int32).build(); + * + * json::Parse("42").into(top); + * // equivalent to + * top.from(42); + * + * and + * + * json::Parse("\"42\"").into(top); + * // equivalent to + * top.from("42"); + * @endcode + * + * @since UNRELEASED + */ +struct Parse { +private: + const char* base; + size_t count; +public: + + //! Parse from Nil terminated string. + inline + Parse(const char *s) noexcept + :base(s) + ,count(s ? strlen(s) : 0u) + {} + //! Parse Character array without terminating Nil. + inline + constexpr Parse(const char *s, size_t c) noexcept + :base(s) + ,count(c) + {} + //! Parse from String + inline + Parse(const std::string& s) noexcept + :base(s.c_str()) + ,count(s.size()) + {} + + /** Assign previously created Value from parsed string. + * + * Provided Value may be modified on error. + * + * @pre v.valid() + * @throws std::runtime_error on Parse error + */ + PVXS_API + void into(Value& v) const; + + /** Infer type from JSON value + * + * Attempts to interpret the provided JSON into one of the supported Value types. + * The returned Value will be assigned from the parsed JSON. + * + * @throws std::runtime_error on Parse or type inference error + * + * Not all types may be inferred. The currently supported inferences are: + * + * - null -> TypeCode::Any + * - Boolean -> TypeCode::Bool + * - String -> TypeCode::String + * - Number -> TypeCode::Int64 or TypeCode::Float64 (if value appears to be real) + * - Mapping -> Struct + * - Sequence -> Array depending on element types. + * Currently only BoolA, Int64A, Float64A, or StringA may be inferred. + */ + PVXS_API + Value as() const; +}; + +} // namespace json +} // namespace pvxs + +#endif // PVXS_EXPERT_API_ENABLED + +#endif // PVXS_JSON_H diff --git a/test/Makefile b/test/Makefile index 80fd6d6..bfe42c3 100644 --- a/test/Makefile +++ b/test/Makefile @@ -51,6 +51,12 @@ TESTPROD_HOST += testdata testdata_SRCS += testdata.cpp TESTS += testdata +ifdef BASE_3_15 +TESTPROD_HOST += testjson +testjson_SRCS += testjson.cpp +TESTS += testjson +endif + TESTPROD_HOST += testnt testnt_SRCS += testnt.cpp TESTS += testnt diff --git a/test/testjson.cpp b/test/testjson.cpp new file mode 100644 index 0000000..161437e --- /dev/null +++ b/test/testjson.cpp @@ -0,0 +1,330 @@ +/** + * Copyright - See the COPYRIGHT that is included with this distribution. + * pvxs is distributed subject to a Software License Agreement found + * in file LICENSE that is included with this distribution. + */ + +#include + +#include + +#include +#include + +#define PVXS_ENABLE_EXPERT_API + +#include +#include +#include +#include + +#include "utilpvt.h" + +using namespace pvxs; +namespace { + +void testBad() +{ + testDiag("%s", __func__); + + static const char* inputs[] = { + "", + " 14 x", + " {", + " {} extra", + R"({"value":{"A":[1,2], "B":"[1.5, 2.5]}}")", + }; + + for(size_t i=0; i([inp](){ + Value empty; + json::Parse(inp).into(empty); + })<<" Expected error from "<(), "hello"); + } + + { + Value val(TypeDef(TypeCode::Int16).create()); + json::Parse("42").into(val); + testEq(val.as(), 42); + } + + { + Value val(TypeDef(TypeCode::Int64).create()); + json::Parse(" -42 ").into(val); + testEq(val.as(), -42); + } + + { + Value val(TypeDef(TypeCode::Float64).create()); + json::Parse(" -42.5 ").into(val); + testEq(val.as(), -42.5); + } +} + +void testStruct() +{ + testDiag("%s", __func__); + + { + Value top(nt::NTScalar{TypeCode::UInt16, true}.create()); + json::Parse(R"( {"value":43, "alarm":{"severity":1, "status":2, "message":"hello"}, "display":{}} )").into(top); + testEq(top["value"].as(), 43); + testEq(top["alarm.severity"].as(), 1); + testEq(top["alarm.status"].as(), 2); + testEq(top["alarm.message"].as(), "hello"); + } +} + +void testArrayOfScalar() +{ + testDiag("%s", __func__); + + { + Value val(TypeDef(TypeCode::Int32A).create()); + json::Parse(R"( [1, 2,3 ] )").into(val); + shared_array expect({1, 2, 3}); + testArrEq(val.as>(), expect); + } + + { + Value val(TypeDef(TypeCode::Int32A).create()); + json::Parse(R"( [1, 2.5,3 ] )").into(val); + shared_array expect({1, 2, 3}); + testArrEq(val.as>(), expect); + } + + { + Value val(TypeDef(TypeCode::Float64A).create()); + json::Parse(R"( [1.5, 2,3 ] )").into(val); + shared_array expect({1.5, 2.0, 3.0}); + testArrEq(val.as>(), expect); + } + + { + Value val(TypeDef(TypeCode::StringA).create()); + json::Parse(R"( ["1", "hello", "world" ] )").into(val); + shared_array expect({"1", "hello", "world"}); + testArrEq(val.as>(), expect); + } +} + +void testStructArray() +{ + using namespace pvxs::members; + + testDiag("%s", __func__); + + { + Value val(TypeDef(TypeCode::StructA, { + Int32("ival"), + String("sval"), + Int32A("aval"), + }).create()); + json::Parse("[" + "{\"ival\":1, \"sval\":\"hello\"}," + "null," + "{\"aval\": [4,5,6]}" + "]").into(val); + testStrEq(std::string(SB()<(), 42); + } + { + auto val(json::Parse(R"(42.5)").as()); + testEq(val.type(), TypeCode::Float64); + testEq(val.as(), 42.5); + } + { + auto val(json::Parse(R"("foo")").as()); + testEq(val.type(), TypeCode::String); + testEq(val.as(), "foo"); + } + { + auto val(json::Parse(R"([])").as()); + testEq(val.type(), TypeCode::Float64A); + testEq(val.as>().size(), 0u); + } + { + auto val(json::Parse(R"([1,2])").as()); + testEq(val.type(), TypeCode::Int64A); + shared_array expect({1,2}); + testArrEq(val.as>(), expect); + } + { + auto val(json::Parse(R"([1,2.0])").as()); + testEq(val.type(), TypeCode::Float64A); + shared_array expect({1,2}); + testArrEq(val.as>(), expect); + } + { + auto val(json::Parse(R"([2.0,1])").as()); + testEq(val.type(), TypeCode::Float64A); + shared_array expect({2,1}); + testArrEq(val.as>(), expect); + } + { + auto val(json::Parse(R"([1,false])").as()); + testEq(val.type(), TypeCode::Int64A); + shared_array expect({1,0}); + testArrEq(val.as>(), expect); + } + { + auto val(json::Parse(R"([false,1])").as()); + testEq(val.type(), TypeCode::Int64A); + shared_array expect({0,1}); + testArrEq(val.as>(), expect); + } + { + auto val(json::Parse(R"([true, false])").as()); + testEq(val.type(), TypeCode::BoolA); + shared_array expect({true, false}); + testArrEq(val.as>(), expect); + } + { + auto val(json::Parse(R"(["hello", "world"])").as()); + testEq(val.type(), TypeCode::StringA); + shared_array expect({"hello", "world"}); + testArrEq(val.as>(), expect); + } + { + auto val(json::Parse(R"({"foo":1, "bar":{"a":"blah", "b":null, "c":false}})").as()); + testStrEq((SB()<