From ee516614813c4ef5762d436f26f2138f09c96410 Mon Sep 17 00:00:00 2001 From: Qianqian Fang Date: Fri, 29 Apr 2022 15:17:30 -0400 Subject: [PATCH] Support UBJSON-derived Binary JData (BJData) format (#3336) * support UBJSON-derived Binary JData (BJData) format * fix Codacy warning * partially fix VS compilation errors * fix additional VS errors * fix more VS compilation errors * fix additional warnings and errors for clang and msvc * add more tests to cover the new bjdata types * add tests for optimized ndarray, improve coverage, fix clang/gcc warnings * gcc warn useless conversion but msvc gives an error * fix ci_test errors * complete test coverage, fix ci_test errors * add half precision error test * fix No newline at end of file error by clang * simplify endian condition, format unit-bjdata * remove broken test due to alloc limit * full coverage, I hope * move bjdata new markers from default to the same level as ubjson markers * fix ci errors, add tests for new bjdata switch structure * make is_bjdata const after using initializer list * remove the unwanted assert * move is_bjdata to an optional param to write_ubjson * pass use_bjdata via output adapter * revert order to avoid msvc 2015 unreferenced formal param error * update BJData Spect V1 Draft-2 URL after spec release * amalgamate code * code polishing following @gregmarr's feedback * make use_bjdata a non-default parameter * fix ci error, remove unwanted param comment * encode and decode bjdata ndarray in jdata annotations, enable roundtrip tests * partially fix ci errors, add tests to improve coverage * polish patch to remove ci errors * fix a ndarray dim vector condition * fix clang tidy error * add sax test cases for ndarray * add additional sax event tests * adjust sax event numbering * fix sax tests * ndarray can only be used with array containers, discard if used in object * complete test coverage * disable [{SHTFNZ in optimized type due to security risks in #2793 and hampered readability * fix ci error * move OutputIsLittleEndian from tparam to param to replace use_bjdata * fix ci clang gcc error * fix ci static analysis error * update json_test_data to 3.1.0, enable file-based bjdata unit tests * fix stack overflow error on msvc 2019 and 2022 * use https link, update sax_parse_error after rebase * make input_format const and use initializer * return bool for write_bjdata_ndarray * test write_bjdata_ndarray return value as boolean * fix ci error --- README.md | 6 +- cmake/ci.cmake | 2 +- cmake/download_test_data.cmake | 2 +- .../nlohmann/detail/input/binary_reader.hpp | 420 ++- .../nlohmann/detail/input/input_adapters.hpp | 2 +- .../nlohmann/detail/output/binary_writer.hpp | 291 +- include/nlohmann/json.hpp | 111 +- single_include/nlohmann/json.hpp | 824 +++- test/CMakeLists.txt | 2 +- test/Makefile | 5 +- test/src/fuzzer-parse_bjdata.cpp | 84 + test/src/unit-bjdata.cpp | 3355 +++++++++++++++++ 12 files changed, 4877 insertions(+), 227 deletions(-) create mode 100644 test/src/fuzzer-parse_bjdata.cpp create mode 100644 test/src/unit-bjdata.cpp diff --git a/README.md b/README.md index 20e7c48ea..c3990a69b 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ - [Implicit conversions](#implicit-conversions) - [Conversions to/from arbitrary types](#arbitrary-types-conversions) - [Specializing enum conversion](#specializing-enum-conversion) - - [Binary formats (BSON, CBOR, MessagePack, and UBJSON)](#binary-formats-bson-cbor-messagepack-and-ubjson) + - [Binary formats (BSON, CBOR, MessagePack, UBJSON, and BJData)](#binary-formats-bson-cbor-messagepack-ubjson-and-bjdata) - [Supported compilers](#supported-compilers) - [Integration](#integration) - [CMake](#cmake) @@ -961,9 +961,9 @@ Other Important points: - When using `get()`, undefined JSON values will default to the first pair specified in your map. Select this default pair carefully. - If an enum or JSON value is specified more than once in your map, the first matching occurrence from the top of the map will be returned when converting to or from JSON. -### Binary formats (BSON, CBOR, MessagePack, and UBJSON) +### Binary formats (BSON, CBOR, MessagePack, UBJSON, and BJData) -Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [BSON](https://bsonspec.org) (Binary JSON), [CBOR](https://cbor.io) (Concise Binary Object Representation), [MessagePack](https://msgpack.org), and [UBJSON](https://ubjson.org) (Universal Binary JSON Specification) to efficiently encode JSON values to byte vectors and to decode such vectors. +Though JSON is a ubiquitous data format, it is not a very compact format suitable for data exchange, for instance over a network. Hence, the library supports [BSON](https://bsonspec.org) (Binary JSON), [CBOR](https://cbor.io) (Concise Binary Object Representation), [MessagePack](https://msgpack.org), [UBJSON](https://ubjson.org) (Universal Binary JSON Specification) and [BJData](https://neurojson.org/bjdata) (Binary JData) to efficiently encode JSON values to byte vectors and to decode such vectors. ```cpp // create a JSON value diff --git a/cmake/ci.cmake b/cmake/ci.cmake index be564fa99..5ae05a9d9 100644 --- a/cmake/ci.cmake +++ b/cmake/ci.cmake @@ -683,7 +683,7 @@ add_custom_target(ci_infer add_custom_target(ci_offline_testdata COMMAND mkdir -p ${PROJECT_BINARY_DIR}/build_offline_testdata/test_data - COMMAND cd ${PROJECT_BINARY_DIR}/build_offline_testdata/test_data && ${GIT_TOOL} clone -c advice.detachedHead=false --branch v3.0.0 https://github.com/nlohmann/json_test_data.git --quiet --depth 1 + COMMAND cd ${PROJECT_BINARY_DIR}/build_offline_testdata/test_data && ${GIT_TOOL} clone -c advice.detachedHead=false --branch v3.1.0 https://github.com/nlohmann/json_test_data.git --quiet --depth 1 COMMAND ${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=Debug -GNinja -DJSON_BuildTests=ON -DJSON_FastTests=ON -DJSON_TestDataDirectory=${PROJECT_BINARY_DIR}/build_offline_testdata/test_data/json_test_data diff --git a/cmake/download_test_data.cmake b/cmake/download_test_data.cmake index f516a7c3b..1bb998dae 100644 --- a/cmake/download_test_data.cmake +++ b/cmake/download_test_data.cmake @@ -1,5 +1,5 @@ set(JSON_TEST_DATA_URL https://github.com/nlohmann/json_test_data) -set(JSON_TEST_DATA_VERSION 3.0.0) +set(JSON_TEST_DATA_VERSION 3.1.0) # if variable is set, use test data from given directory rather than downloading them if(JSON_TestDataDirectory) diff --git a/include/nlohmann/detail/input/binary_reader.hpp b/include/nlohmann/detail/input/binary_reader.hpp index 4041cedba..6474b8b05 100644 --- a/include/nlohmann/detail/input/binary_reader.hpp +++ b/include/nlohmann/detail/input/binary_reader.hpp @@ -12,6 +12,7 @@ #include // char_traits, string #include // make_pair, move #include // vector +#include // map #include #include @@ -74,7 +75,7 @@ class binary_reader @param[in] adapter input adapter to read from */ - explicit binary_reader(InputAdapterType&& adapter) noexcept : ia(std::move(adapter)) + explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json) noexcept : ia(std::move(adapter)), input_format(format) { (void)detail::is_sax_static_asserts {}; } @@ -118,6 +119,7 @@ class binary_reader break; case input_format_t::ubjson: + case input_format_t::bjdata: result = parse_ubjson_internal(); break; @@ -129,7 +131,7 @@ class binary_reader // strict mode: next byte must be EOF if (result && strict) { - if (format == input_format_t::ubjson) + if (input_format == input_format_t::ubjson || input_format == input_format_t::bjdata) { get_ignore_noop(); } @@ -141,7 +143,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != std::char_traits::eof())) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read, - exception_message(format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); + exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); } } @@ -1844,7 +1846,7 @@ class binary_reader get(); // TODO(niels): may we ignore N here? } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) { return false; } @@ -1854,52 +1856,154 @@ class binary_reader case 'U': { std::uint8_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'i': { std::int8_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'I': { std::int16_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'l': { std::int32_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'L': { std::int64_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); + } + + case 'u': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t len{}; + return get_number(input_format, len) && get_string(input_format, len, result); + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t len{}; + return get_number(input_format, len) && get_string(input_format, len, result); + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t len{}; + return get_number(input_format, len) && get_string(input_format, len, result); } default: - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::ubjson, concat("expected length type specification (U, i, I, l, L); last byte: 0x", last_token), "string"), nullptr)); + break; + } + auto last_token = get_token_string(); + std::string message; + + if (input_format != input_format_t::bjdata) + { + message = "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token; + } + else + { + message = "expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x" + last_token; + } + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "string"), nullptr)); + } + + /*! + @param[out] dim an integer vector storing the ND array dimensions + @return whether reading ND array size vector is successful + */ + bool get_ubjson_ndarray_size(std::vector& dim) + { + std::pair size_and_type; + size_t dimlen = 0; + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, size_and_type.second))) + { + return false; + } + dim.push_back(dimlen); + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen))) + { + return false; + } + dim.push_back(dimlen); + } + } } + else + { + while (current != ']') + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, current))) + { + return false; + } + dim.push_back(dimlen); + get_ignore_noop(); + } + } + return true; } /*! @param[out] result determined size @return whether size determination completed */ - bool get_ubjson_size_value(std::size_t& result) + bool get_ubjson_size_value(std::size_t& result, char_int_type prefix = 0) { - switch (get_ignore_noop()) + if (prefix == 0) + { + prefix = get_ignore_noop(); + } + + switch (prefix) { case 'U': { std::uint8_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -1910,7 +2014,7 @@ class binary_reader case 'i': { std::int8_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -1921,7 +2025,7 @@ class binary_reader case 'I': { std::int16_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -1932,7 +2036,7 @@ class binary_reader case 'l': { std::int32_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -1943,7 +2047,7 @@ class binary_reader case 'L': { std::int64_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -1951,13 +2055,105 @@ class binary_reader return true; } - default: + case 'u': { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::ubjson, concat("expected length type specification (U, i, I, l, L) after '#'; last byte: 0x", last_token), "size"), nullptr)); + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) + { + return false; + } + result = detail::conditional_static_cast(number); + return true; } + + case '[': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::vector dim; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) + { + return false; + } + if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector + { + result = dim.at(dim.size() - 1); + return true; + } + if (!dim.empty()) // if ndarray, convert to an object in JData annotated array format + { + string_t key = "_ArraySize_"; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size()))) + { + return false; + } + result = 1; + for (auto i : dim) + { + result *= i; + if (JSON_HEDLEY_UNLIKELY(!sax->number_integer(static_cast(i)))) + { + return false; + } + } + result |= (1ull << (sizeof(result) * 8 - 1)); // low 63 bit of result stores the total element count, sign-bit indicates ndarray + return sax->end_array(); + } + result = 0; + return true; + } + + default: + break; + } + auto last_token = get_token_string(); + std::string message; + + if (input_format != input_format_t::bjdata) + { + message = "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token; + } + else + { + message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; } + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); } /*! @@ -1979,8 +2175,10 @@ class binary_reader if (current == '$') { + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "type"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type") || (input_format == input_format_t::bjdata && std::find(bjdx.begin(), bjdx.end(), result.second) != bjdx.end() ))) { return false; } @@ -1988,13 +2186,13 @@ class binary_reader get_ignore_noop(); if (JSON_HEDLEY_UNLIKELY(current != '#')) { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) { return false; } auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::ubjson, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); + exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); } return get_ubjson_size_value(result.first); @@ -2017,7 +2215,7 @@ class binary_reader switch (prefix) { case std::char_traits::eof(): // EOF - return unexpect_eof(input_format_t::ubjson, "value"); + return unexpect_eof(input_format, "value"); case 'T': // true return sax->boolean(true); @@ -2030,43 +2228,125 @@ class binary_reader case 'U': { std::uint8_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_unsigned(number); + return get_number(input_format, number) && sax->number_unsigned(number); } case 'i': { std::int8_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); } case 'I': { std::int16_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); } case 'l': { std::int32_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); } case 'L': { std::int64_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); + } + + case 'u': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t number{}; + return get_number(input_format, number) && sax->number_unsigned(number); + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t number{}; + return get_number(input_format, number) && sax->number_unsigned(number); + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t number{}; + return get_number(input_format, number) && sax->number_unsigned(number); + } + + case 'h': + { + if (input_format != input_format_t::bjdata) + { + break; + } + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte2 << 8u) + byte1); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), ""); } case 'd': { float number{}; - return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && sax->number_float(static_cast(number), ""); } case 'D': { double number{}; - return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && sax->number_float(static_cast(number), ""); } case 'H': @@ -2077,7 +2357,7 @@ class binary_reader case 'C': // char { get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "char"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "char"))) { return false; } @@ -2085,7 +2365,7 @@ class binary_reader { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::ubjson, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); } string_t s(1, static_cast(current)); return sax->string(s); @@ -2104,12 +2384,10 @@ class binary_reader return get_ubjson_object(); default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::ubjson, concat("invalid byte: 0x", last_token), "value"), nullptr)); - } + break; } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); } /*! @@ -2123,6 +2401,44 @@ class binary_reader return false; } + // detect and encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): + // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} + + if (input_format == input_format_t::bjdata && size_and_type.first != string_t::npos && size_and_type.first >= (1ull << (sizeof(std::size_t) * 8 - 1))) + { + std::map bjdtype = {{'U', "uint8"}, {'i', "int8"}, {'u', "uint16"}, {'I', "int16"}, + {'m', "uint32"}, {'l', "int32"}, {'M', "uint64"}, {'L', "int64"}, {'d', "single"}, {'D', "double"}, {'C', "char"} + }; + + string_t key = "_ArrayType_"; + if (JSON_HEDLEY_UNLIKELY(bjdtype.count(size_and_type.second) == 0 || !sax->key(key) || !sax->string(bjdtype[size_and_type.second]) )) + { + return false; + } + + if (size_and_type.second == 'C') + { + size_and_type.second = 'U'; + } + + size_and_type.first &= ~(1ull << (sizeof(std::size_t) * 8 - 1)); + key = "_ArrayData_"; + if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) )) + { + return false; + } + + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + } + + return (sax->end_array() && sax->end_object()); + } + if (size_and_type.first != string_t::npos) { if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) @@ -2185,6 +2501,11 @@ class binary_reader return false; } + if (input_format == input_format_t::bjdata && size_and_type.first != string_t::npos && size_and_type.first >= (1ull << (sizeof(std::size_t) * 8 - 1))) + { + return false; + } + string_t key; if (size_and_type.first != string_t::npos) { @@ -2267,7 +2588,7 @@ class binary_reader for (std::size_t i = 0; i < size; ++i) { get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "number"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) { return false; } @@ -2286,7 +2607,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input)) { return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format_t::ubjson, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } switch (result_number) @@ -2313,7 +2634,7 @@ class binary_reader case token_type::literal_or_value: default: return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format_t::ubjson, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } } @@ -2362,6 +2683,8 @@ class binary_reader @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ template bool get_number(const input_format_t format, NumberType& result) @@ -2377,7 +2700,7 @@ class binary_reader } // reverse byte order prior to conversion if necessary - if (is_little_endian != InputIsLittleEndian) + if (is_little_endian != (InputIsLittleEndian || format == input_format_t::bjdata)) { vec[sizeof(NumberType) - i - 1] = static_cast(current); } @@ -2514,6 +2837,10 @@ class binary_reader error_msg += "BSON"; break; + case input_format_t::bjdata: + error_msg += "BJData"; + break; + case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -2535,6 +2862,9 @@ class binary_reader /// whether we can assume little endianness const bool is_little_endian = little_endianness(); + /// input format + const input_format_t input_format = input_format_t::json; + /// the SAX parser json_sax_t* sax = nullptr; }; diff --git a/include/nlohmann/detail/input/input_adapters.hpp b/include/nlohmann/detail/input/input_adapters.hpp index d196aec54..4c1ad3cef 100644 --- a/include/nlohmann/detail/input/input_adapters.hpp +++ b/include/nlohmann/detail/input/input_adapters.hpp @@ -23,7 +23,7 @@ namespace nlohmann namespace detail { /// the supported input formats -enum class input_format_t { json, cbor, msgpack, ubjson, bson }; +enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata }; //////////////////// // input adapters // diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index c018b8ccd..bb44a1811 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -2,12 +2,14 @@ #include // reverse #include // array +#include // map #include // isnan, isinf #include // uint8_t, uint16_t, uint32_t, uint64_t #include // memcpy #include // numeric_limits #include // string #include // move +#include // vector #include #include @@ -724,9 +726,11 @@ class binary_writer @param[in] use_count whether to use '#' prefixes (optimized format) @param[in] use_type whether to use '$' prefixes (optimized format) @param[in] add_prefix whether prefixes need to be used for this value + @param[in] use_bjdata whether write in BJData format, default is false */ void write_ubjson(const BasicJsonType& j, const bool use_count, - const bool use_type, const bool add_prefix = true) + const bool use_type, const bool add_prefix = true, + const bool use_bjdata = false) { switch (j.type()) { @@ -752,19 +756,19 @@ class binary_writer case value_t::number_integer: { - write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix, use_bjdata); break; } case value_t::number_unsigned: { - write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix, use_bjdata); break; } case value_t::number_float: { - write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix, use_bjdata); break; } @@ -774,7 +778,7 @@ class binary_writer { oa->write_character(to_char_type('S')); } - write_number_with_ubjson_prefix(j.m_value.string->size(), true); + write_number_with_ubjson_prefix(j.m_value.string->size(), true, use_bjdata); oa->write_characters( reinterpret_cast(j.m_value.string->c_str()), j.m_value.string->size()); @@ -792,14 +796,16 @@ class binary_writer if (use_type && !j.m_value.array->empty()) { JSON_ASSERT(use_count); - const CharType first_prefix = ubjson_prefix(j.front()); + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin() + 1, j.end(), - [this, first_prefix](const BasicJsonType & v) + [this, first_prefix, use_bjdata](const BasicJsonType & v) { - return ubjson_prefix(v) == first_prefix; + return ubjson_prefix(v, use_bjdata) == first_prefix; }); - if (same_prefix) + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); @@ -810,12 +816,12 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.array->size(), true); + write_number_with_ubjson_prefix(j.m_value.array->size(), true, use_bjdata); } for (const auto& el : *j.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required); + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata); } if (!use_count) @@ -843,7 +849,7 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.binary->size(), true); + write_number_with_ubjson_prefix(j.m_value.binary->size(), true, use_bjdata); } if (use_type) @@ -871,6 +877,14 @@ class binary_writer case value_t::object: { + if (use_bjdata && j.m_value.object->size() == 3 && j.m_value.object->find("_ArrayType_") != j.m_value.object->end() && j.m_value.object->find("_ArraySize_") != j.m_value.object->end() && j.m_value.object->find("_ArrayData_") != j.m_value.object->end()) + { + if (!write_bjdata_ndarray(*j.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) + { + break; + } + } + if (add_prefix) { oa->write_character(to_char_type('{')); @@ -880,14 +894,16 @@ class binary_writer if (use_type && !j.m_value.object->empty()) { JSON_ASSERT(use_count); - const CharType first_prefix = ubjson_prefix(j.front()); + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin(), j.end(), - [this, first_prefix](const BasicJsonType & v) + [this, first_prefix, use_bjdata](const BasicJsonType & v) { - return ubjson_prefix(v) == first_prefix; + return ubjson_prefix(v, use_bjdata) == first_prefix; }); - if (same_prefix) + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); @@ -898,16 +914,16 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.object->size(), true); + write_number_with_ubjson_prefix(j.m_value.object->size(), true, use_bjdata); } for (const auto& el : *j.m_value.object) { - write_number_with_ubjson_prefix(el.first.size(), true); + write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); oa->write_characters( reinterpret_cast(el.first.c_str()), el.first.size()); - write_ubjson(el.second, use_count, use_type, prefix_required); + write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata); } if (!use_count) @@ -974,7 +990,7 @@ class binary_writer const double value) { write_bson_entry_header(name, 0x01); - write_number(value); + write_number(value, true); } /*! @@ -993,7 +1009,7 @@ class binary_writer { write_bson_entry_header(name, 0x02); - write_number(static_cast(value.size() + 1ul)); + write_number(static_cast(value.size() + 1ul), true); oa->write_characters( reinterpret_cast(value.c_str()), value.size() + 1); @@ -1026,12 +1042,12 @@ class binary_writer if ((std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)()) { write_bson_entry_header(name, 0x10); // int32 - write_number(static_cast(value)); + write_number(static_cast(value), true); } else { write_bson_entry_header(name, 0x12); // int64 - write_number(static_cast(value)); + write_number(static_cast(value), true); } } @@ -1054,12 +1070,12 @@ class binary_writer if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { write_bson_entry_header(name, 0x10 /* int32 */); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_value.number_unsigned), true); } else if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { write_bson_entry_header(name, 0x12 /* int64 */); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_value.number_unsigned), true); } else { @@ -1107,7 +1123,7 @@ class binary_writer const typename BasicJsonType::array_t& value) { write_bson_entry_header(name, 0x04); // array - write_number(static_cast(calc_bson_array_size(value))); + write_number(static_cast(calc_bson_array_size(value)), true); std::size_t array_index = 0ul; @@ -1127,7 +1143,7 @@ class binary_writer { write_bson_entry_header(name, 0x05); - write_number(static_cast(value.size())); + write_number(static_cast(value.size()), true); write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); oa->write_characters(reinterpret_cast(value.data()), value.size()); @@ -1249,7 +1265,7 @@ class binary_writer */ void write_bson_object(const typename BasicJsonType::object_t& value) { - write_number(static_cast(calc_bson_object_size(value))); + write_number(static_cast(calc_bson_object_size(value)), true); for (const auto& el : value) { @@ -1295,20 +1311,22 @@ class binary_writer template::value, int>::type = 0> void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) + const bool add_prefix, + const bool use_bjdata) { if (add_prefix) { oa->write_character(get_ubjson_float_prefix(n)); } - write_number(n); + write_number(n, use_bjdata); } // UBJSON: write number (unsigned integer) template::value, int>::type = 0> void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) + const bool add_prefix, + const bool use_bjdata) { if (n <= static_cast((std::numeric_limits::max)())) { @@ -1316,7 +1334,7 @@ class binary_writer { oa->write_character(to_char_type('i')); // int8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if (n <= (std::numeric_limits::max)()) { @@ -1324,7 +1342,7 @@ class binary_writer { oa->write_character(to_char_type('U')); // uint8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if (n <= static_cast((std::numeric_limits::max)())) { @@ -1332,7 +1350,15 @@ class binary_writer { oa->write_character(to_char_type('I')); // int16 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('u')); // uint16 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if (n <= static_cast((std::numeric_limits::max)())) { @@ -1340,7 +1366,15 @@ class binary_writer { oa->write_character(to_char_type('l')); // int32 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('m')); // uint32 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if (n <= static_cast((std::numeric_limits::max)())) { @@ -1348,7 +1382,15 @@ class binary_writer { oa->write_character(to_char_type('L')); // int64 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('M')); // uint64 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else { @@ -1358,7 +1400,7 @@ class binary_writer } const auto number = BasicJsonType(n).dump(); - write_number_with_ubjson_prefix(number.size(), true); + write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { oa->write_character(to_char_type(static_cast(number[i]))); @@ -1371,7 +1413,8 @@ class binary_writer std::is_signed::value&& !std::is_floating_point::value, int >::type = 0 > void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) + const bool add_prefix, + const bool use_bjdata) { if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -1379,7 +1422,7 @@ class binary_writer { oa->write_character(to_char_type('i')); // int8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)())) { @@ -1387,7 +1430,7 @@ class binary_writer { oa->write_character(to_char_type('U')); // uint8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -1395,7 +1438,15 @@ class binary_writer { oa->write_character(to_char_type('I')); // int16 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)()))) + { + if (add_prefix) + { + oa->write_character(to_char_type('u')); // uint16 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -1403,7 +1454,15 @@ class binary_writer { oa->write_character(to_char_type('l')); // int32 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)()))) + { + if (add_prefix) + { + oa->write_character(to_char_type('m')); // uint32 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -1411,7 +1470,7 @@ class binary_writer { oa->write_character(to_char_type('L')); // int64 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } // LCOV_EXCL_START else @@ -1422,7 +1481,7 @@ class binary_writer } const auto number = BasicJsonType(n).dump(); - write_number_with_ubjson_prefix(number.size(), true); + write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { oa->write_character(to_char_type(static_cast(number[i]))); @@ -1434,7 +1493,7 @@ class binary_writer /*! @brief determine the type prefix of container values */ - CharType ubjson_prefix(const BasicJsonType& j) const noexcept + CharType ubjson_prefix(const BasicJsonType& j, const bool use_bjdata) const noexcept { switch (j.type()) { @@ -1458,10 +1517,18 @@ class binary_writer { return 'I'; } + if (use_bjdata && ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)())) + { + return 'u'; + } if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'l'; } + if (use_bjdata && ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)())) + { + return 'm'; + } if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'L'; @@ -1484,14 +1551,26 @@ class binary_writer { return 'I'; } + if (use_bjdata && j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'u'; + } if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'l'; } + if (use_bjdata && j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'm'; + } if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'L'; } + if (use_bjdata && j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'M'; + } // anything else is treated as high-precision number return 'H'; // LCOV_EXCL_LINE } @@ -1525,6 +1604,118 @@ class binary_writer return 'D'; // float 64 } + /*! + @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid + */ + bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type) + { + std::map bjdtype = {{"uint8", 'U'}, {"int8", 'i'}, {"uint16", 'u'}, {"int16", 'I'}, + {"uint32", 'm'}, {"int32", 'l'}, {"uint64", 'M'}, {"int64", 'L'}, {"single", 'd'}, {"double", 'D'}, {"char", 'C'} + }; + + string_t key = "_ArrayType_"; + auto it = bjdtype.find(static_cast(value.at(key))); + if (it == bjdtype.end()) + { + return true; + } + CharType dtype = it->second; + + key = "_ArraySize_"; + std::size_t len = (value.at(key).empty() ? 0 : 1); + for (const auto& el : value.at(key)) + { + len *= static_cast(el.m_value.number_unsigned); + } + + key = "_ArrayData_"; + if (value.at(key).size() != len) + { + return true; + } + + oa->write_character('['); + oa->write_character('$'); + oa->write_character(dtype); + oa->write_character('#'); + + key = "_ArraySize_"; + write_ubjson(value.at(key), use_count, use_type, true, true); + + key = "_ArrayData_"; + if (dtype == 'U' || dtype == 'C') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'i') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'u') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'I') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'm') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'l') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'M') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'L') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'd') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_float), true); + } + } + else if (dtype == 'D') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_float), true); + } + } + return false; + } + /////////////////////// // Utility functions // /////////////////////// @@ -1532,16 +1723,18 @@ class binary_writer /* @brief write a number to output input @param[in] n number of type @a NumberType - @tparam NumberType the type of the number - @tparam OutputIsLittleEndian Set to true if output data is + @param[in] OutputIsLittleEndian Set to true if output data is required to be little endian + @tparam NumberType the type of the number @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ - template - void write_number(const NumberType n) + template + void write_number(const NumberType n, const bool OutputIsLittleEndian = false) { // step 1: write number to array of length NumberType std::array vec{}; diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 1a2da8d45..0a18f1634 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -3773,7 +3773,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); } /// @brief generate SAX events @@ -3788,7 +3788,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); } /// @brief generate SAX events @@ -3809,7 +3809,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); } #ifndef JSON_NO_IO /// @brief deserialize from stream @@ -3965,6 +3965,33 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_ubjson(j, use_size, use_type); } + /// @brief create a BJData serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ + static std::vector to_bjdata(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector result; + to_bjdata(j, result, use_size, use_type); + return result; + } + + /// @brief create a BJData serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ + static void to_bjdata(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + } + + /// @brief create a BJData serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ + static void to_bjdata(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + } + /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ static std::vector to_bson(const basic_json& j) @@ -4000,7 +4027,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); return res ? result : basic_json(value_t::discarded); } @@ -4016,7 +4043,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); return res ? result : basic_json(value_t::discarded); } @@ -4043,7 +4070,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); return res ? result : basic_json(value_t::discarded); } @@ -4058,7 +4085,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -4073,7 +4100,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -4097,7 +4124,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -4112,7 +4139,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -4127,7 +4154,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -4151,10 +4178,64 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } + + /// @brief create a JSON value from an input in BJData format + /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BJData format + /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_bjdata(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BSON format /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ template @@ -4166,7 +4247,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -4181,7 +4262,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -4205,7 +4286,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } /// @} diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index a29c52997..d596180c6 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -5402,6 +5402,7 @@ std::size_t hash(const BasicJsonType& j) #include // char_traits, string #include // make_pair, move #include // vector +#include // map // #include @@ -5433,7 +5434,7 @@ namespace nlohmann namespace detail { /// the supported input formats -enum class input_format_t { json, cbor, msgpack, ubjson, bson }; +enum class input_format_t { json, cbor, msgpack, ubjson, bson, bjdata }; //////////////////// // input adapters // @@ -8450,7 +8451,7 @@ class binary_reader @param[in] adapter input adapter to read from */ - explicit binary_reader(InputAdapterType&& adapter) noexcept : ia(std::move(adapter)) + explicit binary_reader(InputAdapterType&& adapter, const input_format_t format = input_format_t::json) noexcept : ia(std::move(adapter)), input_format(format) { (void)detail::is_sax_static_asserts {}; } @@ -8494,6 +8495,7 @@ class binary_reader break; case input_format_t::ubjson: + case input_format_t::bjdata: result = parse_ubjson_internal(); break; @@ -8505,7 +8507,7 @@ class binary_reader // strict mode: next byte must be EOF if (result && strict) { - if (format == input_format_t::ubjson) + if (input_format == input_format_t::ubjson || input_format == input_format_t::bjdata) { get_ignore_noop(); } @@ -8517,7 +8519,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(current != std::char_traits::eof())) { return sax->parse_error(chars_read, get_token_string(), parse_error::create(110, chars_read, - exception_message(format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); + exception_message(input_format, concat("expected end of input; last byte: 0x", get_token_string()), "value"), nullptr)); } } @@ -10220,7 +10222,7 @@ class binary_reader get(); // TODO(niels): may we ignore N here? } - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) { return false; } @@ -10230,52 +10232,154 @@ class binary_reader case 'U': { std::uint8_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'i': { std::int8_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'I': { std::int16_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'l': { std::int32_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); } case 'L': { std::int64_t len{}; - return get_number(input_format_t::ubjson, len) && get_string(input_format_t::ubjson, len, result); + return get_number(input_format, len) && get_string(input_format, len, result); + } + + case 'u': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t len{}; + return get_number(input_format, len) && get_string(input_format, len, result); + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t len{}; + return get_number(input_format, len) && get_string(input_format, len, result); + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t len{}; + return get_number(input_format, len) && get_string(input_format, len, result); } default: - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::ubjson, concat("expected length type specification (U, i, I, l, L); last byte: 0x", last_token), "string"), nullptr)); + break; } + auto last_token = get_token_string(); + std::string message; + + if (input_format != input_format_t::bjdata) + { + message = "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token; + } + else + { + message = "expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x" + last_token; + } + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "string"), nullptr)); + } + + /*! + @param[out] dim an integer vector storing the ND array dimensions + @return whether reading ND array size vector is successful + */ + bool get_ubjson_ndarray_size(std::vector& dim) + { + std::pair size_and_type; + size_t dimlen = 0; + + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, size_and_type.second))) + { + return false; + } + dim.push_back(dimlen); + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen))) + { + return false; + } + dim.push_back(dimlen); + } + } + } + else + { + while (current != ']') + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_size_value(dimlen, current))) + { + return false; + } + dim.push_back(dimlen); + get_ignore_noop(); + } + } + return true; } /*! @param[out] result determined size @return whether size determination completed */ - bool get_ubjson_size_value(std::size_t& result) + bool get_ubjson_size_value(std::size_t& result, char_int_type prefix = 0) { - switch (get_ignore_noop()) + if (prefix == 0) + { + prefix = get_ignore_noop(); + } + + switch (prefix) { case 'U': { std::uint8_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -10286,7 +10390,7 @@ class binary_reader case 'i': { std::int8_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -10297,7 +10401,7 @@ class binary_reader case 'I': { std::int16_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -10308,7 +10412,7 @@ class binary_reader case 'l': { std::int32_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -10319,7 +10423,7 @@ class binary_reader case 'L': { std::int64_t number{}; - if (JSON_HEDLEY_UNLIKELY(!get_number(input_format_t::ubjson, number))) + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) { return false; } @@ -10327,13 +10431,105 @@ class binary_reader return true; } - default: + case 'u': { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::ubjson, concat("expected length type specification (U, i, I, l, L) after '#'; last byte: 0x", last_token), "size"), nullptr)); + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) + { + return false; + } + result = static_cast(number); + return true; + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t number{}; + if (JSON_HEDLEY_UNLIKELY(!get_number(input_format, number))) + { + return false; + } + result = detail::conditional_static_cast(number); + return true; } + + case '[': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::vector dim; + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_ndarray_size(dim))) + { + return false; + } + if (dim.size() == 1 || (dim.size() == 2 && dim.at(0) == 1)) // return normal array size if 1D row vector + { + result = dim.at(dim.size() - 1); + return true; + } + if (!dim.empty()) // if ndarray, convert to an object in JData annotated array format + { + string_t key = "_ArraySize_"; + if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size()))) + { + return false; + } + result = 1; + for (auto i : dim) + { + result *= i; + if (JSON_HEDLEY_UNLIKELY(!sax->number_integer(static_cast(i)))) + { + return false; + } + } + result |= (1ull << (sizeof(result) * 8 - 1)); // low 63 bit of result stores the total element count, sign-bit indicates ndarray + return sax->end_array(); + } + result = 0; + return true; + } + + default: + break; + } + auto last_token = get_token_string(); + std::string message; + + if (input_format != input_format_t::bjdata) + { + message = "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token; } + else + { + message = "expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x" + last_token; + } + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format, message, "size"), nullptr)); } /*! @@ -10355,8 +10551,10 @@ class binary_reader if (current == '$') { + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + result.second = get(); // must not ignore 'N', because 'N' maybe the type - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "type"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "type") || (input_format == input_format_t::bjdata && std::find(bjdx.begin(), bjdx.end(), result.second) != bjdx.end() ))) { return false; } @@ -10364,13 +10562,13 @@ class binary_reader get_ignore_noop(); if (JSON_HEDLEY_UNLIKELY(current != '#')) { - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "value"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "value"))) { return false; } auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::ubjson, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); + exception_message(input_format, concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr)); } return get_ubjson_size_value(result.first); @@ -10393,7 +10591,7 @@ class binary_reader switch (prefix) { case std::char_traits::eof(): // EOF - return unexpect_eof(input_format_t::ubjson, "value"); + return unexpect_eof(input_format, "value"); case 'T': // true return sax->boolean(true); @@ -10406,43 +10604,125 @@ class binary_reader case 'U': { std::uint8_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_unsigned(number); + return get_number(input_format, number) && sax->number_unsigned(number); } case 'i': { std::int8_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); } case 'I': { std::int16_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); } case 'l': { std::int32_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); } case 'L': { std::int64_t number{}; - return get_number(input_format_t::ubjson, number) && sax->number_integer(number); + return get_number(input_format, number) && sax->number_integer(number); + } + + case 'u': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint16_t number{}; + return get_number(input_format, number) && sax->number_unsigned(number); + } + + case 'm': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint32_t number{}; + return get_number(input_format, number) && sax->number_unsigned(number); + } + + case 'M': + { + if (input_format != input_format_t::bjdata) + { + break; + } + std::uint64_t number{}; + return get_number(input_format, number) && sax->number_unsigned(number); + } + + case 'h': + { + if (input_format != input_format_t::bjdata) + { + break; + } + const auto byte1_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + const auto byte2_raw = get(); + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) + { + return false; + } + + const auto byte1 = static_cast(byte1_raw); + const auto byte2 = static_cast(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const auto half = static_cast((byte2 << 8u) + byte1); + const double val = [&half] + { + const int exp = (half >> 10u) & 0x1Fu; + const unsigned int mant = half & 0x3FFu; + JSON_ASSERT(0 <= exp&& exp <= 32); + JSON_ASSERT(mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000u) != 0 + ? static_cast(-val) + : static_cast(val), ""); } case 'd': { float number{}; - return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && sax->number_float(static_cast(number), ""); } case 'D': { double number{}; - return get_number(input_format_t::ubjson, number) && sax->number_float(static_cast(number), ""); + return get_number(input_format, number) && sax->number_float(static_cast(number), ""); } case 'H': @@ -10453,7 +10733,7 @@ class binary_reader case 'C': // char { get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "char"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "char"))) { return false; } @@ -10461,7 +10741,7 @@ class binary_reader { auto last_token = get_token_string(); return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, - exception_message(input_format_t::ubjson, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); + exception_message(input_format, concat("byte after 'C' must be in range 0x00..0x7F; last byte: 0x", last_token), "char"), nullptr)); } string_t s(1, static_cast(current)); return sax->string(s); @@ -10480,12 +10760,10 @@ class binary_reader return get_ubjson_object(); default: // anything else - { - auto last_token = get_token_string(); - return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, - exception_message(input_format_t::ubjson, concat("invalid byte: 0x", last_token), "value"), nullptr)); - } + break; } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format, "invalid byte: 0x" + last_token, "value"), nullptr)); } /*! @@ -10499,6 +10777,44 @@ class binary_reader return false; } + // detect and encode bjdata ndarray as an object in JData annotated array format (https://github.com/NeuroJSON/jdata): + // {"_ArrayType_" : "typeid", "_ArraySize_" : [n1, n2, ...], "_ArrayData_" : [v1, v2, ...]} + + if (input_format == input_format_t::bjdata && size_and_type.first != string_t::npos && size_and_type.first >= (1ull << (sizeof(std::size_t) * 8 - 1))) + { + std::map bjdtype = {{'U', "uint8"}, {'i', "int8"}, {'u', "uint16"}, {'I', "int16"}, + {'m', "uint32"}, {'l', "int32"}, {'M', "uint64"}, {'L', "int64"}, {'d', "single"}, {'D', "double"}, {'C', "char"} + }; + + string_t key = "_ArrayType_"; + if (JSON_HEDLEY_UNLIKELY(bjdtype.count(size_and_type.second) == 0 || !sax->key(key) || !sax->string(bjdtype[size_and_type.second]) )) + { + return false; + } + + if (size_and_type.second == 'C') + { + size_and_type.second = 'U'; + } + + size_and_type.first &= ~(1ull << (sizeof(std::size_t) * 8 - 1)); + key = "_ArrayData_"; + if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) )) + { + return false; + } + + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_HEDLEY_UNLIKELY(!get_ubjson_value(size_and_type.second))) + { + return false; + } + } + + return (sax->end_array() && sax->end_object()); + } + if (size_and_type.first != string_t::npos) { if (JSON_HEDLEY_UNLIKELY(!sax->start_array(size_and_type.first))) @@ -10561,6 +10877,11 @@ class binary_reader return false; } + if (input_format == input_format_t::bjdata && size_and_type.first != string_t::npos && size_and_type.first >= (1ull << (sizeof(std::size_t) * 8 - 1))) + { + return false; + } + string_t key; if (size_and_type.first != string_t::npos) { @@ -10643,7 +10964,7 @@ class binary_reader for (std::size_t i = 0; i < size; ++i) { get(); - if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format_t::ubjson, "number"))) + if (JSON_HEDLEY_UNLIKELY(!unexpect_eof(input_format, "number"))) { return false; } @@ -10662,7 +10983,7 @@ class binary_reader if (JSON_HEDLEY_UNLIKELY(result_remainder != token_type::end_of_input)) { return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format_t::ubjson, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } switch (result_number) @@ -10689,7 +11010,7 @@ class binary_reader case token_type::literal_or_value: default: return sax->parse_error(chars_read, number_string, parse_error::create(115, chars_read, - exception_message(input_format_t::ubjson, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); + exception_message(input_format, concat("invalid number text: ", number_lexer.get_token_string()), "high-precision number"), nullptr)); } } @@ -10738,6 +11059,8 @@ class binary_reader @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ template bool get_number(const input_format_t format, NumberType& result) @@ -10753,7 +11076,7 @@ class binary_reader } // reverse byte order prior to conversion if necessary - if (is_little_endian != InputIsLittleEndian) + if (is_little_endian != (InputIsLittleEndian || format == input_format_t::bjdata)) { vec[sizeof(NumberType) - i - 1] = static_cast(current); } @@ -10890,6 +11213,10 @@ class binary_reader error_msg += "BSON"; break; + case input_format_t::bjdata: + error_msg += "BJData"; + break; + case input_format_t::json: // LCOV_EXCL_LINE default: // LCOV_EXCL_LINE JSON_ASSERT(false); // NOLINT(cert-dcl03-c,hicpp-static-assert,misc-static-assert) LCOV_EXCL_LINE @@ -10911,6 +11238,9 @@ class binary_reader /// whether we can assume little endianness const bool is_little_endian = little_endianness(); + /// input format + const input_format_t input_format = input_format_t::json; + /// the SAX parser json_sax_t* sax = nullptr; }; @@ -13427,12 +13757,14 @@ class json_ref #include // reverse #include // array +#include // map #include // isnan, isinf #include // uint8_t, uint16_t, uint32_t, uint64_t #include // memcpy #include // numeric_limits #include // string #include // move +#include // vector // #include @@ -14292,9 +14624,11 @@ class binary_writer @param[in] use_count whether to use '#' prefixes (optimized format) @param[in] use_type whether to use '$' prefixes (optimized format) @param[in] add_prefix whether prefixes need to be used for this value + @param[in] use_bjdata whether write in BJData format, default is false */ void write_ubjson(const BasicJsonType& j, const bool use_count, - const bool use_type, const bool add_prefix = true) + const bool use_type, const bool add_prefix = true, + const bool use_bjdata = false) { switch (j.type()) { @@ -14320,19 +14654,19 @@ class binary_writer case value_t::number_integer: { - write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix, use_bjdata); break; } case value_t::number_unsigned: { - write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix, use_bjdata); break; } case value_t::number_float: { - write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix, use_bjdata); break; } @@ -14342,7 +14676,7 @@ class binary_writer { oa->write_character(to_char_type('S')); } - write_number_with_ubjson_prefix(j.m_value.string->size(), true); + write_number_with_ubjson_prefix(j.m_value.string->size(), true, use_bjdata); oa->write_characters( reinterpret_cast(j.m_value.string->c_str()), j.m_value.string->size()); @@ -14360,14 +14694,16 @@ class binary_writer if (use_type && !j.m_value.array->empty()) { JSON_ASSERT(use_count); - const CharType first_prefix = ubjson_prefix(j.front()); + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin() + 1, j.end(), - [this, first_prefix](const BasicJsonType & v) + [this, first_prefix, use_bjdata](const BasicJsonType & v) { - return ubjson_prefix(v) == first_prefix; + return ubjson_prefix(v, use_bjdata) == first_prefix; }); - if (same_prefix) + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); @@ -14378,12 +14714,12 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.array->size(), true); + write_number_with_ubjson_prefix(j.m_value.array->size(), true, use_bjdata); } for (const auto& el : *j.m_value.array) { - write_ubjson(el, use_count, use_type, prefix_required); + write_ubjson(el, use_count, use_type, prefix_required, use_bjdata); } if (!use_count) @@ -14411,7 +14747,7 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.binary->size(), true); + write_number_with_ubjson_prefix(j.m_value.binary->size(), true, use_bjdata); } if (use_type) @@ -14439,6 +14775,14 @@ class binary_writer case value_t::object: { + if (use_bjdata && j.m_value.object->size() == 3 && j.m_value.object->find("_ArrayType_") != j.m_value.object->end() && j.m_value.object->find("_ArraySize_") != j.m_value.object->end() && j.m_value.object->find("_ArrayData_") != j.m_value.object->end()) + { + if (!write_bjdata_ndarray(*j.m_value.object, use_count, use_type)) // decode bjdata ndarray in the JData format (https://github.com/NeuroJSON/jdata) + { + break; + } + } + if (add_prefix) { oa->write_character(to_char_type('{')); @@ -14448,14 +14792,16 @@ class binary_writer if (use_type && !j.m_value.object->empty()) { JSON_ASSERT(use_count); - const CharType first_prefix = ubjson_prefix(j.front()); + const CharType first_prefix = ubjson_prefix(j.front(), use_bjdata); const bool same_prefix = std::all_of(j.begin(), j.end(), - [this, first_prefix](const BasicJsonType & v) + [this, first_prefix, use_bjdata](const BasicJsonType & v) { - return ubjson_prefix(v) == first_prefix; + return ubjson_prefix(v, use_bjdata) == first_prefix; }); - if (same_prefix) + std::vector bjdx = {'[', '{', 'S', 'H', 'T', 'F', 'N', 'Z'}; // excluded markers in bjdata optimized type + + if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; oa->write_character(to_char_type('$')); @@ -14466,16 +14812,16 @@ class binary_writer if (use_count) { oa->write_character(to_char_type('#')); - write_number_with_ubjson_prefix(j.m_value.object->size(), true); + write_number_with_ubjson_prefix(j.m_value.object->size(), true, use_bjdata); } for (const auto& el : *j.m_value.object) { - write_number_with_ubjson_prefix(el.first.size(), true); + write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); oa->write_characters( reinterpret_cast(el.first.c_str()), el.first.size()); - write_ubjson(el.second, use_count, use_type, prefix_required); + write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata); } if (!use_count) @@ -14542,7 +14888,7 @@ class binary_writer const double value) { write_bson_entry_header(name, 0x01); - write_number(value); + write_number(value, true); } /*! @@ -14561,7 +14907,7 @@ class binary_writer { write_bson_entry_header(name, 0x02); - write_number(static_cast(value.size() + 1ul)); + write_number(static_cast(value.size() + 1ul), true); oa->write_characters( reinterpret_cast(value.c_str()), value.size() + 1); @@ -14594,12 +14940,12 @@ class binary_writer if ((std::numeric_limits::min)() <= value && value <= (std::numeric_limits::max)()) { write_bson_entry_header(name, 0x10); // int32 - write_number(static_cast(value)); + write_number(static_cast(value), true); } else { write_bson_entry_header(name, 0x12); // int64 - write_number(static_cast(value)); + write_number(static_cast(value), true); } } @@ -14622,12 +14968,12 @@ class binary_writer if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { write_bson_entry_header(name, 0x10 /* int32 */); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_value.number_unsigned), true); } else if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { write_bson_entry_header(name, 0x12 /* int64 */); - write_number(static_cast(j.m_value.number_unsigned)); + write_number(static_cast(j.m_value.number_unsigned), true); } else { @@ -14675,7 +15021,7 @@ class binary_writer const typename BasicJsonType::array_t& value) { write_bson_entry_header(name, 0x04); // array - write_number(static_cast(calc_bson_array_size(value))); + write_number(static_cast(calc_bson_array_size(value)), true); std::size_t array_index = 0ul; @@ -14695,7 +15041,7 @@ class binary_writer { write_bson_entry_header(name, 0x05); - write_number(static_cast(value.size())); + write_number(static_cast(value.size()), true); write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); oa->write_characters(reinterpret_cast(value.data()), value.size()); @@ -14817,7 +15163,7 @@ class binary_writer */ void write_bson_object(const typename BasicJsonType::object_t& value) { - write_number(static_cast(calc_bson_object_size(value))); + write_number(static_cast(calc_bson_object_size(value)), true); for (const auto& el : value) { @@ -14863,20 +15209,22 @@ class binary_writer template::value, int>::type = 0> void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) + const bool add_prefix, + const bool use_bjdata) { if (add_prefix) { oa->write_character(get_ubjson_float_prefix(n)); } - write_number(n); + write_number(n, use_bjdata); } // UBJSON: write number (unsigned integer) template::value, int>::type = 0> void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) + const bool add_prefix, + const bool use_bjdata) { if (n <= static_cast((std::numeric_limits::max)())) { @@ -14884,7 +15232,7 @@ class binary_writer { oa->write_character(to_char_type('i')); // int8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if (n <= (std::numeric_limits::max)()) { @@ -14892,7 +15240,7 @@ class binary_writer { oa->write_character(to_char_type('U')); // uint8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if (n <= static_cast((std::numeric_limits::max)())) { @@ -14900,7 +15248,15 @@ class binary_writer { oa->write_character(to_char_type('I')); // int16 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('u')); // uint16 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if (n <= static_cast((std::numeric_limits::max)())) { @@ -14908,7 +15264,15 @@ class binary_writer { oa->write_character(to_char_type('l')); // int32 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && n <= static_cast((std::numeric_limits::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('m')); // uint32 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if (n <= static_cast((std::numeric_limits::max)())) { @@ -14916,7 +15280,15 @@ class binary_writer { oa->write_character(to_char_type('L')); // int64 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && n <= (std::numeric_limits::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('M')); // uint64 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else { @@ -14926,7 +15298,7 @@ class binary_writer } const auto number = BasicJsonType(n).dump(); - write_number_with_ubjson_prefix(number.size(), true); + write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { oa->write_character(to_char_type(static_cast(number[i]))); @@ -14939,7 +15311,8 @@ class binary_writer std::is_signed::value&& !std::is_floating_point::value, int >::type = 0 > void write_number_with_ubjson_prefix(const NumberType n, - const bool add_prefix) + const bool add_prefix, + const bool use_bjdata) { if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -14947,7 +15320,7 @@ class binary_writer { oa->write_character(to_char_type('i')); // int8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)())) { @@ -14955,7 +15328,7 @@ class binary_writer { oa->write_character(to_char_type('U')); // uint8 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -14963,7 +15336,15 @@ class binary_writer { oa->write_character(to_char_type('I')); // int16 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)()))) + { + if (add_prefix) + { + oa->write_character(to_char_type('u')); // uint16 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -14971,7 +15352,15 @@ class binary_writer { oa->write_character(to_char_type('l')); // int32 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); + } + else if (use_bjdata && (static_cast((std::numeric_limits::min)()) <= n && n <= static_cast((std::numeric_limits::max)()))) + { + if (add_prefix) + { + oa->write_character(to_char_type('m')); // uint32 - bjdata only + } + write_number(static_cast(n), use_bjdata); } else if ((std::numeric_limits::min)() <= n && n <= (std::numeric_limits::max)()) { @@ -14979,7 +15368,7 @@ class binary_writer { oa->write_character(to_char_type('L')); // int64 } - write_number(static_cast(n)); + write_number(static_cast(n), use_bjdata); } // LCOV_EXCL_START else @@ -14990,7 +15379,7 @@ class binary_writer } const auto number = BasicJsonType(n).dump(); - write_number_with_ubjson_prefix(number.size(), true); + write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { oa->write_character(to_char_type(static_cast(number[i]))); @@ -15002,7 +15391,7 @@ class binary_writer /*! @brief determine the type prefix of container values */ - CharType ubjson_prefix(const BasicJsonType& j) const noexcept + CharType ubjson_prefix(const BasicJsonType& j, const bool use_bjdata) const noexcept { switch (j.type()) { @@ -15026,10 +15415,18 @@ class binary_writer { return 'I'; } + if (use_bjdata && ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)())) + { + return 'u'; + } if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'l'; } + if (use_bjdata && ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)())) + { + return 'm'; + } if ((std::numeric_limits::min)() <= j.m_value.number_integer && j.m_value.number_integer <= (std::numeric_limits::max)()) { return 'L'; @@ -15052,14 +15449,26 @@ class binary_writer { return 'I'; } + if (use_bjdata && j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'u'; + } if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'l'; } + if (use_bjdata && j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) + { + return 'm'; + } if (j.m_value.number_unsigned <= static_cast((std::numeric_limits::max)())) { return 'L'; } + if (use_bjdata && j.m_value.number_unsigned <= (std::numeric_limits::max)()) + { + return 'M'; + } // anything else is treated as high-precision number return 'H'; // LCOV_EXCL_LINE } @@ -15093,6 +15502,118 @@ class binary_writer return 'D'; // float 64 } + /*! + @return false if the object is successfully converted to a bjdata ndarray, true if the type or size is invalid + */ + bool write_bjdata_ndarray(const typename BasicJsonType::object_t& value, const bool use_count, const bool use_type) + { + std::map bjdtype = {{"uint8", 'U'}, {"int8", 'i'}, {"uint16", 'u'}, {"int16", 'I'}, + {"uint32", 'm'}, {"int32", 'l'}, {"uint64", 'M'}, {"int64", 'L'}, {"single", 'd'}, {"double", 'D'}, {"char", 'C'} + }; + + string_t key = "_ArrayType_"; + auto it = bjdtype.find(static_cast(value.at(key))); + if (it == bjdtype.end()) + { + return true; + } + CharType dtype = it->second; + + key = "_ArraySize_"; + std::size_t len = (value.at(key).empty() ? 0 : 1); + for (const auto& el : value.at(key)) + { + len *= static_cast(el.m_value.number_unsigned); + } + + key = "_ArrayData_"; + if (value.at(key).size() != len) + { + return true; + } + + oa->write_character('['); + oa->write_character('$'); + oa->write_character(dtype); + oa->write_character('#'); + + key = "_ArraySize_"; + write_ubjson(value.at(key), use_count, use_type, true, true); + + key = "_ArrayData_"; + if (dtype == 'U' || dtype == 'C') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'i') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'u') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'I') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'm') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'l') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'M') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_unsigned), true); + } + } + else if (dtype == 'L') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_integer), true); + } + } + else if (dtype == 'd') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_float), true); + } + } + else if (dtype == 'D') + { + for (const auto& el : value.at(key)) + { + write_number(static_cast(el.m_value.number_float), true); + } + } + return false; + } + /////////////////////// // Utility functions // /////////////////////// @@ -15100,16 +15621,18 @@ class binary_writer /* @brief write a number to output input @param[in] n number of type @a NumberType - @tparam NumberType the type of the number - @tparam OutputIsLittleEndian Set to true if output data is + @param[in] OutputIsLittleEndian Set to true if output data is required to be little endian + @tparam NumberType the type of the number @note This function needs to respect the system's endianness, because bytes in CBOR, MessagePack, and UBJSON are stored in network order (big endian) and therefore need reordering on little endian systems. + On the other hand, BSON and BJData use little endian and should reorder + on big endian systems. */ - template - void write_number(const NumberType n) + template + void write_number(const NumberType n, const bool OutputIsLittleEndian = false) { // step 1: write number to array of length NumberType std::array vec{}; @@ -21221,7 +21744,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::forward(i)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); } /// @brief generate SAX events @@ -21236,7 +21759,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec auto ia = detail::input_adapter(std::move(first), std::move(last)); return format == input_format_t::json ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) - : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); } /// @brief generate SAX events @@ -21257,7 +21780,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); + : detail::binary_reader(std::move(ia), format).sax_parse(format, sax, strict); } #ifndef JSON_NO_IO /// @brief deserialize from stream @@ -21413,6 +21936,33 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec binary_writer(o).write_ubjson(j, use_size, use_type); } + /// @brief create a BJData serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ + static std::vector to_bjdata(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector result; + to_bjdata(j, result, use_size, use_type); + return result; + } + + /// @brief create a BJData serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ + static void to_bjdata(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + } + + /// @brief create a BJData serialization of a given JSON value + /// @sa https://json.nlohmann.me/api/basic_json/to_bjdata/ + static void to_bjdata(const basic_json& j, detail::output_adapter o, + const bool use_size = false, const bool use_type = false) + { + binary_writer(o).write_ubjson(j, use_size, use_type, true, true); + } + /// @brief create a BSON serialization of a given JSON value /// @sa https://json.nlohmann.me/api/basic_json/to_bson/ static std::vector to_bson(const basic_json& j) @@ -21448,7 +21998,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); return res ? result : basic_json(value_t::discarded); } @@ -21464,7 +22014,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); return res ? result : basic_json(value_t::discarded); } @@ -21491,7 +22041,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); + const bool res = binary_reader(std::move(ia), input_format_t::cbor).sax_parse(input_format_t::cbor, &sdp, strict, tag_handler); return res ? result : basic_json(value_t::discarded); } @@ -21506,7 +22056,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -21521,7 +22071,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -21545,7 +22095,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::msgpack, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::msgpack).sax_parse(input_format_t::msgpack, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -21560,7 +22110,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -21575,7 +22125,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -21599,10 +22149,64 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::ubjson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::ubjson).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + + /// @brief create a JSON value from an input in BJData format + /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(InputType&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::forward(i)); + const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /// @brief create a JSON value from an input in BJData format + /// @sa https://json.nlohmann.me/api/basic_json/from_bjdata/ + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(IteratorType first, IteratorType last, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = detail::input_adapter(std::move(first), std::move(last)); + const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + template + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(const T* ptr, std::size_t len, + const bool strict = true, + const bool allow_exceptions = true) + { + return from_bjdata(ptr, ptr + len, strict, allow_exceptions); + } + + JSON_HEDLEY_WARN_UNUSED_RESULT + static basic_json from_bjdata(detail::span_input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser sdp(result, allow_exceptions); + auto ia = i.get(); + // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) + const bool res = binary_reader(std::move(ia), input_format_t::bjdata).sax_parse(input_format_t::bjdata, &sdp, strict); return res ? result : basic_json(value_t::discarded); } + /// @brief create a JSON value from an input in BSON format /// @sa https://json.nlohmann.me/api/basic_json/from_bson/ template @@ -21614,7 +22218,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::forward(i)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -21629,7 +22233,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec basic_json result; detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = detail::input_adapter(std::move(first), std::move(last)); - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } @@ -21653,7 +22257,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec detail::json_sax_dom_parser sdp(result, allow_exceptions); auto ia = i.get(); // NOLINTNEXTLINE(hicpp-move-const-arg,performance-move-const-arg) - const bool res = binary_reader(std::move(ia)).sax_parse(input_format_t::bson, &sdp, strict); + const bool res = binary_reader(std::move(ia), input_format_t::bson).sax_parse(input_format_t::bson, &sdp, strict); return res ? result : basic_json(value_t::discarded); } /// @} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2f06def50..1d5c78697 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -72,7 +72,7 @@ endif() if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") # avoid stack overflow, see https://github.com/nlohmann/json/issues/2955 - json_test_set_test_options("test-cbor;test-msgpack;test-ubjson" LINK_OPTIONS /STACK:4000000) + json_test_set_test_options("test-cbor;test-msgpack;test-ubjson;test-bjdata" LINK_OPTIONS /STACK:4000000) endif() # disable exceptions for test-disabled_exceptions diff --git a/test/Makefile b/test/Makefile index 2d08a8f7b..3a11ce7dd 100644 --- a/test/Makefile +++ b/test/Makefile @@ -10,7 +10,7 @@ CXXFLAGS += -std=c++11 CPPFLAGS += -I ../single_include FUZZER_ENGINE = src/fuzzer-driver_afl.cpp -FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer +FUZZERS = parse_afl_fuzzer parse_bson_fuzzer parse_cbor_fuzzer parse_msgpack_fuzzer parse_ubjson_fuzzer parse_bjdata_fuzzer fuzzers: $(FUZZERS) parse_afl_fuzzer: @@ -27,3 +27,6 @@ parse_msgpack_fuzzer: parse_ubjson_fuzzer: $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_ubjson.cpp -o $@ + +parse_bjdata_fuzzer: + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUZZER_ENGINE) src/fuzzer-parse_bjdata.cpp -o $@ diff --git a/test/src/fuzzer-parse_bjdata.cpp b/test/src/fuzzer-parse_bjdata.cpp new file mode 100644 index 000000000..99e799bd3 --- /dev/null +++ b/test/src/fuzzer-parse_bjdata.cpp @@ -0,0 +1,84 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ (fuzz test support) +| | |__ | | | | | | version 3.10.5 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +This file implements a parser test suitable for fuzz testing. Given a byte +array data, it performs the following steps: + +- j1 = from_bjdata(data) +- vec = to_bjdata(j1) +- j2 = from_bjdata(vec) +- assert(j1 == j2) +- vec2 = to_bjdata(j1, use_size = true, use_type = false) +- j3 = from_bjdata(vec2) +- assert(j1 == j3) +- vec3 = to_bjdata(j1, use_size = true, use_type = true) +- j4 = from_bjdata(vec3) +- assert(j1 == j4) + +The provided function `LLVMFuzzerTestOneInput` can be used in different fuzzer +drivers. + +Licensed under the MIT License . +*/ + +#include +#include +#include + +using json = nlohmann::json; + +// see http://llvm.org/docs/LibFuzzer.html +extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) +{ + try + { + // step 1: parse input + std::vector vec1(data, data + size); + json j1 = json::from_bjdata(vec1); + + try + { + // step 2.1: round trip without adding size annotations to container types + std::vector vec2 = json::to_bjdata(j1, false, false); + + // step 2.2: round trip with adding size annotations but without adding type annonations to container types + std::vector vec3 = json::to_bjdata(j1, true, false); + + // step 2.3: round trip with adding size as well as type annotations to container types + std::vector vec4 = json::to_bjdata(j1, true, true); + + // parse serialization + json j2 = json::from_bjdata(vec2); + json j3 = json::from_bjdata(vec3); + json j4 = json::from_bjdata(vec4); + + // serializations must match + assert(json::to_bjdata(j2, false, false) == vec2); + assert(json::to_bjdata(j3, true, false) == vec3); + assert(json::to_bjdata(j4, true, true) == vec4); + } + catch (const json::parse_error&) + { + // parsing a BJData serialization must not fail + assert(false); + } + } + catch (const json::parse_error&) + { + // parse errors are ok, because input may be random bytes + } + catch (const json::type_error&) + { + // type errors can occur during parsing, too + } + catch (const json::out_of_range&) + { + // out of range errors may happen if provided sizes are excessive + } + + // return 0 - non-zero return values are reserved for future use + return 0; +} diff --git a/test/src/unit-bjdata.cpp b/test/src/unit-bjdata.cpp new file mode 100644 index 000000000..bc6c52833 --- /dev/null +++ b/test/src/unit-bjdata.cpp @@ -0,0 +1,3355 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ (test suite) +| | |__ | | | | | | version 3.10.5 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2022 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include +#include +#include "test_utils.hpp" + +namespace +{ +class SaxCountdown +{ + public: + explicit SaxCountdown(const int count) : events_left(count) + {} + + bool null() + { + return events_left-- > 0; + } + + bool boolean(bool /*unused*/) + { + return events_left-- > 0; + } + + bool number_integer(json::number_integer_t /*unused*/) + { + return events_left-- > 0; + } + + bool number_unsigned(json::number_unsigned_t /*unused*/) + { + return events_left-- > 0; + } + + bool number_float(json::number_float_t /*unused*/, const std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool string(std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool binary(std::vector& /*unused*/) + { + return events_left-- > 0; + } + + bool start_object(std::size_t /*unused*/) + { + return events_left-- > 0; + } + + bool key(std::string& /*unused*/) + { + return events_left-- > 0; + } + + bool end_object() + { + return events_left-- > 0; + } + + bool start_array(std::size_t /*unused*/) + { + return events_left-- > 0; + } + + bool end_array() + { + return events_left-- > 0; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const json::exception& /*unused*/) // NOLINT(readability-convert-member-functions-to-static) + { + return false; + } + + private: + int events_left = 0; +}; +} // namespace + +TEST_CASE("BJData") +{ + SECTION("individual values") + { + SECTION("discarded") + { + // discarded values are not serialized + json j = json::value_t::discarded; + const auto result = json::to_bjdata(j); + CHECK(result.empty()); + } + + SECTION("null") + { + json j = nullptr; + std::vector expected = {'Z'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("boolean") + { + SECTION("true") + { + json j = true; + std::vector expected = {'T'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("false") + { + json j = false; + std::vector expected = {'F'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("number") + { + SECTION("signed") + { + SECTION("-9223372036854775808..-2147483649 (int64)") + { + std::vector numbers; + numbers.push_back((std::numeric_limits::min)()); + numbers.push_back(-1000000000000000000LL); + numbers.push_back(-100000000000000000LL); + numbers.push_back(-10000000000000000LL); + numbers.push_back(-1000000000000000LL); + numbers.push_back(-100000000000000LL); + numbers.push_back(-10000000000000LL); + numbers.push_back(-1000000000000LL); + numbers.push_back(-100000000000LL); + numbers.push_back(-10000000000LL); + numbers.push_back(-2147483649LL); + for (auto i : numbers) + { + CAPTURE(i) + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('L')); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 24) & 0xff)); + expected.push_back(static_cast((i >> 32) & 0xff)); + expected.push_back(static_cast((i >> 40) & 0xff)); + expected.push_back(static_cast((i >> 48) & 0xff)); + expected.push_back(static_cast((i >> 56) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'L'); + int64_t restored = (static_cast(result[8]) << 070) + + (static_cast(result[7]) << 060) + + (static_cast(result[6]) << 050) + + (static_cast(result[5]) << 040) + + (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("-2147483648..-32769 (int32)") + { + std::vector numbers; + numbers.push_back(-32769); + numbers.push_back(-100000); + numbers.push_back(-1000000); + numbers.push_back(-10000000); + numbers.push_back(-100000000); + numbers.push_back(-1000000000); + numbers.push_back(-2147483647 - 1); // https://stackoverflow.com/a/29356002/266378 + for (auto i : numbers) + { + CAPTURE(i) + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('l')); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 24) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'l'); + int32_t restored = (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("-32768..-129 (int16)") + { + for (int32_t i = -32768; i <= -129; ++i) + { + CAPTURE(i) + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('I')); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + auto restored = static_cast(((result[2] << 8) + result[1])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("-9263 (int16)") + { + json j = -9263; + std::vector expected = {'I', 0xd1, 0xdb}; + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + auto restored = static_cast(((result[2] << 8) + result[1])); + CHECK(restored == -9263); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("-128..-1 (int8)") + { + for (auto i = -128; i <= -1; ++i) + { + CAPTURE(i) + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back('i'); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'i'); + CHECK(static_cast(result[1]) == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("0..127 (int8)") + { + for (size_t i = 0; i <= 127; ++i) + { + CAPTURE(i) + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('i')); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'i'); + CHECK(result[1] == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("128..255 (uint8)") + { + for (size_t i = 128; i <= 255; ++i) + { + CAPTURE(i) + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('U')); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'U'); + CHECK(result[1] == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("256..32767 (int16)") + { + for (size_t i = 256; i <= 32767; ++i) + { + CAPTURE(i) + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('I')); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + auto restored = static_cast(static_cast(result[2]) * 256 + static_cast(result[1])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("32768..65535 (uint16)") + { + for (uint32_t i : + { + 32768u, 55555u, 65535u + }) + { + CAPTURE(i) + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('u')); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'u'); + auto restored = static_cast(static_cast(result[2]) * 256 + static_cast(result[1])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("65536..2147483647 (int32)") + { + for (uint32_t i : + { + 65536u, 77777u, 2147483647u + }) + { + CAPTURE(i) + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back('l'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 24) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'l'); + uint32_t restored = (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("2147483648..4294967295 (uint32)") + { + for (uint32_t i : + { + 2147483648u, 3333333333u, 4294967295u + }) + { + CAPTURE(i) + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back('m'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 24) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'm'); + uint32_t restored = (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("4294967296..9223372036854775807 (int64)") + { + std::vector v = {4294967296LU, 9223372036854775807LU}; + for (uint64_t i : v) + { + CAPTURE(i) + + // create JSON value with integer number + json j = -1; + j.get_ref() = static_cast(i); + + // check type + CHECK(j.is_number_integer()); + + // create expected byte vector + std::vector expected; + expected.push_back('L'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 010) & 0xff)); + expected.push_back(static_cast((i >> 020) & 0xff)); + expected.push_back(static_cast((i >> 030) & 0xff)); + expected.push_back(static_cast((i >> 040) & 0xff)); + expected.push_back(static_cast((i >> 050) & 0xff)); + expected.push_back(static_cast((i >> 060) & 0xff)); + expected.push_back(static_cast((i >> 070) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'L'); + uint64_t restored = (static_cast(result[8]) << 070) + + (static_cast(result[7]) << 060) + + (static_cast(result[6]) << 050) + + (static_cast(result[5]) << 040) + + (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("9223372036854775808..18446744073709551615 (uint64)") + { + std::vector v = {9223372036854775808ull, 18446744073709551615ull}; + for (uint64_t i : v) + { + CAPTURE(i) + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('M'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 010) & 0xff)); + expected.push_back(static_cast((i >> 020) & 0xff)); + expected.push_back(static_cast((i >> 030) & 0xff)); + expected.push_back(static_cast((i >> 040) & 0xff)); + expected.push_back(static_cast((i >> 050) & 0xff)); + expected.push_back(static_cast((i >> 060) & 0xff)); + expected.push_back(static_cast((i >> 070) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'M'); + uint64_t restored = (static_cast(result[8]) << 070) + + (static_cast(result[7]) << 060) + + (static_cast(result[6]) << 050) + + (static_cast(result[5]) << 040) + + (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + } + + SECTION("unsigned") + { + SECTION("0..127 (int8)") + { + for (size_t i = 0; i <= 127; ++i) + { + CAPTURE(i) + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('i'); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'i'); + auto restored = static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("128..255 (uint8)") + { + for (size_t i = 128; i <= 255; ++i) + { + CAPTURE(i) + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('U'); + expected.push_back(static_cast(i)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 2); + + // check individual bytes + CHECK(result[0] == 'U'); + auto restored = static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("256..32767 (int16)") + { + for (size_t i = 256; i <= 32767; ++i) + { + CAPTURE(i) + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('I'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'I'); + auto restored = static_cast(static_cast(result[2]) * 256 + static_cast(result[1])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("32768..65535 (uint16)") + { + for (uint32_t i : + { + 32768u, 55555u, 65535u + }) + { + CAPTURE(i) + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('u'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 3); + + // check individual bytes + CHECK(result[0] == 'u'); + auto restored = static_cast(static_cast(result[2]) * 256 + static_cast(result[1])); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + SECTION("65536..2147483647 (int32)") + { + for (uint32_t i : + { + 65536u, 77777u, 2147483647u + }) + { + CAPTURE(i) + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('l'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 24) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'l'); + uint32_t restored = (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("2147483648..4294967295 (uint32)") + { + for (uint32_t i : + { + 2147483648u, 3333333333u, 4294967295u + }) + { + CAPTURE(i) + + // create JSON value with unsigned integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('m'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 8) & 0xff)); + expected.push_back(static_cast((i >> 16) & 0xff)); + expected.push_back(static_cast((i >> 24) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 5); + + // check individual bytes + CHECK(result[0] == 'm'); + uint32_t restored = (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("4294967296..9223372036854775807 (int64)") + { + std::vector v = {4294967296ul, 9223372036854775807ul}; + for (uint64_t i : v) + { + CAPTURE(i) + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('L'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 010) & 0xff)); + expected.push_back(static_cast((i >> 020) & 0xff)); + expected.push_back(static_cast((i >> 030) & 0xff)); + expected.push_back(static_cast((i >> 040) & 0xff)); + expected.push_back(static_cast((i >> 050) & 0xff)); + expected.push_back(static_cast((i >> 060) & 0xff)); + expected.push_back(static_cast((i >> 070) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'L'); + uint64_t restored = (static_cast(result[8]) << 070) + + (static_cast(result[7]) << 060) + + (static_cast(result[6]) << 050) + + (static_cast(result[5]) << 040) + + (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("9223372036854775808..18446744073709551615 (uint64)") + { + std::vector v = {9223372036854775808ull, 18446744073709551615ull}; + for (uint64_t i : v) + { + CAPTURE(i) + + // create JSON value with integer number + json j = i; + + // check type + CHECK(j.is_number_unsigned()); + + // create expected byte vector + std::vector expected; + expected.push_back('M'); + expected.push_back(static_cast(i & 0xff)); + expected.push_back(static_cast((i >> 010) & 0xff)); + expected.push_back(static_cast((i >> 020) & 0xff)); + expected.push_back(static_cast((i >> 030) & 0xff)); + expected.push_back(static_cast((i >> 040) & 0xff)); + expected.push_back(static_cast((i >> 050) & 0xff)); + expected.push_back(static_cast((i >> 060) & 0xff)); + expected.push_back(static_cast((i >> 070) & 0xff)); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == 9); + + // check individual bytes + CHECK(result[0] == 'M'); + uint64_t restored = (static_cast(result[8]) << 070) + + (static_cast(result[7]) << 060) + + (static_cast(result[6]) << 050) + + (static_cast(result[5]) << 040) + + (static_cast(result[4]) << 030) + + (static_cast(result[3]) << 020) + + (static_cast(result[2]) << 010) + + static_cast(result[1]); + CHECK(restored == i); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + } + SECTION("float64") + { + SECTION("3.1415925") + { + double v = 3.1415925; + json j = v; + std::vector expected = + { + 'D', 0xfc, 0xde, 0xa6, 0x3f, 0xfb, 0x21, 0x09, 0x40 + }; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result) == v); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("half-precision float") + { + SECTION("simple half floats") + { + CHECK(json::parse("0.0") == json::from_bjdata(std::vector({'h', 0x00, 0x00}))); + CHECK(json::parse("-0.0") == json::from_bjdata(std::vector({'h', 0x00, 0x80}))); + CHECK(json::parse("1.0") == json::from_bjdata(std::vector({'h', 0x00, 0x3c}))); + CHECK(json::parse("1.5") == json::from_bjdata(std::vector({'h', 0x00, 0x3e}))); + CHECK(json::parse("65504.0") == json::from_bjdata(std::vector({'h', 0xff, 0x7b}))); + } + + SECTION("errors") + { + SECTION("no byte follows") + { + json _; + std::vector vec0 = {'h'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vec0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vec0), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vec0, true, false).is_discarded()); + } + + SECTION("only one byte follows") + { + json _; + std::vector vec1 = {'h', 0x00}; + CHECK_THROWS_AS(_ = json::from_bjdata(vec1), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vec1), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vec1, true, false).is_discarded()); + } + } + } + + SECTION("half-precision float (edge cases)") + { + SECTION("exp = 0b00000") + { + SECTION("0 (0 00000 0000000000)") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0x00})); + json::number_float_t d{j}; + CHECK(d == 0.0); + } + + SECTION("-0 (1 00000 0000000000)") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0x80})); + json::number_float_t d{j}; + CHECK(d == -0.0); + } + + SECTION("2**-24 (0 00000 0000000001)") + { + json j = json::from_bjdata(std::vector({'h', 0x01, 0x00})); + json::number_float_t d{j}; + CHECK(d == std::pow(2.0, -24.0)); + } + } + + SECTION("exp = 0b11111") + { + SECTION("infinity (0 11111 0000000000)") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0x7c})); + json::number_float_t d{j}; + CHECK(d == std::numeric_limits::infinity()); + CHECK(j.dump() == "null"); + } + + SECTION("-infinity (1 11111 0000000000)") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0xfc})); + json::number_float_t d{j}; + CHECK(d == -std::numeric_limits::infinity()); + CHECK(j.dump() == "null"); + } + } + + SECTION("other values from https://en.wikipedia.org/wiki/Half-precision_floating-point_format") + { + SECTION("1 (0 01111 0000000000)") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0x3c})); + json::number_float_t d{j}; + CHECK(d == 1); + } + + SECTION("-2 (1 10000 0000000000)") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0xc0})); + json::number_float_t d{j}; + CHECK(d == -2); + } + + SECTION("65504 (0 11110 1111111111)") + { + json j = json::from_bjdata(std::vector({'h', 0xff, 0x7b})); + json::number_float_t d{j}; + CHECK(d == 65504); + } + } + + SECTION("infinity") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0x7c})); + json::number_float_t d{j}; + CHECK(!std::isfinite(d)); + CHECK(j.dump() == "null"); + } + + SECTION("NaN") + { + json j = json::from_bjdata(std::vector({'h', 0x00, 0x7e })); + json::number_float_t d{j}; + CHECK(std::isnan(d)); + CHECK(j.dump() == "null"); + } + } + + SECTION("high-precision number") + { + SECTION("unsigned integer number") + { + std::vector vec = {'H', 'i', 0x14, '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0'}; + const auto j = json::from_bjdata(vec); + CHECK(j.is_number_unsigned()); + CHECK(j.dump() == "12345678901234567890"); + } + + SECTION("signed integer number") + { + std::vector vec = {'H', 'i', 0x13, '-', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0', '1', '2', '3', '4', '5', '6', '7', '8'}; + const auto j = json::from_bjdata(vec); + CHECK(j.is_number_integer()); + CHECK(j.dump() == "-123456789012345678"); + } + + SECTION("floating-point number") + { + std::vector vec = {'H', 'i', 0x16, '3', '.', '1', '4', '1', '5', '9', '2', '6', '5', '3', '5', '8', '9', '7', '9', '3', '2', '3', '8', '4', '6'}; + const auto j = json::from_bjdata(vec); + CHECK(j.is_number_float()); + CHECK(j.dump() == "3.141592653589793"); + } + + SECTION("errors") + { + // error while parsing length + std::vector vec0 = {'H', 'i'}; + CHECK(json::from_bjdata(vec0, true, false).is_discarded()); + // error while parsing string + std::vector vec1 = {'H', 'i', '1'}; + CHECK(json::from_bjdata(vec1, true, false).is_discarded()); + + json _; + std::vector vec2 = {'H', 'i', 2, '1', 'A', '3'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec2), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing BJData high-precision number: invalid number text: 1A", json::parse_error); + std::vector vec3 = {'H', 'i', 2, '1', '.'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec3), "[json.exception.parse_error.115] parse error at byte 5: syntax error while parsing BJData high-precision number: invalid number text: 1.", json::parse_error); + std::vector vec4 = {'H', 2, '1', '0'}; + CHECK_THROWS_WITH_AS(_ = json::from_bjdata(vec4), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x02", json::parse_error); + } + } + } + + SECTION("string") + { + SECTION("N = 0..127") + { + for (size_t N = 0; N <= 127; ++N) + { + CAPTURE(N) + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector + std::vector expected; + expected.push_back('S'); + expected.push_back('i'); + expected.push_back(static_cast(N)); + for (size_t i = 0; i < N; ++i) + { + expected.push_back('x'); + } + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == N + 3); + // check that no null byte is appended + if (N > 0) + { + CHECK(result.back() != '\x00'); + } + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("N = 128..255") + { + for (size_t N = 128; N <= 255; ++N) + { + CAPTURE(N) + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector + std::vector expected; + expected.push_back('S'); + expected.push_back('U'); + expected.push_back(static_cast(N)); + for (size_t i = 0; i < N; ++i) + { + expected.push_back('x'); + } + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == N + 3); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("N = 256..32767") + { + for (size_t N : + { + 256u, 999u, 1025u, 3333u, 2048u, 32767u + }) + { + CAPTURE(N) + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector (hack: create string first) + std::vector expected(N, 'x'); + // reverse order of commands, because we insert at begin() + expected.insert(expected.begin(), static_cast((N >> 8) & 0xff)); + expected.insert(expected.begin(), static_cast(N & 0xff)); + expected.insert(expected.begin(), 'I'); + expected.insert(expected.begin(), 'S'); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == N + 4); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("N = 32768..65535") + { + for (size_t N : + { + 32768u, 55555u, 65535u + }) + { + CAPTURE(N) + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector (hack: create string first) + std::vector expected(N, 'x'); + // reverse order of commands, because we insert at begin() + expected.insert(expected.begin(), static_cast((N >> 8) & 0xff)); + expected.insert(expected.begin(), static_cast(N & 0xff)); + expected.insert(expected.begin(), 'u'); + expected.insert(expected.begin(), 'S'); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == N + 4); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("N = 65536..2147483647") + { + for (size_t N : + { + 65536u, 77777u, 1048576u + }) + { + CAPTURE(N) + + // create JSON value with string containing of N * 'x' + const auto s = std::string(N, 'x'); + json j = s; + + // create expected byte vector (hack: create string first) + std::vector expected(N, 'x'); + // reverse order of commands, because we insert at begin() + expected.insert(expected.begin(), static_cast((N >> 24) & 0xff)); + expected.insert(expected.begin(), static_cast((N >> 16) & 0xff)); + expected.insert(expected.begin(), static_cast((N >> 8) & 0xff)); + expected.insert(expected.begin(), static_cast(N & 0xff)); + expected.insert(expected.begin(), 'l'); + expected.insert(expected.begin(), 'S'); + + // compare result + size + const auto result = json::to_bjdata(j); + CHECK(result == expected); + CHECK(result.size() == N + 6); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + } + + + SECTION("binary") + { + SECTION("N = 0..127") + { + for (std::size_t N = 0; N <= 127; ++N) + { + CAPTURE(N) + + // create JSON value with byte array containing of N * 'x' + const auto s = std::vector(N, 'x'); + json j = json::binary(s); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('[')); + if (N != 0) + { + expected.push_back(static_cast('$')); + expected.push_back(static_cast('U')); + } + expected.push_back(static_cast('#')); + expected.push_back(static_cast('i')); + expected.push_back(static_cast(N)); + for (size_t i = 0; i < N; ++i) + { + expected.push_back(0x78); + } + + // compare result + size + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + if (N == 0) + { + CHECK(result.size() == N + 4); + } + else + { + CHECK(result.size() == N + 6); + } + + // check that no null byte is appended + if (N > 0) + { + CHECK(result.back() != '\x00'); + } + + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } + } + + SECTION("N = 128..255") + { + for (std::size_t N = 128; N <= 255; ++N) + { + CAPTURE(N) + + // create JSON value with byte array containing of N * 'x' + const auto s = std::vector(N, 'x'); + json j = json::binary(s); + + // create expected byte vector + std::vector expected; + expected.push_back(static_cast('[')); + expected.push_back(static_cast('$')); + expected.push_back(static_cast('U')); + expected.push_back(static_cast('#')); + expected.push_back(static_cast('U')); + expected.push_back(static_cast(N)); + for (size_t i = 0; i < N; ++i) + { + expected.push_back(0x78); + } + + // compare result + size + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + CHECK(result.size() == N + 6); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } + } + + SECTION("N = 256..32767") + { + for (std::size_t N : + { + 256u, 999u, 1025u, 3333u, 2048u, 32767u + }) + { + CAPTURE(N) + + // create JSON value with byte array containing of N * 'x' + const auto s = std::vector(N, 'x'); + json j = json::binary(s); + + // create expected byte vector + std::vector expected(N + 7, 'x'); + expected[0] = '['; + expected[1] = '$'; + expected[2] = 'U'; + expected[3] = '#'; + expected[4] = 'I'; + expected[5] = static_cast(N & 0xFF); + expected[6] = static_cast((N >> 8) & 0xFF); + + // compare result + size + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + CHECK(result.size() == N + 7); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } + } + + SECTION("N = 32768..65535") + { + for (std::size_t N : + { + 32768u, 55555u, 65535u + }) + { + CAPTURE(N) + + // create JSON value with byte array containing of N * 'x' + const auto s = std::vector(N, 'x'); + json j = json::binary(s); + + // create expected byte vector + std::vector expected(N + 7, 'x'); + expected[0] = '['; + expected[1] = '$'; + expected[2] = 'U'; + expected[3] = '#'; + expected[4] = 'u'; + expected[5] = static_cast(N & 0xFF); + expected[6] = static_cast((N >> 8) & 0xFF); + + // compare result + size + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + CHECK(result.size() == N + 7); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } + } + + SECTION("N = 65536..2147483647") + { + for (std::size_t N : + { + 65536u, 77777u, 1048576u + }) + { + CAPTURE(N) + + // create JSON value with byte array containing of N * 'x' + const auto s = std::vector(N, 'x'); + json j = json::binary(s); + + // create expected byte vector + std::vector expected(N + 9, 'x'); + expected[0] = '['; + expected[1] = '$'; + expected[2] = 'U'; + expected[3] = '#'; + expected[4] = 'l'; + expected[5] = static_cast(N & 0xFF); + expected[6] = static_cast((N >> 8) & 0xFF); + expected[7] = static_cast((N >> 16) & 0xFF); + expected[8] = static_cast((N >> 24) & 0xFF); + + // compare result + size + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + CHECK(result.size() == N + 9); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } + } + + SECTION("Other Serializations") + { + const std::size_t N = 10; + const auto s = std::vector(N, 'x'); + json j = json::binary(s); + + SECTION("No Count No Type") + { + std::vector expected; + expected.push_back(static_cast('[')); + for (std::size_t i = 0; i < N; ++i) + { + expected.push_back(static_cast('U')); + expected.push_back(static_cast(0x78)); + } + expected.push_back(static_cast(']')); + + // compare result + size + const auto result = json::to_bjdata(j, false, false); + CHECK(result == expected); + CHECK(result.size() == N + 12); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } + + SECTION("Yes Count No Type") + { + std::vector expected; + expected.push_back(static_cast('[')); + expected.push_back(static_cast('#')); + expected.push_back(static_cast('i')); + expected.push_back(static_cast(N)); + + for (size_t i = 0; i < N; ++i) + { + expected.push_back(static_cast('U')); + expected.push_back(static_cast(0x78)); + } + + // compare result + size + const auto result = json::to_bjdata(j, true, false); + CHECK(result == expected); + CHECK(result.size() == N + 14); + // check that no null byte is appended + CHECK(result.back() != '\x00'); + + // roundtrip only works to an array of numbers + json j_out = s; + CHECK(json::from_bjdata(result) == j_out); + CHECK(json::from_bjdata(result, true, false) == j_out); + } + } + } + SECTION("array") + { + SECTION("empty") + { + SECTION("size=false type=false") + { + json j = json::array(); + std::vector expected = {'[', ']'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j = json::array(); + std::vector expected = {'[', '#', 'i', 0}; + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=true") + { + json j = json::array(); + std::vector expected = {'[', '#', 'i', 0}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("[null]") + { + SECTION("size=false type=false") + { + json j = {nullptr}; + std::vector expected = {'[', 'Z', ']'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j = {nullptr}; + std::vector expected = {'[', '#', 'i', 1, 'Z'}; + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=true") + { + json j = {nullptr}; + std::vector expected = {'[', '#', 'i', 1, 'Z'}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("[1,2,3,4,5]") + { + SECTION("size=false type=false") + { + json j = json::parse("[1,2,3,4,5]"); + std::vector expected = {'[', 'i', 1, 'i', 2, 'i', 3, 'i', 4, 'i', 5, ']'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j = json::parse("[1,2,3,4,5]"); + std::vector expected = {'[', '#', 'i', 5, 'i', 1, 'i', 2, 'i', 3, 'i', 4, 'i', 5}; + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=true") + { + json j = json::parse("[1,2,3,4,5]"); + std::vector expected = {'[', '$', 'i', '#', 'i', 5, 1, 2, 3, 4, 5}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("[[[[]]]]") + { + SECTION("size=false type=false") + { + json j = json::parse("[[[[]]]]"); + std::vector expected = {'[', '[', '[', '[', ']', ']', ']', ']'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j = json::parse("[[[[]]]]"); + std::vector expected = {'[', '#', 'i', 1, '[', '#', 'i', 1, '[', '#', 'i', 1, '[', '#', 'i', 0}; + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=true") + { + json j = json::parse("[[[[]]]]"); + std::vector expected = {'[', '#', 'i', 1, '[', '#', 'i', 1, '[', '#', 'i', 1, '[', '#', 'i', 0}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("array with int16_t elements") + { + SECTION("size=false type=false") + { + json j(257, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[258] = ']'; // closing array + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j(257, nullptr); + std::vector expected(j.size() + 5, 'Z'); // all null + expected[0] = '['; // opening array + expected[1] = '#'; // array size + expected[2] = 'I'; // int16 + expected[3] = 0x01; // 0x0101, first byte + expected[4] = 0x01; // 0x0101, second byte + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("array with uint16_t elements") + { + SECTION("size=false type=false") + { + json j(32768, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[32769] = ']'; // closing array + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j(32768, nullptr); + std::vector expected(j.size() + 5, 'Z'); // all null + expected[0] = '['; // opening array + expected[1] = '#'; // array size + expected[2] = 'u'; // int16 + expected[3] = 0x00; // 0x0101, first byte + expected[4] = 0x80; // 0x0101, second byte + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("array with int32_t elements") + { + SECTION("size=false type=false") + { + json j(65793, nullptr); + std::vector expected(j.size() + 2, 'Z'); // all null + expected[0] = '['; // opening array + expected[65794] = ']'; // closing array + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j(65793, nullptr); + std::vector expected(j.size() + 7, 'Z'); // all null + expected[0] = '['; // opening array + expected[1] = '#'; // array size + expected[2] = 'l'; // int32 + expected[3] = 0x01; // 0x00010101, fourth byte + expected[4] = 0x01; // 0x00010101, third byte + expected[5] = 0x01; // 0x00010101, second byte + expected[6] = 0x00; // 0x00010101, first byte + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + } + + SECTION("object") + { + SECTION("empty") + { + SECTION("size=false type=false") + { + json j = json::object(); + std::vector expected = {'{', '}'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j = json::object(); + std::vector expected = {'{', '#', 'i', 0}; + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=true") + { + json j = json::object(); + std::vector expected = {'{', '#', 'i', 0}; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("{\"\":null}") + { + SECTION("size=false type=false") + { + json j = {{"", nullptr}}; + std::vector expected = {'{', 'i', 0, 'Z', '}'}; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j = {{"", nullptr}}; + std::vector expected = {'{', '#', 'i', 1, 'i', 0, 'Z'}; + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + + SECTION("{\"a\": {\"b\": {\"c\": {}}}}") + { + SECTION("size=false type=false") + { + json j = json::parse(R"({"a": {"b": {"c": {}}}})"); + std::vector expected = + { + '{', 'i', 1, 'a', '{', 'i', 1, 'b', '{', 'i', 1, 'c', '{', '}', '}', '}', '}' + }; + const auto result = json::to_bjdata(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=false") + { + json j = json::parse(R"({"a": {"b": {"c": {}}}})"); + std::vector expected = + { + '{', '#', 'i', 1, 'i', 1, 'a', '{', '#', 'i', 1, 'i', 1, 'b', '{', '#', 'i', 1, 'i', 1, 'c', '{', '#', 'i', 0 + }; + const auto result = json::to_bjdata(j, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + + SECTION("size=true type=true ignore object type marker") + { + json j = json::parse(R"({"a": {"b": {"c": {}}}})"); + std::vector expected = + { + '{', '#', 'i', 1, 'i', 1, 'a', '{', '#', 'i', 1, 'i', 1, 'b', '{', '#', 'i', 1, 'i', 1, 'c', '{', '#', 'i', 0 + }; + const auto result = json::to_bjdata(j, true, true); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_bjdata(result) == j); + CHECK(json::from_bjdata(result, true, false) == j); + } + } + } + } + + SECTION("errors") + { + SECTION("strict mode") + { + std::vector vec = {'Z', 'Z'}; + SECTION("non-strict mode") + { + const auto result = json::from_bjdata(vec, false); + CHECK(result == json()); + } + + SECTION("strict mode") + { + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(vec), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vec), + "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BJData value: expected end of input; last byte: 0x5A"); + } + } + } + + SECTION("SAX aborts") + { + SECTION("start_array()") + { + std::vector v = {'[', 'T', 'F', ']'}; + SaxCountdown scp(0); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("start_object()") + { + std::vector v = {'{', 'i', 3, 'f', 'o', 'o', 'F', '}'}; + SaxCountdown scp(0); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("key() in object") + { + std::vector v = {'{', 'i', 3, 'f', 'o', 'o', 'F', '}'}; + SaxCountdown scp(1); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("start_array(len)") + { + std::vector v = {'[', '#', 'i', '2', 'T', 'F'}; + SaxCountdown scp(0); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("start_object(len)") + { + std::vector v = {'{', '#', 'i', '1', 3, 'f', 'o', 'o', 'F'}; + SaxCountdown scp(0); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("key() in object with length") + { + std::vector v = {'{', 'i', 3, 'f', 'o', 'o', 'F', '}'}; + SaxCountdown scp(1); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("start_array() in ndarray _ArraySize_") + { + std::vector v = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 2, 1, 1, 2}; + SaxCountdown scp(2); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("number_integer() in ndarray _ArraySize_") + { + std::vector v = {'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 1, 1, 2}; + SaxCountdown scp(3); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("key() in ndarray _ArrayType_") + { + std::vector v = {'[', '$', 'U', '#', '[', '$', 'U', '#', 'i', 2, 2, 2, 1, 2, 3, 4}; + SaxCountdown scp(8); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("string() in ndarray _ArrayType_") + { + std::vector v = {'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 3, 2, 6, 5, 4, 3, 2, 1}; + SaxCountdown scp(11); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + + SECTION("start_array() in ndarray _ArrayData_") + { + std::vector v = {'[', '$', 'U', '#', '[', 'i', 2, 'i', 3, ']', 6, 5, 4, 3, 2, 1}; + SaxCountdown scp(13); + CHECK(!json::sax_parse(v, &scp, json::input_format_t::bjdata)); + } + } + + SECTION("parsing values") + { + SECTION("strings") + { + // create a single-character string for all number types + std::vector s_i = {'S', 'i', 1, 'a'}; + std::vector s_U = {'S', 'U', 1, 'a'}; + std::vector s_I = {'S', 'I', 1, 0, 'a'}; + std::vector s_u = {'S', 'u', 1, 0, 'a'}; + std::vector s_l = {'S', 'l', 1, 0, 0, 0, 'a'}; + std::vector s_m = {'S', 'm', 1, 0, 0, 0, 'a'}; + std::vector s_L = {'S', 'L', 1, 0, 0, 0, 0, 0, 0, 0, 'a'}; + std::vector s_M = {'S', 'M', 1, 0, 0, 0, 0, 0, 0, 0, 'a'}; + + // check if string is parsed correctly to "a" + CHECK(json::from_bjdata(s_i) == "a"); + CHECK(json::from_bjdata(s_U) == "a"); + CHECK(json::from_bjdata(s_I) == "a"); + CHECK(json::from_bjdata(s_u) == "a"); + CHECK(json::from_bjdata(s_l) == "a"); + CHECK(json::from_bjdata(s_m) == "a"); + CHECK(json::from_bjdata(s_L) == "a"); + CHECK(json::from_bjdata(s_M) == "a"); + + // roundtrip: output should be optimized + CHECK(json::to_bjdata(json::from_bjdata(s_i)) == s_i); + CHECK(json::to_bjdata(json::from_bjdata(s_U)) == s_i); + CHECK(json::to_bjdata(json::from_bjdata(s_I)) == s_i); + CHECK(json::to_bjdata(json::from_bjdata(s_u)) == s_i); + CHECK(json::to_bjdata(json::from_bjdata(s_l)) == s_i); + CHECK(json::to_bjdata(json::from_bjdata(s_m)) == s_i); + CHECK(json::to_bjdata(json::from_bjdata(s_L)) == s_i); + CHECK(json::to_bjdata(json::from_bjdata(s_M)) == s_i); + } + + SECTION("number") + { + SECTION("float") + { + // float32 + std::vector v_d = {'d', 0xd0, 0x0f, 0x49, 0x40}; + CHECK(json::from_bjdata(v_d) == 3.14159f); + + // float64 + std::vector v_D = {'D', 0x6e, 0x86, 0x1b, 0xf0, 0xf9, 0x21, 0x09, 0x40}; + CHECK(json::from_bjdata(v_D) == 3.14159); + + // float32 is serialized as float64 as the library does not support float32 + CHECK(json::to_bjdata(json::from_bjdata(v_d)) == json::to_bjdata(3.14159f)); + } + } + + SECTION("array") + { + SECTION("optimized version (length only)") + { + // create vector with two elements of the same type + std::vector v_TU = {'[', '#', 'U', 2, 'T', 'T'}; + std::vector v_T = {'[', '#', 'i', 2, 'T', 'T'}; + std::vector v_F = {'[', '#', 'i', 2, 'F', 'F'}; + std::vector v_Z = {'[', '#', 'i', 2, 'Z', 'Z'}; + std::vector v_i = {'[', '#', 'i', 2, 'i', 0x7F, 'i', 0x7F}; + std::vector v_U = {'[', '#', 'i', 2, 'U', 0xFF, 'U', 0xFF}; + std::vector v_I = {'[', '#', 'i', 2, 'I', 0xFF, 0x7F, 'I', 0xFF, 0x7F}; + std::vector v_u = {'[', '#', 'i', 2, 'u', 0x0F, 0xA7, 'u', 0x0F, 0xA7}; + std::vector v_l = {'[', '#', 'i', 2, 'l', 0xFF, 0xFF, 0xFF, 0x7F, 'l', 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_m = {'[', '#', 'i', 2, 'm', 0xFF, 0xC9, 0x9A, 0xBB, 'm', 0xFF, 0xC9, 0x9A, 0xBB}; + std::vector v_L = {'[', '#', 'i', 2, 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 'L', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_M = {'[', '#', 'i', 2, 'M', 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, 'M', 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D}; + std::vector v_D = {'[', '#', 'i', 2, 'D', 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40, 'D', 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40}; + std::vector v_S = {'[', '#', 'i', 2, 'S', 'i', 1, 'a', 'S', 'i', 1, 'a'}; + std::vector v_C = {'[', '#', 'i', 2, 'C', 'a', 'C', 'a'}; + + // check if vector is parsed correctly + CHECK(json::from_bjdata(v_TU) == json({true, true})); + CHECK(json::from_bjdata(v_T) == json({true, true})); + CHECK(json::from_bjdata(v_F) == json({false, false})); + CHECK(json::from_bjdata(v_Z) == json({nullptr, nullptr})); + CHECK(json::from_bjdata(v_i) == json({127, 127})); + CHECK(json::from_bjdata(v_U) == json({255, 255})); + CHECK(json::from_bjdata(v_I) == json({32767, 32767})); + CHECK(json::from_bjdata(v_u) == json({42767, 42767})); + CHECK(json::from_bjdata(v_l) == json({2147483647, 2147483647})); + CHECK(json::from_bjdata(v_m) == json({3147483647, 3147483647})); + CHECK(json::from_bjdata(v_L) == json({9223372036854775807, 9223372036854775807})); + CHECK(json::from_bjdata(v_M) == json({10223372036854775807ull, 10223372036854775807ull})); + CHECK(json::from_bjdata(v_D) == json({3.1415926, 3.1415926})); + CHECK(json::from_bjdata(v_S) == json({"a", "a"})); + CHECK(json::from_bjdata(v_C) == json({"a", "a"})); + + // roundtrip: output should be optimized + CHECK(json::to_bjdata(json::from_bjdata(v_T), true) == v_T); + CHECK(json::to_bjdata(json::from_bjdata(v_F), true) == v_F); + CHECK(json::to_bjdata(json::from_bjdata(v_Z), true) == v_Z); + CHECK(json::to_bjdata(json::from_bjdata(v_i), true) == v_i); + CHECK(json::to_bjdata(json::from_bjdata(v_U), true) == v_U); + CHECK(json::to_bjdata(json::from_bjdata(v_I), true) == v_I); + CHECK(json::to_bjdata(json::from_bjdata(v_u), true) == v_u); + CHECK(json::to_bjdata(json::from_bjdata(v_l), true) == v_l); + CHECK(json::to_bjdata(json::from_bjdata(v_m), true) == v_m); + CHECK(json::to_bjdata(json::from_bjdata(v_L), true) == v_L); + CHECK(json::to_bjdata(json::from_bjdata(v_M), true) == v_M); + CHECK(json::to_bjdata(json::from_bjdata(v_D), true) == v_D); + CHECK(json::to_bjdata(json::from_bjdata(v_S), true) == v_S); + CHECK(json::to_bjdata(json::from_bjdata(v_C), true) == v_S); // char is serialized to string + } + + SECTION("optimized version (type and length)") + { + // create vector with two elements of the same type + std::vector v_i = {'[', '$', 'i', '#', 'i', 2, 0x7F, 0x7F}; + std::vector v_U = {'[', '$', 'U', '#', 'i', 2, 0xFF, 0xFF}; + std::vector v_I = {'[', '$', 'I', '#', 'i', 2, 0xFF, 0x7F, 0xFF, 0x7F}; + std::vector v_u = {'[', '$', 'u', '#', 'i', 2, 0x0F, 0xA7, 0x0F, 0xA7}; + std::vector v_l = {'[', '$', 'l', '#', 'i', 2, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_m = {'[', '$', 'm', '#', 'i', 2, 0xFF, 0xC9, 0x9A, 0xBB, 0xFF, 0xC9, 0x9A, 0xBB}; + std::vector v_L = {'[', '$', 'L', '#', 'i', 2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_M = {'[', '$', 'M', '#', 'i', 2, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D}; + std::vector v_D = {'[', '$', 'D', '#', 'i', 2, 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40, 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40}; + std::vector v_S = {'[', '#', 'i', 2, 'S', 'i', 1, 'a', 'S', 'i', 1, 'a'}; + std::vector v_C = {'[', '$', 'C', '#', 'i', 2, 'a', 'a'}; + + // check if vector is parsed correctly + CHECK(json::from_bjdata(v_i) == json({127, 127})); + CHECK(json::from_bjdata(v_U) == json({255, 255})); + CHECK(json::from_bjdata(v_I) == json({32767, 32767})); + CHECK(json::from_bjdata(v_u) == json({42767, 42767})); + CHECK(json::from_bjdata(v_l) == json({2147483647, 2147483647})); + CHECK(json::from_bjdata(v_m) == json({3147483647, 3147483647})); + CHECK(json::from_bjdata(v_L) == json({9223372036854775807, 9223372036854775807})); + CHECK(json::from_bjdata(v_M) == json({10223372036854775807ull, 10223372036854775807ull})); + CHECK(json::from_bjdata(v_D) == json({3.1415926, 3.1415926})); + CHECK(json::from_bjdata(v_S) == json({"a", "a"})); + CHECK(json::from_bjdata(v_C) == json({"a", "a"})); + + // roundtrip: output should be optimized + std::vector v_empty = {'[', '#', 'i', 0}; + CHECK(json::to_bjdata(json::from_bjdata(v_i), true, true) == v_i); + CHECK(json::to_bjdata(json::from_bjdata(v_U), true, true) == v_U); + CHECK(json::to_bjdata(json::from_bjdata(v_I), true, true) == v_I); + CHECK(json::to_bjdata(json::from_bjdata(v_u), true, true) == v_u); + CHECK(json::to_bjdata(json::from_bjdata(v_l), true, true) == v_l); + CHECK(json::to_bjdata(json::from_bjdata(v_m), true, true) == v_m); + CHECK(json::to_bjdata(json::from_bjdata(v_L), true, true) == v_L); + CHECK(json::to_bjdata(json::from_bjdata(v_M), true, true) == v_M); + CHECK(json::to_bjdata(json::from_bjdata(v_D), true, true) == v_D); + CHECK(json::to_bjdata(json::from_bjdata(v_S), true, true) == v_S); + CHECK(json::to_bjdata(json::from_bjdata(v_C), true, true) == v_S); // char is serialized to string + } + + SECTION("optimized ndarray (type and vector-size as optimized 1D array)") + { + // create vector with two elements of the same type + std::vector v_0 = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 1, 0}; + std::vector v_1 = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 1, 2, 0x7F, 0x7F}; + std::vector v_i = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0x7F, 0x7F}; + std::vector v_U = {'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0xFF, 0xFF}; + std::vector v_I = {'[', '$', 'I', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0xFF, 0x7F, 0xFF, 0x7F}; + std::vector v_u = {'[', '$', 'u', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0x0F, 0xA7, 0x0F, 0xA7}; + std::vector v_l = {'[', '$', 'l', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_m = {'[', '$', 'm', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0xFF, 0xC9, 0x9A, 0xBB, 0xFF, 0xC9, 0x9A, 0xBB}; + std::vector v_L = {'[', '$', 'L', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_M = {'[', '$', 'M', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D}; + std::vector v_D = {'[', '$', 'D', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40, 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40}; + std::vector v_S = {'[', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 'S', 'i', 1, 'a', 'S', 'i', 1, 'a'}; + std::vector v_C = {'[', '$', 'C', '#', '[', '$', 'i', '#', 'i', 2, 1, 2, 'a', 'a'}; + + // check if vector is parsed correctly + CHECK(json::from_bjdata(v_0) == json::array()); + CHECK(json::from_bjdata(v_1) == json({127, 127})); + CHECK(json::from_bjdata(v_i) == json({127, 127})); + CHECK(json::from_bjdata(v_U) == json({255, 255})); + CHECK(json::from_bjdata(v_I) == json({32767, 32767})); + CHECK(json::from_bjdata(v_u) == json({42767, 42767})); + CHECK(json::from_bjdata(v_l) == json({2147483647, 2147483647})); + CHECK(json::from_bjdata(v_m) == json({3147483647, 3147483647})); + CHECK(json::from_bjdata(v_L) == json({9223372036854775807, 9223372036854775807})); + CHECK(json::from_bjdata(v_M) == json({10223372036854775807ull, 10223372036854775807ull})); + CHECK(json::from_bjdata(v_D) == json({3.1415926, 3.1415926})); + CHECK(json::from_bjdata(v_S) == json({"a", "a"})); + CHECK(json::from_bjdata(v_C) == json({"a", "a"})); + } + + SECTION("optimized ndarray (type and vector-size ndarray with JData annotations)") + { + // create vector with 0, 1, 2 elements of the same type + std::vector v_e = {'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 1, 0xFE, 0xFF}; + std::vector v_U = {'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06}; + std::vector v_i = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06}; + std::vector v_u = {'[', '$', 'u', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00}; + std::vector v_I = {'[', '$', 'I', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00}; + std::vector v_m = {'[', '$', 'm', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00}; + std::vector v_l = {'[', '$', 'l', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00}; + std::vector v_M = {'[', '$', 'M', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + std::vector v_L = {'[', '$', 'L', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + std::vector v_d = {'[', '$', 'd', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x00, 0x00, 0x80, 0x3F, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x40, 0x40, 0x00, 0x00, 0x80, 0x40, 0x00, 0x00, 0xA0, 0x40, 0x00, 0x00, 0xC0, 0x40}; + std::vector v_D = {'[', '$', 'D', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x3F, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x40}; + std::vector v_C = {'[', '$', 'C', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 'a', 'b', 'c', 'd', 'e', 'f'}; + + // check if vector is parsed correctly + CHECK(json::from_bjdata(v_e) == json({{"_ArrayData_", {254, 255}}, {"_ArraySize_", {2, 1}}, {"_ArrayType_", "uint8"}})); + CHECK(json::from_bjdata(v_U) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "uint8"}})); + CHECK(json::from_bjdata(v_i) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "int8"}})); + CHECK(json::from_bjdata(v_i) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "int8"}})); + CHECK(json::from_bjdata(v_u) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "uint16"}})); + CHECK(json::from_bjdata(v_I) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "int16"}})); + CHECK(json::from_bjdata(v_m) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "uint32"}})); + CHECK(json::from_bjdata(v_l) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "int32"}})); + CHECK(json::from_bjdata(v_M) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "uint64"}})); + CHECK(json::from_bjdata(v_L) == json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "int64"}})); + CHECK(json::from_bjdata(v_d) == json({{"_ArrayData_", {1.f, 2.f, 3.f, 4.f, 5.f, 6.f}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "single"}})); + CHECK(json::from_bjdata(v_D) == json({{"_ArrayData_", {1., 2., 3., 4., 5., 6.}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "double"}})); + CHECK(json::from_bjdata(v_C) == json({{"_ArrayData_", {'a', 'b', 'c', 'd', 'e', 'f'}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "char"}})); + + // roundtrip: output should be optimized + CHECK(json::to_bjdata(json::from_bjdata(v_e), true, true) == v_e); + CHECK(json::to_bjdata(json::from_bjdata(v_U), true, true) == v_U); + CHECK(json::to_bjdata(json::from_bjdata(v_i), true, true) == v_i); + CHECK(json::to_bjdata(json::from_bjdata(v_u), true, true) == v_u); + CHECK(json::to_bjdata(json::from_bjdata(v_I), true, true) == v_I); + CHECK(json::to_bjdata(json::from_bjdata(v_m), true, true) == v_m); + CHECK(json::to_bjdata(json::from_bjdata(v_l), true, true) == v_l); + CHECK(json::to_bjdata(json::from_bjdata(v_M), true, true) == v_M); + CHECK(json::to_bjdata(json::from_bjdata(v_L), true, true) == v_L); + CHECK(json::to_bjdata(json::from_bjdata(v_d), true, true) == v_d); + CHECK(json::to_bjdata(json::from_bjdata(v_D), true, true) == v_D); + CHECK(json::to_bjdata(json::from_bjdata(v_C), true, true) == v_C); + } + + SECTION("optimized ndarray (type and vector-size as 1D array)") + { + // create vector with two elements of the same type + std::vector v_0 = {'[', '$', 'i', '#', '[', ']'}; + std::vector v_i = {'[', '$', 'i', '#', '[', 'i', 1, 'i', 2, ']', 0x7F, 0x7F}; + std::vector v_U = {'[', '$', 'U', '#', '[', 'i', 1, 'i', 2, ']', 0xFF, 0xFF}; + std::vector v_I = {'[', '$', 'I', '#', '[', 'i', 1, 'i', 2, ']', 0xFF, 0x7F, 0xFF, 0x7F}; + std::vector v_u = {'[', '$', 'u', '#', '[', 'i', 1, 'i', 2, ']', 0x0F, 0xA7, 0x0F, 0xA7}; + std::vector v_l = {'[', '$', 'l', '#', '[', 'i', 1, 'i', 2, ']', 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_m = {'[', '$', 'm', '#', '[', 'i', 1, 'i', 2, ']', 0xFF, 0xC9, 0x9A, 0xBB, 0xFF, 0xC9, 0x9A, 0xBB}; + std::vector v_L = {'[', '$', 'L', '#', '[', 'i', 1, 'i', 2, ']', 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_M = {'[', '$', 'M', '#', '[', 'i', 1, 'i', 2, ']', 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D}; + std::vector v_D = {'[', '$', 'D', '#', '[', 'i', 1, 'i', 2, ']', 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40, 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40}; + std::vector v_S = {'[', '#', '[', 'i', 1, 'i', 2, ']', 'S', 'i', 1, 'a', 'S', 'i', 1, 'a'}; + std::vector v_C = {'[', '$', 'C', '#', '[', 'i', 1, 'i', 2, ']', 'a', 'a'}; + + // check if vector is parsed correctly + CHECK(json::from_bjdata(v_0) == json::array()); + CHECK(json::from_bjdata(v_i) == json({127, 127})); + CHECK(json::from_bjdata(v_U) == json({255, 255})); + CHECK(json::from_bjdata(v_I) == json({32767, 32767})); + CHECK(json::from_bjdata(v_u) == json({42767, 42767})); + CHECK(json::from_bjdata(v_l) == json({2147483647, 2147483647})); + CHECK(json::from_bjdata(v_m) == json({3147483647, 3147483647})); + CHECK(json::from_bjdata(v_L) == json({9223372036854775807, 9223372036854775807})); + CHECK(json::from_bjdata(v_M) == json({10223372036854775807ull, 10223372036854775807ull})); + CHECK(json::from_bjdata(v_D) == json({3.1415926, 3.1415926})); + CHECK(json::from_bjdata(v_S) == json({"a", "a"})); + CHECK(json::from_bjdata(v_C) == json({"a", "a"})); + } + + SECTION("optimized ndarray (type and vector-size as size-optimized array)") + { + // create vector with two elements of the same type + std::vector v_i = {'[', '$', 'i', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0x7F, 0x7F}; + std::vector v_U = {'[', '$', 'U', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0xFF, 0xFF}; + std::vector v_I = {'[', '$', 'I', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0xFF, 0x7F, 0xFF, 0x7F}; + std::vector v_u = {'[', '$', 'u', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0x0F, 0xA7, 0x0F, 0xA7}; + std::vector v_l = {'[', '$', 'l', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_m = {'[', '$', 'm', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0xFF, 0xC9, 0x9A, 0xBB, 0xFF, 0xC9, 0x9A, 0xBB}; + std::vector v_L = {'[', '$', 'L', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F}; + std::vector v_M = {'[', '$', 'M', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D}; + std::vector v_D = {'[', '$', 'D', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40, 0x4a, 0xd8, 0x12, 0x4d, 0xfb, 0x21, 0x09, 0x40}; + std::vector v_S = {'[', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 'S', 'i', 1, 'a', 'S', 'i', 1, 'a'}; + std::vector v_C = {'[', '$', 'C', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2, 'a', 'a'}; + + // check if vector is parsed correctly + CHECK(json::from_bjdata(v_i) == json({127, 127})); + CHECK(json::from_bjdata(v_U) == json({255, 255})); + CHECK(json::from_bjdata(v_I) == json({32767, 32767})); + CHECK(json::from_bjdata(v_u) == json({42767, 42767})); + CHECK(json::from_bjdata(v_l) == json({2147483647, 2147483647})); + CHECK(json::from_bjdata(v_m) == json({3147483647, 3147483647})); + CHECK(json::from_bjdata(v_L) == json({9223372036854775807, 9223372036854775807})); + CHECK(json::from_bjdata(v_M) == json({10223372036854775807ull, 10223372036854775807ull})); + CHECK(json::from_bjdata(v_D) == json({3.1415926, 3.1415926})); + CHECK(json::from_bjdata(v_S) == json({"a", "a"})); + CHECK(json::from_bjdata(v_C) == json({"a", "a"})); + } + + SECTION("invalid ndarray annotations remains as object") + { + // check if invalid ND array annotations stay as object + json j_type = json({{"_ArrayData_", {1, 2, 3, 4, 5, 6}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "invalidtype"}}); + json j_size = json({{"_ArrayData_", {1, 2, 3, 4, 5}}, {"_ArraySize_", {2, 3}}, {"_ArrayType_", "uint8"}}); + + // roundtrip: output should stay as object + CHECK(json::from_bjdata(json::to_bjdata(j_type), true, true) == j_type); + CHECK(json::from_bjdata(json::to_bjdata(j_size), true, true) == j_size); + } + + SECTION("do not accept NTFZ markers in ndarray optimized type") + { + json _; + std::vector v_N = {'[', '$', 'N', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2}; + std::vector v_T = {'[', '$', 'T', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2}; + std::vector v_F = {'[', '$', 'F', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2}; + std::vector v_Z = {'[', '$', 'Z', '#', '[', '#', 'i', 2, 'i', 1, 'i', 2}; + + CHECK(json::from_bjdata(v_N, true, true).is_discarded()); + CHECK(json::from_bjdata(v_T, true, true).is_discarded()); + CHECK(json::from_bjdata(v_F, true, true).is_discarded()); + CHECK(json::from_bjdata(v_Z, true, true).is_discarded()); + } + + SECTION("do not accept NTFZ markers in ndarray optimized type") + { + json _; + std::vector v_N = {'[', '$', 'N', '#', '[', 'i', 1, 'i', 2, ']'}; + std::vector v_T = {'[', '$', 'T', '#', '[', 'i', 1, 'i', 2, ']'}; + std::vector v_F = {'[', '$', 'F', '#', '[', 'i', 1, 'i', 2, ']'}; + std::vector v_Z = {'[', '$', 'Z', '#', '[', 'i', 1, 'i', 2, ']'}; + + CHECK(json::from_bjdata(v_N, true, true).is_discarded()); + CHECK(json::from_bjdata(v_T, true, true).is_discarded()); + CHECK(json::from_bjdata(v_F, true, true).is_discarded()); + CHECK(json::from_bjdata(v_Z, true, true).is_discarded()); + } + } + } + + SECTION("parse errors") + { + SECTION("empty byte vector") + { + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(std::vector()), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(std::vector()), + "[json.exception.parse_error.110] parse error at byte 1: syntax error while parsing BJData value: unexpected end of input"); + } + + SECTION("char") + { + SECTION("eof after C byte") + { + std::vector v = {'C'}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BJData char: unexpected end of input"); + } + + SECTION("byte out of range") + { + std::vector v = {'C', 130}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing BJData char: byte after 'C' must be in range 0x00..0x7F; last byte: 0x82"); + } + } + + SECTION("strings") + { + SECTION("eof after S byte") + { + std::vector v = {'S'}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BJData value: unexpected end of input"); + } + + SECTION("invalid byte") + { + std::vector v = {'S', '1', 'a'}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing BJData string: expected length type specification (U, i, u, I, m, l, M, L); last byte: 0x31"); + } + + SECTION("parse bjdata markers in ubjson") + { + // create a single-character string for all number types + std::vector s_u = {'S', 'u', 1, 0, 'a'}; + std::vector s_m = {'S', 'm', 1, 0, 0, 0, 'a'}; + std::vector s_M = {'S', 'M', 1, 0, 0, 0, 0, 0, 0, 0, 'a'}; + + json _; + // check if string is parsed correctly to "a" + CHECK_THROWS_AS(_ = json::from_ubjson(s_u), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_ubjson(s_u), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing UBJSON string: expected length type specification (U, i, I, l, L); last byte: 0x75"); + + CHECK_THROWS_AS(_ = json::from_ubjson(s_m), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_ubjson(s_m), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing UBJSON string: expected length type specification (U, i, I, l, L); last byte: 0x6D"); + + CHECK_THROWS_AS(_ = json::from_ubjson(s_M), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_ubjson(s_M), "[json.exception.parse_error.113] parse error at byte 2: syntax error while parsing UBJSON string: expected length type specification (U, i, I, l, L); last byte: 0x4D"); + } + } + + SECTION("array") + { + SECTION("optimized array: no size following type") + { + std::vector v = {'[', '$', 'i', 2}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.112] parse error at byte 4: syntax error while parsing BJData size: expected '#' after type information; last byte: 0x02"); + } + } + + SECTION("strings") + { + std::vector vS = {'S'}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(vS), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vS), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(vS, true, false).is_discarded()); + + std::vector v = {'S', 'i', '2', 'a'}; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BJData string: unexpected end of input"); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + + std::vector vC = {'C'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vC), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vC), "[json.exception.parse_error.110] parse error at byte 2: syntax error while parsing BJData char: unexpected end of input"); + CHECK(json::from_bjdata(vC, true, false).is_discarded()); + } + + SECTION("sizes") + { + std::vector vU = {'[', '#', 'U'}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(vU), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vU), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vU, true, false).is_discarded()); + + std::vector vi = {'[', '#', 'i'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vi), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vi), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vi, true, false).is_discarded()); + + std::vector vI = {'[', '#', 'I'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vI), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vI), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vI, true, false).is_discarded()); + + std::vector vu = {'[', '#', 'u'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vu), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vu), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vu, true, false).is_discarded()); + + std::vector vl = {'[', '#', 'l'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vl), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vl), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vl, true, false).is_discarded()); + + std::vector vm = {'[', '#', 'm'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vm), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vm), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vm, true, false).is_discarded()); + + std::vector vL = {'[', '#', 'L'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vL), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vL), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vL, true, false).is_discarded()); + + std::vector vM = {'[', '#', 'M'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vM), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vM), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vM, true, false).is_discarded()); + + std::vector v0 = {'[', '#', 'T', ']'}; + CHECK_THROWS_AS(_ = json::from_bjdata(v0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v0), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x54"); + CHECK(json::from_bjdata(v0, true, false).is_discarded()); + } + + SECTION("parse bjdata markers as array size in ubjson") + { + json _; + std::vector vu = {'[', '#', 'u'}; + CHECK_THROWS_AS(_ = json::from_ubjson(vu), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_ubjson(vu), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing UBJSON size: expected length type specification (U, i, I, l, L) after '#'; last byte: 0x75"); + CHECK(json::from_ubjson(vu, true, false).is_discarded()); + + std::vector vm = {'[', '#', 'm'}; + CHECK_THROWS_AS(_ = json::from_ubjson(vm), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_ubjson(vm), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing UBJSON size: expected length type specification (U, i, I, l, L) after '#'; last byte: 0x6D"); + CHECK(json::from_ubjson(vm, true, false).is_discarded()); + + std::vector vM = {'[', '#', 'M'}; + CHECK_THROWS_AS(_ = json::from_ubjson(vM), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_ubjson(vM), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing UBJSON size: expected length type specification (U, i, I, l, L) after '#'; last byte: 0x4D"); + CHECK(json::from_ubjson(vM, true, false).is_discarded()); + + std::vector v0 = {'[', '#', '['}; + CHECK_THROWS_AS(_ = json::from_ubjson(v0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_ubjson(v0), "[json.exception.parse_error.113] parse error at byte 3: syntax error while parsing UBJSON size: expected length type specification (U, i, I, l, L) after '#'; last byte: 0x5B"); + CHECK(json::from_ubjson(v0, true, false).is_discarded()); + } + + SECTION("types") + { + std::vector v0 = {'[', '$'}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(v0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v0), "[json.exception.parse_error.110] parse error at byte 3: syntax error while parsing BJData type: unexpected end of input"); + CHECK(json::from_bjdata(v0, true, false).is_discarded()); + + std::vector vi = {'[', '$', '#'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vi), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vi), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(vi, true, false).is_discarded()); + + std::vector vU = {'[', '$', 'U'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vU), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vU), "[json.exception.parse_error.110] parse error at byte 4: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(vU, true, false).is_discarded()); + } + + SECTION("arrays") + { + std::vector vST = {'[', '$', 'i', '#', 'i', 2, 1}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(vST), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vST), "[json.exception.parse_error.110] parse error at byte 8: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vST, true, false).is_discarded()); + + std::vector vS = {'[', '#', 'i', 2, 'i', 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(vS), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vS), "[json.exception.parse_error.110] parse error at byte 7: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(vS, true, false).is_discarded()); + + std::vector v = {'[', 'i', 2, 'i', 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 6: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + } + + SECTION("ndarrays") + { + std::vector vST = {'[', '$', 'i', '#', '[', '$', 'i', '#'}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(vST), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vST), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0xFF"); + CHECK(json::from_bjdata(vST, true, false).is_discarded()); + + std::vector v = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 1, 2}; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 13: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + + std::vector vS0 = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'i', 2, 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(vS0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vS0), "[json.exception.parse_error.110] parse error at byte 12: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vS0, true, false).is_discarded()); + + std::vector vS = {'[', '$', 'i', '#', '[', '#', 'i', 2, 1, 2, 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(vS), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vS), "[json.exception.parse_error.113] parse error at byte 9: syntax error while parsing BJData size: expected length type specification (U, i, u, I, m, l, M, L) after '#'; last byte: 0x01"); + CHECK(json::from_bjdata(vS, true, false).is_discarded()); + + std::vector vT = {'[', '$', 'i', '#', '[', 'i', 2, 'i'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vT), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vT), "[json.exception.parse_error.110] parse error at byte 9: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vT, true, false).is_discarded()); + + std::vector vT0 = {'[', '$', 'i', '#', '[', 'i'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vT0), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vT0), "[json.exception.parse_error.110] parse error at byte 7: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vT0, true, false).is_discarded()); + + std::vector vu = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'u', 1, 0}; + CHECK_THROWS_AS(_ = json::from_bjdata(vu), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vu), "[json.exception.parse_error.110] parse error at byte 12: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vu, true, false).is_discarded()); + + std::vector vm = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'm', 1, 0, 0, 0}; + CHECK_THROWS_AS(_ = json::from_bjdata(vm), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vm), "[json.exception.parse_error.110] parse error at byte 14: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vm, true, false).is_discarded()); + + std::vector vM = {'[', '$', 'i', '#', '[', '$', 'i', '#', 'M', 1, 0, 0, 0, 0, 0, 0, 0}; + CHECK_THROWS_AS(_ = json::from_bjdata(vM), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vM), "[json.exception.parse_error.110] parse error at byte 18: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vM, true, false).is_discarded()); + + std::vector vU = {'[', '$', 'U', '#', '[', '$', 'i', '#', 'i', 2, 2, 3, 1, 2, 3, 4, 5}; + CHECK_THROWS_AS(_ = json::from_bjdata(vU), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vU), "[json.exception.parse_error.110] parse error at byte 18: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vU, true, false).is_discarded()); + + std::vector vT1 = {'[', '$', 'T', '#', '[', '$', 'i', '#', 'i', 2, 2, 3}; + CHECK(json::from_bjdata(vT1, true, false).is_discarded()); + + std::vector vh = {'[', '$', 'h', '#', '[', '$', 'i', '#', 'i', 2, 2, 3}; + CHECK(json::from_bjdata(vh, true, false).is_discarded()); + } + + SECTION("objects") + { + std::vector vST = {'{', '$', 'i', '#', 'i', 2, 'i', 1, 'a', 1}; + json _; + CHECK_THROWS_AS(_ = json::from_bjdata(vST), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vST), "[json.exception.parse_error.110] parse error at byte 11: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(vST, true, false).is_discarded()); + + std::vector vT = {'{', '$', 'i', 'i', 1, 'a', 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(vT), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vT), "[json.exception.parse_error.112] parse error at byte 4: syntax error while parsing BJData size: expected '#' after type information; last byte: 0x69"); + CHECK(json::from_bjdata(vT, true, false).is_discarded()); + + std::vector vS = {'{', '#', 'i', 2, 'i', 1, 'a', 'i', 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(vS), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vS), "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(vS, true, false).is_discarded()); + + std::vector v = {'{', 'i', 1, 'a', 'i', 1}; + CHECK_THROWS_AS(_ = json::from_bjdata(v), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v), "[json.exception.parse_error.110] parse error at byte 7: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(v, true, false).is_discarded()); + + std::vector v2 = {'{', 'i', 1, 'a', 'i', 1, 'i'}; + CHECK_THROWS_AS(_ = json::from_bjdata(v2), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v2), "[json.exception.parse_error.110] parse error at byte 8: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(v2, true, false).is_discarded()); + + std::vector v3 = {'{', 'i', 1, 'a'}; + CHECK_THROWS_AS(_ = json::from_bjdata(v3), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(v3), "[json.exception.parse_error.110] parse error at byte 5: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(v3, true, false).is_discarded()); + + std::vector vST1 = {'{', '$', 'd', '#', 'i', 2, 'i', 1, 'a'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vST1), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vST1), "[json.exception.parse_error.110] parse error at byte 10: syntax error while parsing BJData number: unexpected end of input"); + CHECK(json::from_bjdata(vST1, true, false).is_discarded()); + + std::vector vST2 = {'{', '#', 'i', 2, 'i', 1, 'a'}; + CHECK_THROWS_AS(_ = json::from_bjdata(vST2), json::parse_error&); + CHECK_THROWS_WITH(_ = json::from_bjdata(vST2), "[json.exception.parse_error.110] parse error at byte 8: syntax error while parsing BJData value: unexpected end of input"); + CHECK(json::from_bjdata(vST2, true, false).is_discarded()); + + std::vector vO = {'{', '#', '[', 'i', 2, 'i', 1, ']', 'i', 1, 'a', 'i', 1, 'i', 1, 'b', 'i', 2}; + CHECK(json::from_bjdata(vO, true, false).is_discarded()); + } + } + + SECTION("writing optimized values") + { + SECTION("integer") + { + SECTION("array of i") + { + json j = {1, -1}; + std::vector expected = {'[', '$', 'i', '#', 'i', 2, 1, 0xff}; + CHECK(json::to_bjdata(j, true, true) == expected); + } + + SECTION("array of U") + { + json j = {200, 201}; + std::vector expected = {'[', '$', 'U', '#', 'i', 2, 0xC8, 0xC9}; + CHECK(json::to_bjdata(j, true, true) == expected); + } + + SECTION("array of I") + { + json j = {30000, -30000}; + std::vector expected = {'[', '$', 'I', '#', 'i', 2, 0x30, 0x75, 0xd0, 0x8a}; + CHECK(json::to_bjdata(j, true, true) == expected); + } + + SECTION("array of u") + { + json j = {50000, 50001}; + std::vector expected = {'[', '$', 'u', '#', 'i', 2, 0x50, 0xC3, 0x51, 0xC3}; + CHECK(json::to_bjdata(j, true, true) == expected); + } + + SECTION("array of l") + { + json j = {70000, -70000}; + std::vector expected = {'[', '$', 'l', '#', 'i', 2, 0x70, 0x11, 0x01, 0x00, 0x90, 0xEE, 0xFE, 0xFF}; + CHECK(json::to_bjdata(j, true, true) == expected); + } + + SECTION("array of m") + { + json j = {3147483647, 3147483648}; + std::vector expected = {'[', '$', 'm', '#', 'i', 2, 0xFF, 0xC9, 0x9A, 0xBB, 0x00, 0xCA, 0x9A, 0xBB}; + CHECK(json::to_bjdata(j, true, true) == expected); + } + + SECTION("array of L") + { + json j = {5000000000, -5000000000}; + std::vector expected = {'[', '$', 'L', '#', 'i', 2, 0x00, 0xF2, 0x05, 0x2A, 0x01, 0x00, 0x00, 0x00, 0x00, 0x0E, 0xFA, 0xD5, 0xFE, 0xFF, 0xFF, 0xFF}; + CHECK(json::to_bjdata(j, true, true) == expected); + } + } + + SECTION("unsigned integer") + { + SECTION("array of i") + { + json j = {1u, 2u}; + std::vector expected = {'[', '$', 'i', '#', 'i', 2, 1, 2}; + std::vector expected_size = {'[', '#', 'i', 2, 'i', 1, 'i', 2}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + + SECTION("array of U") + { + json j = {200u, 201u}; + std::vector expected = {'[', '$', 'U', '#', 'i', 2, 0xC8, 0xC9}; + std::vector expected_size = {'[', '#', 'i', 2, 'U', 0xC8, 'U', 0xC9}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + + SECTION("array of I") + { + json j = {30000u, 30001u}; + std::vector expected = {'[', '$', 'I', '#', 'i', 2, 0x30, 0x75, 0x31, 0x75}; + std::vector expected_size = {'[', '#', 'i', 2, 'I', 0x30, 0x75, 'I', 0x31, 0x75}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + + SECTION("array of u") + { + json j = {50000u, 50001u}; + std::vector expected = {'[', '$', 'u', '#', 'i', 2, 0x50, 0xC3, 0x51, 0xC3}; + std::vector expected_size = {'[', '#', 'i', 2, 'u', 0x50, 0xC3, 'u', 0x51, 0xC3}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + + SECTION("array of l") + { + json j = {70000u, 70001u}; + std::vector expected = {'[', '$', 'l', '#', 'i', 2, 0x70, 0x11, 0x01, 0x00, 0x71, 0x11, 0x01, 0x00}; + std::vector expected_size = {'[', '#', 'i', 2, 'l', 0x70, 0x11, 0x01, 0x00, 'l', 0x71, 0x11, 0x01, 0x00}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + + SECTION("array of m") + { + json j = {3147483647u, 3147483648u}; + std::vector expected = {'[', '$', 'm', '#', 'i', 2, 0xFF, 0xC9, 0x9A, 0xBB, 0x00, 0xCA, 0x9A, 0xBB}; + std::vector expected_size = {'[', '#', 'i', 2, 'm', 0xFF, 0xC9, 0x9A, 0xBB, 'm', 0x00, 0xCA, 0x9A, 0xBB}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + + SECTION("array of L") + { + json j = {5000000000u, 5000000001u}; + std::vector expected = {'[', '$', 'L', '#', 'i', 2, 0x00, 0xF2, 0x05, 0x2A, 0x01, 0x00, 0x00, 0x00, 0x01, 0xF2, 0x05, 0x2A, 0x01, 0x00, 0x00, 0x00}; + std::vector expected_size = {'[', '#', 'i', 2, 'L', 0x00, 0xF2, 0x05, 0x2A, 0x01, 0x00, 0x00, 0x00, 'L', 0x01, 0xF2, 0x05, 0x2A, 0x01, 0x00, 0x00, 0x00}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + + SECTION("array of M") + { + json j = {10223372036854775807ull, 10223372036854775808ull}; + std::vector expected = {'[', '$', 'M', '#', 'i', 2, 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, 0x00, 0x00, 0x64, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D}; + std::vector expected_size = {'[', '#', 'i', 2, 'M', 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, 'M', 0x00, 0x00, 0x64, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D}; + CHECK(json::to_bjdata(j, true, true) == expected); + CHECK(json::to_bjdata(j, true) == expected_size); + } + } + } +} + +TEST_CASE("Universal Binary JSON Specification Examples 1") +{ + SECTION("Null Value") + { + json j = {{"passcode", nullptr}}; + std::vector v = {'{', 'i', 8, 'p', 'a', 's', 's', 'c', 'o', 'd', 'e', 'Z', '}'}; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("No-Op Value") + { + json j = {"foo", "bar", "baz"}; + std::vector v = {'[', 'S', 'i', 3, 'f', 'o', 'o', + 'S', 'i', 3, 'b', 'a', 'r', + 'S', 'i', 3, 'b', 'a', 'z', ']' + }; + std::vector v2 = {'[', 'S', 'i', 3, 'f', 'o', 'o', 'N', + 'S', 'i', 3, 'b', 'a', 'r', 'N', 'N', 'N', + 'S', 'i', 3, 'b', 'a', 'z', 'N', 'N', ']' + }; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + CHECK(json::from_bjdata(v2) == j); + } + + SECTION("Boolean Types") + { + json j = {{"authorized", true}, {"verified", false}}; + std::vector v = {'{', 'i', 10, 'a', 'u', 't', 'h', 'o', 'r', 'i', 'z', 'e', 'd', 'T', + 'i', 8, 'v', 'e', 'r', 'i', 'f', 'i', 'e', 'd', 'F', '}' + }; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Numeric Types") + { + json j = + { + {"int8", 16}, + {"uint8", 255}, + {"int16", 32767}, + {"uint16", 42767}, + {"int32", 2147483647}, + {"uint32", 3147483647}, + {"int64", 9223372036854775807}, + {"uint64", 10223372036854775807ull}, + {"float64", 113243.7863123} + }; + std::vector v = {'{', + 'i', 7, 'f', 'l', 'o', 'a', 't', '6', '4', 'D', 0xcf, 0x34, 0xbc, 0x94, 0xbc, 0xa5, 0xfb, 0x40, + 'i', 5, 'i', 'n', 't', '1', '6', 'I', 0xff, 0x7f, + 'i', 5, 'i', 'n', 't', '3', '2', 'l', 0xff, 0xff, 0xff, 0x7f, + 'i', 5, 'i', 'n', 't', '6', '4', 'L', 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f, + 'i', 4, 'i', 'n', 't', '8', 'i', 16, + 'i', 6, 'u', 'i', 'n', 't', '1', '6', 'u', 0x0F, 0xA7, + 'i', 6, 'u', 'i', 'n', 't', '3', '2', 'm', 0xFF, 0xC9, 0x9A, 0xBB, + 'i', 6, 'u', 'i', 'n', 't', '6', '4', 'M', 0xFF, 0xFF, 0x63, 0xA7, 0xB3, 0xB6, 0xE0, 0x8D, + 'i', 5, 'u', 'i', 'n', 't', '8', 'U', 0xff, + '}' + }; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Char Type") + { + json j = {{"rolecode", "a"}, {"delim", ";"}}; + std::vector v = {'{', 'i', 5, 'd', 'e', 'l', 'i', 'm', 'C', ';', 'i', 8, 'r', 'o', 'l', 'e', 'c', 'o', 'd', 'e', 'C', 'a', '}'}; + //CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("String Type") + { + SECTION("English") + { + json j = "hello"; + std::vector v = {'S', 'i', 5, 'h', 'e', 'l', 'l', 'o'}; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Russian") + { + json j = "привет"; + std::vector v = {'S', 'i', 12, 0xD0, 0xBF, 0xD1, 0x80, 0xD0, 0xB8, 0xD0, 0xB2, 0xD0, 0xB5, 0xD1, 0x82}; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Russian") + { + json j = "مرحبا"; + std::vector v = {'S', 'i', 10, 0xD9, 0x85, 0xD8, 0xB1, 0xD8, 0xAD, 0xD8, 0xA8, 0xD8, 0xA7}; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + } + + SECTION("Array Type") + { + SECTION("size=false type=false") + { + // note the float has been replaced by a double + json j = {nullptr, true, false, 4782345193, 153.132, "ham"}; + std::vector v = {'[', 'Z', 'T', 'F', 'L', 0xE9, 0xCB, 0x0C, 0x1D, 0x01, 0x00, 0x00, 0x00, 'D', 0x4e, 0x62, 0x10, 0x58, 0x39, 0x24, 0x63, 0x40, 'S', 'i', 3, 'h', 'a', 'm', ']'}; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("size=true type=false") + { + // note the float has been replaced by a double + json j = {nullptr, true, false, 4782345193, 153.132, "ham"}; + std::vector v = {'[', '#', 'i', 6, 'Z', 'T', 'F', 'L', 0xE9, 0xCB, 0x0C, 0x1D, 0x01, 0x00, 0x00, 0x00, 'D', 0x4e, 0x62, 0x10, 0x58, 0x39, 0x24, 0x63, 0x40, 'S', 'i', 3, 'h', 'a', 'm'}; + CHECK(json::to_bjdata(j, true) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("size=true type=true") + { + // note the float has been replaced by a double + json j = {nullptr, true, false, 4782345193, 153.132, "ham"}; + std::vector v = {'[', '#', 'i', 6, 'Z', 'T', 'F', 'L', 0xE9, 0xCB, 0x0C, 0x1D, 0x01, 0x00, 0x00, 0x00, 'D', 0x4e, 0x62, 0x10, 0x58, 0x39, 0x24, 0x63, 0x40, 'S', 'i', 3, 'h', 'a', 'm'}; + CHECK(json::to_bjdata(j, true, true) == v); + CHECK(json::from_bjdata(v) == j); + } + } + + SECTION("Object Type") + { + SECTION("size=false type=false") + { + json j = + { + { + "post", { + {"id", 1137}, + {"author", "rkalla"}, + {"timestamp", 1364482090592}, + {"body", "I totally agree!"} + } + } + }; + std::vector v = {'{', 'i', 4, 'p', 'o', 's', 't', '{', + 'i', 6, 'a', 'u', 't', 'h', 'o', 'r', 'S', 'i', 6, 'r', 'k', 'a', 'l', 'l', 'a', + 'i', 4, 'b', 'o', 'd', 'y', 'S', 'i', 16, 'I', ' ', 't', 'o', 't', 'a', 'l', 'l', 'y', ' ', 'a', 'g', 'r', 'e', 'e', '!', + 'i', 2, 'i', 'd', 'I', 0x71, 0x04, + 'i', 9, 't', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', 'L', 0x60, 0x66, 0x78, 0xB1, 0x3D, 0x01, 0x00, 0x00, + '}', '}' + }; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("size=true type=false") + { + json j = + { + { + "post", { + {"id", 1137}, + {"author", "rkalla"}, + {"timestamp", 1364482090592}, + {"body", "I totally agree!"} + } + } + }; + std::vector v = {'{', '#', 'i', 1, 'i', 4, 'p', 'o', 's', 't', '{', '#', 'i', 4, + 'i', 6, 'a', 'u', 't', 'h', 'o', 'r', 'S', 'i', 6, 'r', 'k', 'a', 'l', 'l', 'a', + 'i', 4, 'b', 'o', 'd', 'y', 'S', 'i', 16, 'I', ' ', 't', 'o', 't', 'a', 'l', 'l', 'y', ' ', 'a', 'g', 'r', 'e', 'e', '!', + 'i', 2, 'i', 'd', 'I', 0x71, 0x04, + 'i', 9, 't', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', 'L', 0x60, 0x66, 0x78, 0xB1, 0x3D, 0x01, 0x00, 0x00, + }; + CHECK(json::to_bjdata(j, true) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("size=true type=true") + { + json j = + { + { + "post", { + {"id", 1137}, + {"author", "rkalla"}, + {"timestamp", 1364482090592}, + {"body", "I totally agree!"} + } + } + }; + std::vector v = {'{', '#', 'i', 1, 'i', 4, 'p', 'o', 's', 't', '{', '#', 'i', 4, + 'i', 6, 'a', 'u', 't', 'h', 'o', 'r', 'S', 'i', 6, 'r', 'k', 'a', 'l', 'l', 'a', + 'i', 4, 'b', 'o', 'd', 'y', 'S', 'i', 16, 'I', ' ', 't', 'o', 't', 'a', 'l', 'l', 'y', ' ', 'a', 'g', 'r', 'e', 'e', '!', + 'i', 2, 'i', 'd', 'I', 0x71, 0x04, + 'i', 9, 't', 'i', 'm', 'e', 's', 't', 'a', 'm', 'p', 'L', 0x60, 0x66, 0x78, 0xB1, 0x3D, 0x01, 0x00, 0x00, + }; + CHECK(json::to_bjdata(j, true, true) == v); + CHECK(json::from_bjdata(v) == j); + } + } + + SECTION("Optimized Format") + { + SECTION("Array Example") + { + SECTION("No Optimization") + { + // note the floats have been replaced by doubles + json j = {29.97, 31.13, 67.0, 2.113, 23.888}; + std::vector v = {'[', + 'D', 0xb8, 0x1e, 0x85, 0xeb, 0x51, 0xf8, 0x3d, 0x40, + 'D', 0xe1, 0x7a, 0x14, 0xae, 0x47, 0x21, 0x3f, 0x40, + 'D', 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, + 'D', 0x81, 0x95, 0x43, 0x8b, 0x6c, 0xe7, 0x00, 0x40, + 'D', 0x17, 0xd9, 0xce, 0xf7, 0x53, 0xe3, 0x37, 0x40, + ']' + }; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Optimized with count") + { + // note the floats have been replaced by doubles + json j = {29.97, 31.13, 67.0, 2.113, 23.888}; + std::vector v = {'[', '#', 'i', 5, + 'D', 0xb8, 0x1e, 0x85, 0xeb, 0x51, 0xf8, 0x3d, 0x40, + 'D', 0xe1, 0x7a, 0x14, 0xae, 0x47, 0x21, 0x3f, 0x40, + 'D', 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, + 'D', 0x81, 0x95, 0x43, 0x8b, 0x6c, 0xe7, 0x00, 0x40, + 'D', 0x17, 0xd9, 0xce, 0xf7, 0x53, 0xe3, 0x37, 0x40, + }; + CHECK(json::to_bjdata(j, true) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Optimized with type & count") + { + // note the floats have been replaced by doubles + json j = {29.97, 31.13, 67.0, 2.113, 23.888}; + std::vector v = {'[', '$', 'D', '#', 'i', 5, + 0xb8, 0x1e, 0x85, 0xeb, 0x51, 0xf8, 0x3d, 0x40, + 0xe1, 0x7a, 0x14, 0xae, 0x47, 0x21, 0x3f, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, + 0x81, 0x95, 0x43, 0x8b, 0x6c, 0xe7, 0x00, 0x40, + 0x17, 0xd9, 0xce, 0xf7, 0x53, 0xe3, 0x37, 0x40, + }; + CHECK(json::to_bjdata(j, true, true) == v); + CHECK(json::from_bjdata(v) == j); + } + } + + SECTION("Object Example") + { + SECTION("No Optimization") + { + // note the floats have been replaced by doubles + json j = { {"lat", 29.976}, {"long", 31.131}, {"alt", 67.0} }; + std::vector v = {'{', + 'i', 3, 'a', 'l', 't', 'D', 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, + 'i', 3, 'l', 'a', 't', 'D', 0x60, 0xe5, 0xd0, 0x22, 0xdb, 0xf9, 0x3d, 0x40, + 'i', 4, 'l', 'o', 'n', 'g', 'D', 0xa8, 0xc6, 0x4b, 0x37, 0x89, 0x21, 0x3f, 0x40, + '}' + }; + CHECK(json::to_bjdata(j) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Optimized with count") + { + // note the floats have been replaced by doubles + json j = { {"lat", 29.976}, {"long", 31.131}, {"alt", 67.0} }; + std::vector v = {'{', '#', 'i', 3, + 'i', 3, 'a', 'l', 't', 'D', 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, + 'i', 3, 'l', 'a', 't', 'D', 0x60, 0xe5, 0xd0, 0x22, 0xdb, 0xf9, 0x3d, 0x40, + 'i', 4, 'l', 'o', 'n', 'g', 'D', 0xa8, 0xc6, 0x4b, 0x37, 0x89, 0x21, 0x3f, 0x40, + }; + CHECK(json::to_bjdata(j, true) == v); + CHECK(json::from_bjdata(v) == j); + } + + SECTION("Optimized with type & count") + { + // note the floats have been replaced by doubles + json j = { {"lat", 29.976}, {"long", 31.131}, {"alt", 67.0} }; + std::vector v = {'{', '$', 'D', '#', 'i', 3, + 'i', 3, 'a', 'l', 't', 0x00, 0x00, 0x00, 0x00, 0x00, 0xc0, 0x50, 0x40, + 'i', 3, 'l', 'a', 't', 0x60, 0xe5, 0xd0, 0x22, 0xdb, 0xf9, 0x3d, 0x40, + 'i', 4, 'l', 'o', 'n', 'g', 0xa8, 0xc6, 0x4b, 0x37, 0x89, 0x21, 0x3f, 0x40, + }; + CHECK(json::to_bjdata(j, true, true) == v); + CHECK(json::from_bjdata(v) == j); + } + } + + SECTION("Special Cases (Null, No-Op and Boolean)") + { + SECTION("Array") + { + std::vector v = {'[', '$', 'N', '#', 'I', 0x00, 0x02}; + CHECK(json::from_bjdata(v, true, true).is_discarded()); + } + + SECTION("Object") + { + std::vector v = {'{', '$', 'Z', '#', 'i', 3, 'i', 4, 'n', 'a', 'm', 'e', 'i', 8, 'p', 'a', 's', 's', 'w', 'o', 'r', 'd', 'i', 5, 'e', 'm', 'a', 'i', 'l'}; + CHECK(json::from_bjdata(v, true, true).is_discarded()); + } + } + } +} + +#if !defined(JSON_NOEXCEPTION) +TEST_CASE("all BJData first bytes") +{ + // these bytes will fail immediately with exception parse_error.112 + std::set supported = + { + 'T', 'F', 'Z', 'U', 'i', 'I', 'l', 'L', 'd', 'D', 'C', 'S', '[', '{', 'N', 'H', 'u', 'm', 'M', 'h' + }; + + for (auto i = 0; i < 256; ++i) + { + const auto byte = static_cast(i); + CAPTURE(byte) + + try + { + auto res = json::from_bjdata(std::vector(1, byte)); + } + catch (const json::parse_error& e) + { + // check that parse_error.112 is only thrown if the + // first byte is not in the supported set + INFO_WITH_TEMP(e.what()); + if (supported.find(byte) == supported.end()) + { + CHECK(e.id == 112); + } + else + { + CHECK(e.id != 112); + } + } + } +} +#endif + +TEST_CASE("BJData roundtrips" * doctest::skip()) +{ + SECTION("input from self-generated BJData files") + { + for (std::string filename : + { + TEST_DATA_DIRECTORY "/json_nlohmann_tests/all_unicode.json", + TEST_DATA_DIRECTORY "/json.org/1.json", + TEST_DATA_DIRECTORY "/json.org/2.json", + TEST_DATA_DIRECTORY "/json.org/3.json", + TEST_DATA_DIRECTORY "/json.org/4.json", + TEST_DATA_DIRECTORY "/json.org/5.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip01.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip02.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip03.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip04.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip05.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip06.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip07.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip08.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip09.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip10.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip11.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip12.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip13.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip14.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip15.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip16.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip17.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip18.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip19.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip20.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip21.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip22.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip23.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip24.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip25.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip26.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip27.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip28.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip29.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip30.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip31.json", + TEST_DATA_DIRECTORY "/json_roundtrip/roundtrip32.json", + TEST_DATA_DIRECTORY "/json_testsuite/sample.json", + TEST_DATA_DIRECTORY "/json_tests/pass1.json", + TEST_DATA_DIRECTORY "/json_tests/pass2.json", + TEST_DATA_DIRECTORY "/json_tests/pass3.json" + }) + { + CAPTURE(filename) + + { + INFO_WITH_TEMP(filename + ": std::vector"); + // parse JSON file + std::ifstream f_json(filename); + json j1 = json::parse(f_json); + + // parse BJData file + auto packed = utils::read_binary_file(filename + ".bjdata"); + json j2; + CHECK_NOTHROW(j2 = json::from_bjdata(packed)); + + // compare parsed JSON values + CHECK(j1 == j2); + } + + { + INFO_WITH_TEMP(filename + ": std::ifstream"); + // parse JSON file + std::ifstream f_json(filename); + json j1 = json::parse(f_json); + + // parse BJData file + std::ifstream f_bjdata(filename + ".bjdata", std::ios::binary); + json j2; + CHECK_NOTHROW(j2 = json::from_bjdata(f_bjdata)); + + // compare parsed JSON values + CHECK(j1 == j2); + } + + { + INFO_WITH_TEMP(filename + ": uint8_t* and size"); + // parse JSON file + std::ifstream f_json(filename); + json j1 = json::parse(f_json); + + // parse BJData file + auto packed = utils::read_binary_file(filename + ".bjdata"); + json j2; + CHECK_NOTHROW(j2 = json::from_bjdata({packed.data(), packed.size()})); + + // compare parsed JSON values + CHECK(j1 == j2); + } + + { + INFO_WITH_TEMP(filename + ": output to output adapters"); + // parse JSON file + std::ifstream f_json(filename); + json j1 = json::parse(f_json); + + // parse BJData file + auto packed = utils::read_binary_file(filename + ".bjdata"); + + { + INFO_WITH_TEMP(filename + ": output adapters: std::vector"); + std::vector vec; + json::to_bjdata(j1, vec); + CHECK(vec == packed); + } + } + } + } +} -- GitLab