diff --git a/modules/core/include/opencv2/core/core_c.h b/modules/core/include/opencv2/core/core_c.h index dd1d93638e1c31d0e6b81b89dc47cf03c14f2603..f7e0bbb7508ec15f5b794bcd1f53b578a60befab 100644 --- a/modules/core/include/opencv2/core/core_c.h +++ b/modules/core/include/opencv2/core/core_c.h @@ -1976,7 +1976,7 @@ CVAPI(void) cvSetIPLAllocators( Cv_iplCreateImageHeader create_header, The function opens file storage for reading or writing data. In the latter case, a new file is created or an existing file is rewritten. The type of the read or written file is determined by the -filename extension: .xml for XML and .yml or .yaml for YAML. +filename extension: .xml for XML, .yml or .yaml for YAML and .json for JSON. At the same time, it also supports adding parameters like "example.xml?base64". The three ways are the same: @@ -2031,7 +2031,8 @@ One and only one of the two above flags must be specified @param type_name Optional parameter - the object type name. In case of XML it is written as a type_id attribute of the structure opening tag. In the case of YAML it is written after a colon following the structure name (see the example in - CvFileStorage description). Mainly it is used with user objects. When the storage is read, the + CvFileStorage description). In case of JSON it is written as a name/value pair. + Mainly it is used with user objects. When the storage is read, the encoded type name is used to determine the object type (see CvTypeInfo and cvFindType ). @param attributes This parameter is not used in the current implementation */ @@ -2499,7 +2500,7 @@ CVAPI(void) cvReadRawData( const CvFileStorage* fs, const CvFileNode* src, /** @brief Writes a file node to another file storage. The function writes a copy of a file node to file storage. Possible applications of the function are -merging several file storages into one and conversion between XML and YAML formats. +merging several file storages into one and conversion between XML, YAML and JSON formats. @param fs Destination file storage @param new_node_name New name of the file node in the destination file storage. To keep the existing name, use cvcvGetFileNodeName diff --git a/modules/core/include/opencv2/core/persistence.hpp b/modules/core/include/opencv2/core/persistence.hpp index 75f0c32bd275f85e073da7e33a2073b9b6e713a6..01f28d57a3ded1c935be0018c5ef5b5107a078ed 100644 --- a/modules/core/include/opencv2/core/persistence.hpp +++ b/modules/core/include/opencv2/core/persistence.hpp @@ -57,8 +57,9 @@ Several functions that are described below take CvFileStorage\* as inputs and al save or to load hierarchical collections that consist of scalar values, standard CXCore objects (such as matrices, sequences, graphs), and user-defined objects. -OpenCV can read and write data in XML () or YAML () -formats. Below is an example of 3x3 floating-point identity matrix A, stored in XML and YAML files +OpenCV can read and write data in XML (), YAML () or +JSON () formats. Below is an example of 3x3 floating-point identity matrix A, +stored in XML and YAML files using CXCore functions: XML: @code{.xml} @@ -85,7 +86,8 @@ As it can be seen from the examples, XML uses nested tags to represent hierarchy indentation for that purpose (similar to the Python programming language). The same functions can read and write data in both formats; the particular format is determined by -the extension of the opened file, ".xml" for XML files and ".yml" or ".yaml" for YAML. +the extension of the opened file, ".xml" for XML files, ".yml" or ".yaml" for YAML and ".json" for +JSON. */ typedef struct CvFileStorage CvFileStorage; typedef struct CvFileNode CvFileNode; @@ -101,20 +103,20 @@ namespace cv { /** @addtogroup core_xml -XML/YAML file storages. {#xml_storage} +XML/YAML/JSON file storages. {#xml_storage} ======================= Writing to a file storage. -------------------------- -You can store and then restore various OpenCV data structures to/from XML () -or YAML () formats. Also, it is possible store and load arbitrarily complex -data structures, which include OpenCV data structures, as well as primitive data types (integer and -floating-point numbers and text strings) as their elements. +You can store and then restore various OpenCV data structures to/from XML (), +YAML () or JSON () formats. Also, it is possible store +and load arbitrarily complex data structures, which include OpenCV data structures, as well as +primitive data types (integer and floating-point numbers and text strings) as their elements. -Use the following procedure to write something to XML or YAML: +Use the following procedure to write something to XML, YAML or JSON: -# Create new FileStorage and open it for writing. It can be done with a single call to FileStorage::FileStorage constructor that takes a filename, or you can use the default constructor -and then call FileStorage::open. Format of the file (XML or YAML) is determined from the filename -extension (".xml" and ".yml"/".yaml", respectively) +and then call FileStorage::open. Format of the file (XML, YAML or JSON) is determined from the filename +extension (".xml", ".yml"/".yaml" and ".json", respectively) -# Write all the data you want using the streaming operator `<<`, just like in the case of STL streams. -# Close the file using FileStorage::release. FileStorage destructor also closes the file. @@ -177,19 +179,19 @@ features: - { x:344, y:158, lbp:[ 1, 1, 0, 0, 0, 0, 1, 0 ] } @endcode -As an exercise, you can replace ".yml" with ".xml" in the sample above and see, how the +As an exercise, you can replace ".yml" with ".xml" or ".json" in the sample above and see, how the corresponding XML file will look like. Several things can be noted by looking at the sample code and the output: -- The produced YAML (and XML) consists of heterogeneous collections that can be nested. There are 2 - types of collections: named collections (mappings) and unnamed collections (sequences). In mappings +- The produced YAML (and XML/JSON) consists of heterogeneous collections that can be nested. There are + 2 types of collections: named collections (mappings) and unnamed collections (sequences). In mappings each element has a name and is accessed by name. This is similar to structures and std::map in C/C++ and dictionaries in Python. In sequences elements do not have names, they are accessed by indices. This is similar to arrays and std::vector in C/C++ and lists, tuples in Python. "Heterogeneous" means that elements of each single collection can have different types. - Top-level collection in YAML/XML is a mapping. Each matrix is stored as a mapping, and the matrix + Top-level collection in YAML/XML/JSON is a mapping. Each matrix is stored as a mapping, and the matrix elements are stored as a sequence. Then, there is a sequence of features, where each feature is represented a mapping, and lbp value in a nested sequence. @@ -205,7 +207,7 @@ Several things can be noted by looking at the sample code and the output: - To write a sequence, you first write the special string `[`, then write the elements, then write the closing `]`. -- In YAML (but not XML), mappings and sequences can be written in a compact Python-like inline +- In YAML/JSON (but not XML), mappings and sequences can be written in a compact Python-like inline form. In the sample above matrix elements, as well as each feature, including its lbp value, is stored in such inline form. To store a mapping/sequence in a compact form, put `:` after the opening character, e.g. use `{:` instead of `{` and `[:` instead of `[`. When the @@ -213,7 +215,7 @@ Several things can be noted by looking at the sample code and the output: Reading data from a file storage. --------------------------------- -To read the previously written XML or YAML file, do the following: +To read the previously written XML, YAML or JSON file, do the following: -# Open the file storage using FileStorage::FileStorage constructor or FileStorage::open method. In the current implementation the whole file is parsed and the whole representation of file storage is built in memory as a hierarchy of file nodes (see FileNode) @@ -294,8 +296,8 @@ A complete example using the FileStorage interface class CV_EXPORTS FileNode; class CV_EXPORTS FileNodeIterator; -/** @brief XML/YAML file storage class that encapsulates all the information necessary for writing or reading -data to/from a file. +/** @brief XML/YAML/JSON file storage class that encapsulates all the information necessary for writing or +reading data to/from a file. */ class CV_EXPORTS_W FileStorage { @@ -312,6 +314,7 @@ public: FORMAT_AUTO = 0, //!< flag, auto format FORMAT_XML = (1<<3), //!< flag, XML format FORMAT_YAML = (2<<3), //!< flag, YAML format + FORMAT_JSON = (3<<3), //!< flag, JSON format BASE64 = 64, //!< flag, write rawdata in Base64 by default. (consider using WRITE_BASE64) WRITE_BASE64 = BASE64 | WRITE, //!< flag, enable both WRITE and BASE64 @@ -333,9 +336,9 @@ public: /** @overload @param source Name of the file to open or the text string to read the data from. Extension of the - file (.xml or .yml/.yaml) determines its format (XML or YAML respectively). Also you can append .gz - to work with compressed files, for example myHugeMatrix.xml.gz. If both FileStorage::WRITE and - FileStorage::MEMORY flags are specified, source is used just to specify the output file format (e.g. + file (.xml, .yml/.yaml, or .json) determines its format (XML, YAML or JSON respectively). Also you can + append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both FileStorage::WRITE + and FileStorage::MEMORY flags are specified, source is used just to specify the output file format (e.g. mydata.xml, .yml etc.). @param flags Mode of operation. See FileStorage::Mode @param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and @@ -354,12 +357,12 @@ public: See description of parameters in FileStorage::FileStorage. The method calls FileStorage::release before opening the file. @param filename Name of the file to open or the text string to read the data from. - Extension of the file (.xml or .yml/.yaml) determines its format (XML or YAML respectively). - Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both + Extension of the file (.xml, .yml/.yaml or .json) determines its format (XML, YAML or JSON + respectively). Also you can append .gz to work with compressed files, for example myHugeMatrix.xml.gz. If both FileStorage::WRITE and FileStorage::MEMORY flags are specified, source is used just to specify the output file format (e.g. mydata.xml, .yml etc.). A file name can also contain parameters. - You can use this format, "*?base64" (e.g. "file.xml?base64"), as an alternative to - FileStorage::BASE64 flag. Note: it is case sensitive. + You can use this format, "*?base64" (e.g. "file.json?base64" (case sensitive)), as an alternative to + FileStorage::BASE64 flag. @param flags Mode of operation. One of FileStorage::Mode @param encoding Encoding of the file. Note that UTF-16 XML encoding is not supported currently and you should use 8-bit encoding instead of it. diff --git a/modules/core/include/opencv2/core/types_c.h b/modules/core/include/opencv2/core/types_c.h index e693aa47247da7dc23d52a936f101d2aa2a8dc05..a25a565df67a196bc909946bd5ff935be69ebfa3 100644 --- a/modules/core/include/opencv2/core/types_c.h +++ b/modules/core/include/opencv2/core/types_c.h @@ -1669,6 +1669,7 @@ typedef struct CvFileStorage CvFileStorage; #define CV_STORAGE_FORMAT_AUTO 0 #define CV_STORAGE_FORMAT_XML 8 #define CV_STORAGE_FORMAT_YAML 16 +#define CV_STORAGE_FORMAT_JSON 24 #define CV_STORAGE_BASE64 64 #define CV_STORAGE_WRITE_BASE64 (CV_STORAGE_BASE64 | CV_STORAGE_WRITE) diff --git a/modules/core/perf/perf_io_base64.cpp b/modules/core/perf/perf_io_base64.cpp index 04a4a040a578119325a0114ae60bf53eb68bc196..799c52c394af4309383167c89c865b49cd0c7510 100644 --- a/modules/core/perf/perf_io_base64.cpp +++ b/modules/core/perf/perf_io_base64.cpp @@ -11,7 +11,7 @@ typedef TestBaseWithParam Size_Mat_StrType; #define MAT_SIZES ::perf::sz1080p/*, ::perf::sz4320p*/ #define MAT_TYPES CV_8UC1, CV_32FC1 -#define FILE_EXTENSION String(".xml"), String(".yml") +#define FILE_EXTENSION String(".xml"), String(".yml"), String(".json") PERF_TEST_P(Size_Mat_StrType, fs_text, diff --git a/modules/core/src/persistence.cpp b/modules/core/src/persistence.cpp index 6f9f99298349f72f3c662fc9d915eb3a6e4da5f7..4db6dc4eb9bdc80af2bc58228f246170d2af2222 100644 --- a/modules/core/src/persistence.cpp +++ b/modules/core/src/persistence.cpp @@ -117,6 +117,25 @@ static char* icv_itoa( int _val, char* buffer, int /*radix*/ ) return ptr; } +static inline bool cv_strcasecmp(const char * s1, const char * s2) +{ + if ( s1 == 0 && s2 == 0 ) + return true; + else if ( s1 == 0 || s2 == 0 ) + return false; + + size_t len1 = strlen(s1); + size_t len2 = strlen(s2); + if ( len1 != len2 ) + return false; + + for ( size_t i = 0U; i < len1; i++ ) + if ( tolower( static_cast(s1[i]) ) != tolower( static_cast(s2[i]) ) ) + return false; + + return true; +} + cv::String cv::FileStorage::getDefaultObjectName(const cv::String& _filename) { static const char* stubname = "unnamed"; @@ -621,8 +640,7 @@ icvFSFlush( CvFileStorage* fs ) if( fs->space != indent ) { - if( fs->space < indent ) - memset( fs->buffer_start + fs->space, ' ', indent - fs->space ); + memset( fs->buffer_start, ' ', indent ); fs->space = indent; } @@ -653,6 +671,8 @@ icvClose( CvFileStorage* fs, cv::String* out ) icvFSFlush(fs); if( fs->fmt == CV_STORAGE_FORMAT_XML ) icvPuts( fs, "\n" ); + else if ( fs->fmt == CV_STORAGE_FORMAT_JSON ) + icvPuts( fs, "}\n" ); } icvCloseFile(fs); @@ -1064,6 +1084,7 @@ static double icv_strtod( CvFileStorage* fs, char* ptr, char** endptr ) return fval; } +// this function will convert "aa?bb&cc&dd" to {"aa", "bb", "cc", "dd"} static std::vector analyze_file_name( std::string const & file_name ) { static const char not_file_name = '\n'; @@ -2991,74 +3012,1009 @@ icvXMLStartNextStream( CvFileStorage* fs ) } -static void -icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len ) -{ - check_if_write_struct_is_delayed( fs ); - if ( fs->state_of_writing_base64 == base64::fs::Uncertain ) +static void +icvXMLWriteScalar( CvFileStorage* fs, const char* key, const char* data, int len ) +{ + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == base64::fs::Uncertain ) + { + switch_to_Base64_state( fs, base64::fs::NotUse ); + } + else if ( fs->state_of_writing_base64 == base64::fs::InUse ) + { + CV_Error( CV_StsError, "Currently only Base64 data is allowed." ); + } + + if( CV_NODE_IS_MAP(fs->struct_flags) || + (!CV_NODE_IS_COLLECTION(fs->struct_flags) && key) ) + { + icvXMLWriteTag( fs, key, CV_XML_OPENING_TAG, cvAttrList(0,0) ); + char* ptr = icvFSResizeWriteBuffer( fs, fs->buffer, len ); + memcpy( ptr, data, len ); + fs->buffer = ptr + len; + icvXMLWriteTag( fs, key, CV_XML_CLOSING_TAG, cvAttrList(0,0) ); + } + else + { + char* ptr = fs->buffer; + int new_offset = (int)(ptr - fs->buffer_start) + len; + + if( key ) + CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" ); + + fs->struct_flags = CV_NODE_SEQ; + + if( (new_offset > fs->wrap_margin && new_offset - fs->struct_indent > 10) || + (ptr > fs->buffer_start && ptr[-1] == '>' && !CV_NODE_IS_EMPTY(fs->struct_flags)) ) + { + ptr = icvXMLFlush(fs); + } + else if( ptr > fs->buffer_start + fs->struct_indent && ptr[-1] != '>' ) + *ptr++ = ' '; + + memcpy( ptr, data, len ); + fs->buffer = ptr + len; + } +} + + +static void +icvXMLWriteInt( CvFileStorage* fs, const char* key, int value ) +{ + char buf[128], *ptr = icv_itoa( value, buf, 10 ); + int len = (int)strlen(ptr); + icvXMLWriteScalar( fs, key, ptr, len ); +} + + +static void +icvXMLWriteReal( CvFileStorage* fs, const char* key, double value ) +{ + char buf[128]; + int len = (int)strlen( icvDoubleToString( buf, value )); + icvXMLWriteScalar( fs, key, buf, len ); +} + + +static void +icvXMLWriteString( CvFileStorage* fs, const char* key, const char* str, int quote ) +{ + char buf[CV_FS_MAX_LEN*6+16]; + char* data = (char*)str; + int i, len; + + if( !str ) + CV_Error( CV_StsNullPtr, "Null string pointer" ); + + len = (int)strlen(str); + if( len > CV_FS_MAX_LEN ) + CV_Error( CV_StsBadArg, "The written string is too long" ); + + if( quote || len == 0 || str[0] != '\"' || str[0] != str[len-1] ) + { + int need_quote = quote || len == 0; + data = buf; + *data++ = '\"'; + for( i = 0; i < len; i++ ) + { + char c = str[i]; + + if( (uchar)c >= 128 || c == ' ' ) + { + *data++ = c; + need_quote = 1; + } + else if( !cv_isprint(c) || c == '<' || c == '>' || c == '&' || c == '\'' || c == '\"' ) + { + *data++ = '&'; + if( c == '<' ) + { + memcpy(data, "lt", 2); + data += 2; + } + else if( c == '>' ) + { + memcpy(data, "gt", 2); + data += 2; + } + else if( c == '&' ) + { + memcpy(data, "amp", 3); + data += 3; + } + else if( c == '\'' ) + { + memcpy(data, "apos", 4); + data += 4; + } + else if( c == '\"' ) + { + memcpy( data, "quot", 4); + data += 4; + } + else + { + sprintf( data, "#x%02x", (uchar)c ); + data += 4; + } + *data++ = ';'; + need_quote = 1; + } + else + *data++ = c; + } + if( !need_quote && (cv_isdigit(str[0]) || + str[0] == '+' || str[0] == '-' || str[0] == '.' )) + need_quote = 1; + + if( need_quote ) + *data++ = '\"'; + len = (int)(data - buf) - !need_quote; + *data++ = '\0'; + data = buf + !need_quote; + } + + icvXMLWriteScalar( fs, key, data, len ); +} + + +static void +icvXMLWriteComment( CvFileStorage* fs, const char* comment, int eol_comment ) +{ + int len; + int multiline; + const char* eol; + char* ptr; + + if( !comment ) + CV_Error( CV_StsNullPtr, "Null comment" ); + + if( strstr(comment, "--") != 0 ) + CV_Error( CV_StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); + + len = (int)strlen(comment); + eol = strchr(comment, '\n'); + multiline = eol != 0; + ptr = fs->buffer; + + if( multiline || !eol_comment || fs->buffer_end - ptr < len + 5 ) + ptr = icvXMLFlush( fs ); + else if( ptr > fs->buffer_start + fs->struct_indent ) + *ptr++ = ' '; + + if( !multiline ) + { + ptr = icvFSResizeWriteBuffer( fs, ptr, len + 9 ); + sprintf( ptr, "", comment ); + len = (int)strlen(ptr); + } + else + { + strcpy( ptr, "" ); + fs->buffer = ptr + 3; + icvXMLFlush( fs ); + } +} + + +/****************************************************************************************\ +* JSON Parser * +\****************************************************************************************/ + +static char* +icvJSONSkipSpaces( CvFileStorage* fs, char* ptr ) +{ + bool is_eof = false; + bool is_completed = false; + + while ( is_eof == false && is_completed == false ) + { + switch ( *ptr ) + { + /* comment */ + case '/' : { + ptr++; + if ( *ptr == '\0' ) + { + ptr = icvGets( fs, fs->buffer_start, static_cast(fs->buffer_end - fs->buffer_start) ); + if ( !ptr ) { is_eof = true; break; } + } + + if ( *ptr == '/' ) + { + while ( *ptr != '\n' && *ptr != '\r' ) + { + if ( *ptr == '\0' ) + { + ptr = icvGets( fs, fs->buffer_start, static_cast(fs->buffer_end - fs->buffer_start) ); + if ( !ptr ) { is_eof = true; break; } + } + else + { + ptr++; + } + } + } + else if ( *ptr == '*' ) + { + ptr++; + while ( true ) + { + if ( *ptr == '\0' ) + { + ptr = icvGets( fs, fs->buffer_start, static_cast(fs->buffer_end - fs->buffer_start) ); + if ( !ptr ) { is_eof = true; break; } + } + else if ( *ptr == '*' ) + { + ptr++; + if ( *ptr == '\0' ) + { + ptr = icvGets( fs, fs->buffer_start, static_cast(fs->buffer_end - fs->buffer_start) ); + if ( !ptr ) { is_eof = true; break; } + } + if ( *ptr == '/' ) + break; + } + else + { + ptr++; + } + } + } + else + { + CV_PARSE_ERROR( "Unexpected character" ); + } + } break; + /* whitespace */ + case '\t': + case ' ' : { + ptr++; + } break; + /* newline || end mark */ + case '\0': + case '\n': + case '\r': { + ptr = icvGets( fs, fs->buffer_start, static_cast(fs->buffer_end - fs->buffer_start) ); + if ( !ptr ) { is_eof = true; break; } + } break; + /* other character */ + default: { + if ( !cv_isprint(*ptr) ) + CV_PARSE_ERROR( "Invalid character in the stream" ); + is_completed = true; + } break; + } + } + + if ( is_eof ) + { + ptr = fs->buffer_start; + *ptr = '\0'; + fs->dummy_eof = 1; + } + else if ( !is_completed ) + { + /* should not be executed */ + ptr = 0; + fs->dummy_eof = 1; + CV_PARSE_ERROR( "Abort at parse time" ); + } + return ptr; +} + + +static char* icvJSONParseKey( CvFileStorage* fs, char* ptr, CvFileNode* map, CvFileNode** value_placeholder ) +{ + if( *ptr != '"' ) + CV_PARSE_ERROR( "Key must start with \'\"\'" ); + + char * beg = ptr + 1; + char * end = beg; + + do ++ptr; + while( cv_isprint(*ptr) && *ptr != '"' ); + + if( *ptr != '"' ) + CV_PARSE_ERROR( "Key must end with \'\"\'" ); + + end = ptr; + ptr++; + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + return 0; + + if( *ptr != ':' ) + CV_PARSE_ERROR( "Missing \':\'" ); + + /* [beg, end) */ + if( end <= beg ) + CV_PARSE_ERROR( "An empty key" ); + + if ( end - beg == 7u && memcmp(beg, "type_id", 7u) == 0 ) + { + *value_placeholder = 0; + } + else + { + CvStringHashNode* str_hash_node = cvGetHashedKey( fs, beg, static_cast(end - beg), 1 ); + *value_placeholder = cvGetFileNode( fs, map, str_hash_node, 1 ); + } + + ptr++; + return ptr; +} + +static char* icvJSONParseValue( CvFileStorage* fs, char* ptr, CvFileNode* node ) +{ + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + CV_PARSE_ERROR( "Unexpected End-Of-File" ); + + memset( node, 0, sizeof(*node) ); + + if ( *ptr == '"' ) + { /* must be string or Base64 string */ + ptr++; + char * beg = ptr; + size_t len = 0u; + for ( ; (cv_isalnum(*ptr) || *ptr == '$' ) && len <= 9u; ptr++ ) + len++; + + if ( len >= 8u && memcmp( beg, "$base64$", 8u ) == 0 ) + { /**************** Base64 string ****************/ + ptr = beg += 8; + + std::string base64_buffer; + base64_buffer.reserve( PARSER_BASE64_BUFFER_SIZE ); + + bool is_matching = false; + while ( !is_matching ) + { + switch ( *ptr ) + { + case '\0': + { + base64_buffer.append( beg, ptr ); + + ptr = icvGets( fs, fs->buffer_start, static_cast(fs->buffer_end - fs->buffer_start) ); + if ( !ptr ) + CV_PARSE_ERROR( "'\"' - right-quote of string is missing" ); + + beg = ptr; + break; + } + case '\"': + { + base64_buffer.append( beg, ptr ); + beg = ptr; + is_matching = true; + break; + } + case '\n': + case '\r': + { + CV_PARSE_ERROR( "'\"' - right-quote of string is missing" ); + break; + } + default: + { + ptr++; + break; + } + } + } + + if ( *ptr != '\"' ) + CV_PARSE_ERROR( "'\"' - right-quote of string is missing" ); + else + ptr++; + + if ( base64_buffer.size() >= base64::ENCODED_HEADER_SIZE ) + { + const char * base64_beg = base64_buffer.data(); + const char * base64_end = base64_beg + base64_buffer.size(); + + /* get dt from header */ + std::string dt; + { + std::vector header(base64::HEADER_SIZE + 1, ' '); + base64::base64_decode(base64_beg, header.data(), 0U, base64::ENCODED_HEADER_SIZE); + if ( !base64::read_base64_header(header, dt) || dt.empty() ) + CV_PARSE_ERROR("Cannot parse dt in Base64 header"); + } + + /* set base64_beg to beginning of base64 data */ + base64_beg = &base64_buffer.at( base64::ENCODED_HEADER_SIZE ); + + if ( base64_buffer.size() > base64::ENCODED_HEADER_SIZE ) + { + if ( !base64::base64_valid( base64_beg, 0U, base64_end - base64_beg ) ) + CV_PARSE_ERROR( "Invalid Base64 data." ); + + /* buffer for decoded data(exclude header) */ + std::vector binary_buffer( base64::base64_decode_buffer_size(base64_end - base64_beg) ); + int total_byte_size = static_cast( + base64::base64_decode_buffer_size( base64_end - base64_beg, base64_beg, false ) + ); + { + base64::Base64ContextParser parser(binary_buffer.data(), binary_buffer.size() ); + const uchar * binary_beg = reinterpret_cast( base64_beg ); + const uchar * binary_end = binary_beg + (base64_end - base64_beg); + parser.read( binary_beg, binary_end ); + parser.flush(); + } + + /* save as CvSeq */ + int elem_size = ::icvCalcStructSize(dt.c_str(), 0); + if (total_byte_size % elem_size != 0) + CV_PARSE_ERROR("Byte size not match elememt size"); + int elem_cnt = total_byte_size / elem_size; + + /* after icvFSCreateCollection, node->tag == struct_flags */ + icvFSCreateCollection(fs, CV_NODE_FLOW | CV_NODE_SEQ, node); + base64::make_seq(binary_buffer.data(), elem_cnt, dt.c_str(), *node->data.seq); + } + else + { + /* empty */ + icvFSCreateCollection(fs, CV_NODE_FLOW | CV_NODE_SEQ, node); + } + } + else if ( base64_buffer.empty() ) + { + /* empty */ + icvFSCreateCollection(fs, CV_NODE_FLOW | CV_NODE_SEQ, node); + } + else + { + CV_PARSE_ERROR("Unrecognized Base64 header"); + } + } + else + { /**************** normal string ****************/ + std::string string_buffer; + string_buffer.reserve( PARSER_BASE64_BUFFER_SIZE ); + + ptr = beg; + bool is_matching = false; + while ( !is_matching ) + { + switch ( *ptr ) + { + case '\\': + { + string_buffer.append( beg, ptr ); + ptr++; + switch ( *ptr ) + { + case '\\': + case '\"': + case '\'': { string_buffer.append( 1u, *ptr ); break; } + case 'n' : { string_buffer.append( 1u, '\n' ); break; } + case 'r' : { string_buffer.append( 1u, '\r' ); break; } + case 't' : { string_buffer.append( 1u, '\t' ); break; } + case 'b' : { string_buffer.append( 1u, '\b' ); break; } + case 'f' : { string_buffer.append( 1u, '\f' ); break; } + case 'u' : { CV_PARSE_ERROR( "'\\uXXXX' currently not supported" ); } + default : { CV_PARSE_ERROR( "Invalid escape character" ); } + break; + } + ptr++; + beg = ptr; + break; + } + case '\0': + { + string_buffer.append( beg, ptr ); + + ptr = icvGets( fs, fs->buffer_start, static_cast(fs->buffer_end - fs->buffer_start) ); + if ( !ptr ) + CV_PARSE_ERROR( "'\"' - right-quote of string is missing" ); + + beg = ptr; + break; + } + case '\"': + { + string_buffer.append( beg, ptr ); + beg = ptr; + is_matching = true; + break; + } + case '\n': + case '\r': + { + CV_PARSE_ERROR( "'\"' - right-quote of string is missing" ); + break; + } + default: + { + ptr++; + break; + } + } + } + + if ( *ptr != '\"' ) + CV_PARSE_ERROR( "'\"' - right-quote of string is missing" ); + else + ptr++; + + node->data.str = cvMemStorageAllocString( fs->memstorage, string_buffer.c_str(), string_buffer.size() ); + node->tag = CV_NODE_STRING; + } + } + else if ( cv_isdigit(*ptr) || *ptr == '-' || *ptr == '+' || *ptr == '.' ) + { /**************** number ****************/ + char * beg = ptr; + if ( *ptr == '+' || *ptr == '-' ) + ptr++; + while( cv_isdigit(*ptr) ) + ptr++; + if (*ptr == '.' || *ptr == 'e') + { + node->data.f = icv_strtod( fs, beg, &ptr ); + node->tag = CV_NODE_REAL; + } + else + { + node->data.i = static_cast(strtol( beg, &ptr, 0 )); + node->tag = CV_NODE_INT; + } + + if ( beg >= ptr ) + CV_PARSE_ERROR( "Invalid numeric value (inconsistent explicit type specification?)" ); + } + else + { /**************** other data ****************/ + const char * beg = ptr; + size_t len = 0u; + for ( ; cv_isalpha(*ptr) && len <= 6u; ptr++ ) + len++; + + if ( len >= 4u && memcmp( beg, "null", 4u ) == 0 ) + { + CV_PARSE_ERROR( "Value 'null' is not supported by this parser" ); + } + else if ( len >= 4u && memcmp( beg, "true", 4u ) == 0 ) + { + node->data.i = 1; + node->tag = CV_NODE_INT; + } + else if ( len >= 5u && memcmp( beg, "false", 5u ) == 0 ) + { + node->data.i = 0; + node->tag = CV_NODE_INT; + } + else + { + CV_PARSE_ERROR( "Unrecognized value" ); + } + ptr++; + } + + return ptr; +} + +static char* icvJSONParseSeq( CvFileStorage* fs, char* ptr, CvFileNode* node ); +static char* icvJSONParseMap( CvFileStorage* fs, char* ptr, CvFileNode* node ); + +static char* icvJSONParseSeq( CvFileStorage* fs, char* ptr, CvFileNode* node ) +{ + if ( *ptr != '[' ) + CV_PARSE_ERROR( "'[' - left-brace of seq is missing" ); + else + ptr++; + + memset( node, 0, sizeof(*node) ); + icvFSCreateCollection( fs, CV_NODE_SEQ, node ); + + while ( true ) + { + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + break; + + if ( *ptr != ']' ) + { + CvFileNode* child = (CvFileNode*)cvSeqPush( node->data.seq, 0 ); + + if ( *ptr == '[' ) + ptr = icvJSONParseSeq( fs, ptr, child ); + else if ( *ptr == '{' ) + ptr = icvJSONParseMap( fs, ptr, child ); + else + ptr = icvJSONParseValue( fs, ptr, child ); + } + + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + break; + + if ( *ptr == ',' ) + ptr++; + else if ( *ptr == ']' ) + break; + else + CV_PARSE_ERROR( "Unexpected character" ); + } + + if ( *ptr != ']' ) + CV_PARSE_ERROR( "']' - right-brace of seq is missing" ); + else + ptr++; + + return ptr; +} + +static char* icvJSONParseMap( CvFileStorage* fs, char* ptr, CvFileNode* node ) +{ + if ( *ptr != '{' ) + CV_PARSE_ERROR( "'{' - left-brace of map is missing" ); + else + ptr++; + + memset( node, 0, sizeof(*node) ); + icvFSCreateCollection( fs, CV_NODE_MAP, node ); + + while ( true ) + { + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + break; + + if ( *ptr == '"' ) + { + CvFileNode* child = 0; + ptr = icvJSONParseKey( fs, ptr, node, &child ); + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + break; + + if ( child == 0 ) + { /* type_id */ + CvFileNode tmp; + ptr = icvJSONParseValue( fs, ptr, &tmp ); + if ( CV_NODE_IS_STRING(tmp.tag) ) + { + node->info = cvFindType( tmp.data.str.ptr ); + if ( node->info ) + node->tag |= CV_NODE_USER; + // delete tmp.data.str + } + else + { + CV_PARSE_ERROR( "\"type_id\" should be of type string" ); + } + } + else + { /* normal */ + if ( *ptr == '[' ) + ptr = icvJSONParseSeq( fs, ptr, child ); + else if ( *ptr == '{' ) + ptr = icvJSONParseMap( fs, ptr, child ); + else + ptr = icvJSONParseValue( fs, ptr, child ); + } + } + + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + break; + + if ( *ptr == ',' ) + ptr++; + else if ( *ptr == '}' ) + break; + else + CV_PARSE_ERROR( "Unexpected character" ); + } + + if ( *ptr != '}' ) + CV_PARSE_ERROR( "'}' - right-brace of map is missing" ); + else + ptr++; + + return ptr; +} + + +static void +icvJSONParse( CvFileStorage* fs ) +{ + char* ptr = fs->buffer_start; + ptr = icvJSONSkipSpaces( fs, ptr ); + if ( ptr == 0 || fs->dummy_eof ) + return; + + if ( *ptr == '{' ) + { + CvFileNode* root_node = (CvFileNode*)cvSeqPush( fs->roots, 0 ); + ptr = icvJSONParseMap( fs, ptr, root_node ); + } + else if ( *ptr == '[' ) + { + CvFileNode* root_node = (CvFileNode*)cvSeqPush( fs->roots, 0 ); + ptr = icvJSONParseSeq( fs, ptr, root_node ); + } + else + { + CV_PARSE_ERROR( "left-brace of top level is missing" ); + } + + if ( fs->dummy_eof != 0 ) + CV_PARSE_ERROR( "Unexpected End-Of-File" ); +} + + +/****************************************************************************************\ +* JSON Emitter * +\****************************************************************************************/ + +static void +icvJSONWrite( CvFileStorage* fs, const char* key, const char* data ) +{ + /* check write_struct */ + + check_if_write_struct_is_delayed( fs ); + if ( fs->state_of_writing_base64 == base64::fs::Uncertain ) + { + switch_to_Base64_state( fs, base64::fs::NotUse ); + } + else if ( fs->state_of_writing_base64 == base64::fs::InUse ) + { + CV_Error( CV_StsError, "At present, output Base64 data only." ); + } + + /* check parameters */ + + size_t key_len = 0u; + if( key && *key == '\0' ) + key = 0; + if ( key ) + { + key_len = strlen(key); + if ( key_len == 0u ) + CV_Error( CV_StsBadArg, "The key is an empty" ); + else if ( static_cast(key_len) > CV_FS_MAX_LEN ) + CV_Error( CV_StsBadArg, "The key is too long" ); + } + + size_t data_len = 0u; + if ( data ) + data_len = strlen(data); + + int struct_flags = fs->struct_flags; + if( CV_NODE_IS_COLLECTION(struct_flags) ) + { + if ( (CV_NODE_IS_MAP(struct_flags) ^ (key != 0)) ) + CV_Error( CV_StsBadArg, "An attempt to add element without a key to a map, " + "or add element with key to sequence" ); + } else { + fs->is_first = 0; + struct_flags = CV_NODE_EMPTY | (key ? CV_NODE_MAP : CV_NODE_SEQ); + } + + /* start to write */ + + char* ptr = 0; + + if( CV_NODE_IS_FLOW(struct_flags) ) + { + int new_offset; + ptr = fs->buffer; + if( !CV_NODE_IS_EMPTY(struct_flags) ) + *ptr++ = ','; + new_offset = static_cast(ptr - fs->buffer_start + key_len + data_len); + if( new_offset > fs->wrap_margin && new_offset - fs->struct_indent > 10 ) + { + fs->buffer = ptr; + ptr = icvFSFlush(fs); + } + else + *ptr++ = ' '; + } + else + { + if ( !CV_NODE_IS_EMPTY(struct_flags) ) + { + ptr = fs->buffer; + *ptr++ = ','; + *ptr++ = '\n'; + *ptr++ = '\0'; + ::icvPuts( fs, fs->buffer_start ); + ptr = fs->buffer = fs->buffer_start; + } + ptr = icvFSFlush(fs); + } + + if( key ) + { + if( !cv_isalpha(key[0]) && key[0] != '_' ) + CV_Error( CV_StsBadArg, "Key must start with a letter or _" ); + + ptr = icvFSResizeWriteBuffer( fs, ptr, static_cast(key_len) ); + *ptr++ = '\"'; + + for( size_t i = 0u; i < key_len; i++ ) + { + char c = key[i]; + + ptr[i] = c; + if( !cv_isalnum(c) && c != '-' && c != '_' && c != ' ' ) + CV_Error( CV_StsBadArg, "Key names may only contain alphanumeric characters [a-zA-Z0-9], '-', '_' and ' '" ); + } + + ptr += key_len; + *ptr++ = '\"'; + *ptr++ = ':'; + *ptr++ = ' '; + } + + if( data ) + { + ptr = icvFSResizeWriteBuffer( fs, ptr, static_cast(data_len) ); + memcpy( ptr, data, data_len ); + ptr += data_len; + } + + fs->buffer = ptr; + fs->struct_flags = struct_flags & ~CV_NODE_EMPTY; +} + + +static void +icvJSONStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, + const char* type_name CV_DEFAULT(0)) +{ + int parent_flags; + char data[CV_FS_MAX_LEN + 1024]; + + struct_flags = (struct_flags & (CV_NODE_TYPE_MASK|CV_NODE_FLOW)) | CV_NODE_EMPTY; + if( !CV_NODE_IS_COLLECTION(struct_flags)) + CV_Error( CV_StsBadArg, + "Some collection type - CV_NODE_SEQ or CV_NODE_MAP, must be specified" ); + + if ( type_name && *type_name == '\0' ) + type_name = 0; + + bool has_type_id = false; + bool is_real_collection = true; + if (type_name && memcmp(type_name, "binary", 6) == 0) { - switch_to_Base64_state( fs, base64::fs::NotUse ); + struct_flags = CV_NODE_STR; + data[0] = '\0'; + is_real_collection = false; } - else if ( fs->state_of_writing_base64 == base64::fs::InUse ) + else if( type_name ) { - CV_Error( CV_StsError, "Currently only Base64 data is allowed." ); + has_type_id = true; } - if( CV_NODE_IS_MAP(fs->struct_flags) || - (!CV_NODE_IS_COLLECTION(fs->struct_flags) && key) ) + if ( is_real_collection ) { - icvXMLWriteTag( fs, key, CV_XML_OPENING_TAG, cvAttrList(0,0) ); - char* ptr = icvFSResizeWriteBuffer( fs, fs->buffer, len ); - memcpy( ptr, data, len ); - fs->buffer = ptr + len; - icvXMLWriteTag( fs, key, CV_XML_CLOSING_TAG, cvAttrList(0,0) ); + char c = CV_NODE_IS_MAP(struct_flags) ? '{' : '['; + data[0] = c; + data[1] = '\0'; } - else - { - char* ptr = fs->buffer; - int new_offset = (int)(ptr - fs->buffer_start) + len; - if( key ) - CV_Error( CV_StsBadArg, "elements with keys can not be written to sequence" ); + icvJSONWrite( fs, key, data ); - fs->struct_flags = CV_NODE_SEQ; + parent_flags = fs->struct_flags; + cvSeqPush( fs->write_stack, &parent_flags ); + fs->struct_flags = struct_flags; + fs->struct_indent += 4; - if( (new_offset > fs->wrap_margin && new_offset - fs->struct_indent > 10) || - (ptr > fs->buffer_start && ptr[-1] == '>' && !CV_NODE_IS_EMPTY(fs->struct_flags)) ) + if ( has_type_id ) + fs->write_string( fs, "type_id", type_name, 1 ); +} + + +static void +icvJSONEndWriteStruct( CvFileStorage* fs ) +{ + if( fs->write_stack->total == 0 ) + CV_Error( CV_StsError, "EndWriteStruct w/o matching StartWriteStruct" ); + + int parent_flags = 0; + int struct_flags = fs->struct_flags; + cvSeqPop( fs->write_stack, &parent_flags ); + fs->struct_indent -= 4; + fs->struct_flags = parent_flags & ~CV_NODE_EMPTY; + assert( fs->struct_indent >= 0 ); + + if ( CV_NODE_IS_COLLECTION(struct_flags) ) + { + if ( !CV_NODE_IS_FLOW(struct_flags) ) { - ptr = icvXMLFlush(fs); + if ( fs->buffer <= fs->buffer_start + fs->space ) + { + /* some bad code for base64_writer... */ + *fs->buffer++ = '\n'; + *fs->buffer++ = '\0'; + icvPuts( fs, fs->buffer_start ); + fs->buffer = fs->buffer_start; + } + icvFSFlush(fs); } - else if( ptr > fs->buffer_start + fs->struct_indent && ptr[-1] != '>' ) + + char* ptr = fs->buffer; + if( ptr > fs->buffer_start + fs->struct_indent && !CV_NODE_IS_EMPTY(struct_flags) ) *ptr++ = ' '; + *ptr++ = CV_NODE_IS_MAP(struct_flags) ? '}' : ']'; + fs->buffer = ptr; + } +} - memcpy( ptr, data, len ); - fs->buffer = ptr + len; + +static void +icvJSONStartNextStream( CvFileStorage* fs ) +{ + if( !fs->is_first ) + { + while( fs->write_stack->total > 0 ) + icvJSONEndWriteStruct(fs); + + fs->struct_indent = 4; + icvFSFlush(fs); + fs->buffer = fs->buffer_start; } } static void -icvXMLWriteInt( CvFileStorage* fs, const char* key, int value ) +icvJSONWriteInt( CvFileStorage* fs, const char* key, int value ) { - char buf[128], *ptr = icv_itoa( value, buf, 10 ); - int len = (int)strlen(ptr); - icvXMLWriteScalar( fs, key, ptr, len ); + char buf[128]; + icvJSONWrite( fs, key, icv_itoa( value, buf, 10 )); } static void -icvXMLWriteReal( CvFileStorage* fs, const char* key, double value ) +icvJSONWriteReal( CvFileStorage* fs, const char* key, double value ) { char buf[128]; - int len = (int)strlen( icvDoubleToString( buf, value )); - icvXMLWriteScalar( fs, key, buf, len ); + icvJSONWrite( fs, key, icvDoubleToString( buf, value )); } static void -icvXMLWriteString( CvFileStorage* fs, const char* key, const char* str, int quote ) +icvJSONWriteString( CvFileStorage* fs, const char* key, + const char* str, int quote CV_DEFAULT(0)) { - char buf[CV_FS_MAX_LEN*6+16]; + char buf[CV_FS_MAX_LEN*4+16]; char* data = (char*)str; int i, len; @@ -3069,139 +4025,77 @@ icvXMLWriteString( CvFileStorage* fs, const char* key, const char* str, int quot if( len > CV_FS_MAX_LEN ) CV_Error( CV_StsBadArg, "The written string is too long" ); - if( quote || len == 0 || str[0] != '\"' || str[0] != str[len-1] ) + if( quote || len == 0 || str[0] != str[len-1] || (str[0] != '\"' && str[0] != '\'') ) { - int need_quote = quote || len == 0; + int need_quote = 1; data = buf; *data++ = '\"'; for( i = 0; i < len; i++ ) { char c = str[i]; - if( (uchar)c >= 128 || c == ' ' ) - { - *data++ = c; - need_quote = 1; - } - else if( !cv_isprint(c) || c == '<' || c == '>' || c == '&' || c == '\'' || c == '\"' ) + switch ( c ) { - *data++ = '&'; - if( c == '<' ) - { - memcpy(data, "lt", 2); - data += 2; - } - else if( c == '>' ) - { - memcpy(data, "gt", 2); - data += 2; - } - else if( c == '&' ) - { - memcpy(data, "amp", 3); - data += 3; - } - else if( c == '\'' ) - { - memcpy(data, "apos", 4); - data += 4; - } - else if( c == '\"' ) - { - memcpy( data, "quot", 4); - data += 4; - } - else - { - sprintf( data, "#x%02x", (uchar)c ); - data += 4; - } - *data++ = ';'; - need_quote = 1; + case '\\': + case '\"': + case '\'': { *data++ = '\\'; *data++ = c; break; } + case '\n': { *data++ = '\\'; *data++ = 'n'; break; } + case '\r': { *data++ = '\\'; *data++ = 'r'; break; } + case '\t': { *data++ = '\\'; *data++ = 't'; break; } + case '\b': { *data++ = '\\'; *data++ = 'b'; break; } + case '\f': { *data++ = '\\'; *data++ = 'f'; break; } + default : { *data++ = c; } + break; } - else - *data++ = c; } - if( !need_quote && (cv_isdigit(str[0]) || - str[0] == '+' || str[0] == '-' || str[0] == '.' )) - need_quote = 1; - if( need_quote ) - *data++ = '\"'; - len = (int)(data - buf) - !need_quote; + *data++ = '\"'; *data++ = '\0'; data = buf + !need_quote; } - icvXMLWriteScalar( fs, key, data, len ); + icvJSONWrite( fs, key, data ); } static void -icvXMLWriteComment( CvFileStorage* fs, const char* comment, int eol_comment ) +icvJSONWriteComment( CvFileStorage* fs, const char* comment, int eol_comment ) { - int len; - int multiline; - const char* eol; - char* ptr; - if( !comment ) CV_Error( CV_StsNullPtr, "Null comment" ); - if( strstr(comment, "--") != 0 ) - CV_Error( CV_StsBadArg, "Double hyphen \'--\' is not allowed in the comments" ); - - len = (int)strlen(comment); - eol = strchr(comment, '\n'); - multiline = eol != 0; - ptr = fs->buffer; - - if( multiline || !eol_comment || fs->buffer_end - ptr < len + 5 ) - ptr = icvXMLFlush( fs ); - else if( ptr > fs->buffer_start + fs->struct_indent ) - *ptr++ = ' '; + int len = static_cast(strlen(comment)); + char* ptr = fs->buffer; + const char* eol = strchr(comment, '\n'); + bool multiline = eol != 0; - if( !multiline ) - { - ptr = icvFSResizeWriteBuffer( fs, ptr, len + 9 ); - sprintf( ptr, "", comment ); - len = (int)strlen(ptr); - } + if( !eol_comment || multiline || fs->buffer_end - ptr < len || ptr == fs->buffer_start ) + ptr = icvFSFlush( fs ); else - { - strcpy( ptr, "" ); - fs->buffer = ptr + 3; - icvXMLFlush( fs ); + else + { + len = (int)strlen(comment); + ptr = icvFSResizeWriteBuffer( fs, ptr, len ); + memcpy( ptr, comment, len ); + fs->buffer = ptr + len; + comment = 0; + } + ptr = icvFSFlush( fs ); } } @@ -3218,7 +4112,7 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const bool append = (flags & 3) == CV_STORAGE_APPEND; bool mem = (flags & CV_STORAGE_MEMORY) != 0; bool write_mode = (flags & 3) != 0; - bool write_base64 = write_mode && (flags & CV_STORAGE_BASE64) != 0; + bool write_base64 = (write_mode || append) && (flags & CV_STORAGE_BASE64) != 0; bool isGZ = false; size_t fnamelen = 0; const char * filename = query; @@ -3231,7 +4125,7 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const filename = params.begin()->c_str(); if ( write_base64 == false && is_param_exist( params, "base64" ) ) - write_base64 = true; + write_base64 = (write_mode || append); } if( !filename || filename[0] == '\0' ) @@ -3311,13 +4205,23 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const if( fmt == CV_STORAGE_FORMAT_AUTO && filename ) { - const char* dot_pos = filename + fnamelen - (isGZ ? 7 : 4); - fs->fmt = (dot_pos >= filename && (memcmp( dot_pos, ".xml", 4) == 0 || - memcmp(dot_pos, ".XML", 4) == 0 || memcmp(dot_pos, ".Xml", 4) == 0)) ? - CV_STORAGE_FORMAT_XML : CV_STORAGE_FORMAT_YAML; + const char* dot_pos = strrchr( filename, '.' ); + fs->fmt + = cv_strcasecmp( dot_pos, ".xml" ) + ? CV_STORAGE_FORMAT_XML + : cv_strcasecmp( dot_pos, ".json" ) + ? CV_STORAGE_FORMAT_JSON + : CV_STORAGE_FORMAT_YAML + ; + } + else if ( fmt != CV_STORAGE_FORMAT_AUTO ) + { + fs->fmt = fmt; } else - fs->fmt = fmt != CV_STORAGE_FORMAT_AUTO ? fmt : CV_STORAGE_FORMAT_XML; + { + fs->fmt = CV_STORAGE_FORMAT_XML; + } // we use factor=6 for XML (the longest characters (' and ") are encoded with 6 bytes (' and ") // and factor=4 for YAML ( as we use 4 bytes for non ASCII characters (e.g. \xAB)) @@ -3415,7 +4319,7 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const fs->write_comment = icvXMLWriteComment; fs->start_next_stream = icvXMLStartNextStream; } - else + else if( fs->fmt == CV_STORAGE_FORMAT_YAML ) { if( !append ) icvPuts( fs, "%YAML 1.0\n---\n" ); @@ -3429,6 +4333,49 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const fs->write_comment = icvYMLWriteComment; fs->start_next_stream = icvYMLStartNextStream; } + else + { + // TODO: JSON func + if( !append ) + icvPuts( fs, "{\n" ); + else + { + bool valid = false; + long roffset = 0; + for ( ; + fseek( fs->file, roffset, SEEK_END ) == 0; + roffset -= 1 ) + { + const char end_mark = '}'; + if ( fgetc( fs->file ) == end_mark ) + { + fseek( fs->file, roffset, SEEK_END ); + valid = true; + break; + } + } + + if ( valid ) + { + icvCloseFile( fs ); + fs->file = fopen( fs->filename, "r+t" ); + fseek( fs->file, roffset, SEEK_END ); + fputs( ",", fs->file ); + } + else + { + CV_Error( CV_StsError, "Could not find '}' in the end of file.\n" ); + } + } + fs->struct_indent = 4; + fs->start_write_struct = icvJSONStartWriteStruct; + fs->end_write_struct = icvJSONEndWriteStruct; + fs->write_int = icvJSONWriteInt; + fs->write_real = icvJSONWriteReal; + fs->write_string = icvJSONWriteString; + fs->write_comment = icvJSONWriteComment; + fs->start_next_stream = icvJSONStartNextStream; + } } else { @@ -3440,10 +4387,16 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const size_t buf_size = 1 << 20; const char* yaml_signature = "%YAML"; + const char* json_signature = "{"; char buf[16]; icvGets( fs, buf, sizeof(buf)-2 ); - fs->fmt = strncmp( buf, yaml_signature, strlen(yaml_signature) ) == 0 ? - CV_STORAGE_FORMAT_YAML : CV_STORAGE_FORMAT_XML; + fs->fmt + = strncmp( buf, yaml_signature, strlen(yaml_signature) ) == 0 + ? CV_STORAGE_FORMAT_YAML + : strncmp( buf, json_signature, strlen(json_signature) ) == 0 + ? CV_STORAGE_FORMAT_JSON + : CV_STORAGE_FORMAT_XML + ; if( !isGZ ) { @@ -3474,10 +4427,13 @@ cvOpenFileStorage( const char* query, CvMemStorage* dststorage, int flags, const //cvSetErrMode( CV_ErrModeSilent ); try { - if( fs->fmt == CV_STORAGE_FORMAT_XML ) - icvXMLParse( fs ); - else - icvYMLParse( fs ); + switch (fs->fmt) + { + case CV_STORAGE_FORMAT_XML : { icvXMLParse ( fs ); break; } + case CV_STORAGE_FORMAT_YAML: { icvYMLParse ( fs ); break; } + case CV_STORAGE_FORMAT_JSON: { icvJSONParse( fs ); break; } + default: break; + } } catch (...) { @@ -3530,7 +4486,7 @@ cvStartWriteStruct( CvFileStorage* fs, const char* key, int struct_flags, type_name == 0 ) { - /* Uncertain if output Base64 data */ + /* Uncertain whether output Base64 data */ make_write_struct_delayed( fs, key, struct_flags, type_name ); } else if ( type_name && memcmp(type_name, "binary", 6) == 0 ) @@ -3841,8 +4797,14 @@ cvWriteRawData( CvFileStorage* fs, const void* _data, int len, const char* dt ) int buf_len = (int)strlen(ptr); icvXMLWriteScalar( fs, 0, ptr, buf_len ); } - else + else if ( fs->fmt == CV_STORAGE_FORMAT_YAML ) + { icvYMLWrite( fs, 0, ptr ); + } + else + { + icvJSONWrite( fs, 0, ptr ); + } } offset = (int)(data - data0); @@ -6770,7 +7732,19 @@ public: CV_CHECK_OUTPUT_FILE_STORAGE(fs); - ::icvFSFlush(file_storage); + if ( fs->fmt == CV_STORAGE_FORMAT_JSON ) + { + /* clean and break buffer */ + *fs->buffer++ = '\0'; + ::icvPuts( fs, fs->buffer_start ); + fs->buffer = fs->buffer_start; + memset( file_storage->buffer_start, 0, static_cast(file_storage->space) ); + ::icvPuts( fs, "\"$base64$" ); + } + else + { + ::icvFSFlush(file_storage); + } } ~Base64ContextEmitter() @@ -6778,6 +7752,16 @@ public: /* cleaning */ if (src_cur != src_beg) flush(); /* encode the rest binary data to base64 buffer */ + + if ( file_storage->fmt == CV_STORAGE_FORMAT_JSON ) + { + /* clean and break buffer */ + ::icvPuts(file_storage, "\""); + file_storage->buffer = file_storage->buffer_start; + ::icvFSFlush( file_storage ); + memset( file_storage->buffer_start, 0, static_cast(file_storage->space) ); + file_storage->buffer = file_storage->buffer_start; + } } Base64ContextEmitter & write(const uchar * beg, const uchar * end) @@ -6835,17 +7819,25 @@ public: src_cur = src_beg; { // TODO: better solutions. - const char newline[] = "\n"; - char space[80]; - int ident = file_storage->struct_indent; - memset(space, ' ', ident); - space[ident] = '\0'; + if ( file_storage->fmt == CV_STORAGE_FORMAT_JSON ) + { + ::icvPuts(file_storage, (const char*)base64_buffer.data()); + } + else + { + const char newline[] = "\n"; + char space[80]; + int ident = file_storage->struct_indent; + memset(space, ' ', static_cast(ident)); + space[ident] = '\0'; + + ::icvPuts(file_storage, space); + ::icvPuts(file_storage, (const char*)base64_buffer.data()); + ::icvPuts(file_storage, newline); + ::icvFSFlush(file_storage); + } - ::icvPuts(file_storage, space); - ::icvPuts(file_storage, (const char*)base64_buffer.data()); - ::icvPuts(file_storage, newline); - ::icvFSFlush(file_storage); } return true; @@ -7175,7 +8167,6 @@ base64::Base64Writer::Base64Writer(::CvFileStorage * fs) , data_type_string() { CV_CHECK_OUTPUT_FILE_STORAGE(fs); - icvFSFlush(fs); } void base64::Base64Writer::write(const void* _data, size_t len, const char* dt) diff --git a/modules/core/test/test_io.cpp b/modules/core/test/test_io.cpp index 7e2d6ec58378e3299a784528279c50f1bee77177..5ba4fd190964a45f8a32c0061c182cee2a9a73e1 100644 --- a/modules/core/test/test_io.cpp +++ b/modules/core/test/test_io.cpp @@ -91,9 +91,10 @@ protected: {-1000000, 1000000}, {-10, 10}, {-10, 10}}; RNG& rng = ts->get_rng(); RNG rng0; - test_case_count = 4; int progress = 0; MemStorage storage(cvCreateMemStorage(0)); + const char * suffixs[3] = {".yml", ".xml", ".json" }; + test_case_count = 6; for( int idx = 0; idx < test_case_count; idx++ ) { @@ -102,8 +103,8 @@ protected: cvClearMemStorage(storage); - bool mem = (idx % 4) >= 2; - string filename = tempfile(idx % 2 ? ".yml" : ".xml"); + bool mem = (idx % test_case_count) >= (test_case_count >> 1); + string filename = tempfile(suffixs[idx % (test_case_count >> 1)]); FileStorage fs(filename, FileStorage::WRITE + (mem ? FileStorage::MEMORY : 0)); @@ -430,82 +431,91 @@ public: protected: void run(int) { - try - { - string fname = cv::tempfile(".xml"); - vector mi, mi2, mi3, mi4; - vector mv, mv2, mv3, mv4; - vector vudt, vudt2, vudt3, vudt4; - Mat m(10, 9, CV_32F); - Mat empty; - UserDefinedType udt = { 8, 3.3f }; - randu(m, 0, 1); - mi3.push_back(5); - mv3.push_back(m); - vudt3.push_back(udt); - Point_ p1(1.1f, 2.2f), op1; - Point3i p2(3, 4, 5), op2; - Size s1(6, 7), os1; - Complex c1(9, 10), oc1; - Rect r1(11, 12, 13, 14), or1; - Vec v1(15, 16, 17, 18, 19), ov1; - Scalar sc1(20.0, 21.1, 22.2, 23.3), osc1; - Range g1(7, 8), og1; - - FileStorage fs(fname, FileStorage::WRITE); - fs << "mi" << mi; - fs << "mv" << mv; - fs << "mi3" << mi3; - fs << "mv3" << mv3; - fs << "vudt" << vudt; - fs << "vudt3" << vudt3; - fs << "empty" << empty; - fs << "p1" << p1; - fs << "p2" << p2; - fs << "s1" << s1; - fs << "c1" << c1; - fs << "r1" << r1; - fs << "v1" << v1; - fs << "sc1" << sc1; - fs << "g1" << g1; - fs.release(); + const char * suffix[3] = { + ".yml", + ".xml", + ".json" + }; - fs.open(fname, FileStorage::READ); - fs["mi"] >> mi2; - fs["mv"] >> mv2; - fs["mi3"] >> mi4; - fs["mv3"] >> mv4; - fs["vudt"] >> vudt2; - fs["vudt3"] >> vudt4; - fs["empty"] >> empty; - fs["p1"] >> op1; - fs["p2"] >> op2; - fs["s1"] >> os1; - fs["c1"] >> oc1; - fs["r1"] >> or1; - fs["v1"] >> ov1; - fs["sc1"] >> osc1; - fs["g1"] >> og1; - CV_Assert( mi2.empty() ); - CV_Assert( mv2.empty() ); - CV_Assert( cvtest::norm(Mat(mi3), Mat(mi4), CV_C) == 0 ); - CV_Assert( mv4.size() == 1 ); - double n = cvtest::norm(mv3[0], mv4[0], CV_C); - CV_Assert( vudt2.empty() ); - CV_Assert( vudt3 == vudt4 ); - CV_Assert( n == 0 ); - CV_Assert( op1 == p1 ); - CV_Assert( op2 == p2 ); - CV_Assert( os1 == s1 ); - CV_Assert( oc1 == c1 ); - CV_Assert( or1 == r1 ); - CV_Assert( ov1 == v1 ); - CV_Assert( osc1 == sc1 ); - CV_Assert( og1 == g1 ); - } - catch(...) + for ( size_t i = 0u; i < 3u; i++ ) { - ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH); + try + { + string fname = cv::tempfile(suffix[i]); + vector mi, mi2, mi3, mi4; + vector mv, mv2, mv3, mv4; + vector vudt, vudt2, vudt3, vudt4; + Mat m(10, 9, CV_32F); + Mat empty; + UserDefinedType udt = { 8, 3.3f }; + randu(m, 0, 1); + mi3.push_back(5); + mv3.push_back(m); + vudt3.push_back(udt); + Point_ p1(1.1f, 2.2f), op1; + Point3i p2(3, 4, 5), op2; + Size s1(6, 7), os1; + Complex c1(9, 10), oc1; + Rect r1(11, 12, 13, 14), or1; + Vec v1(15, 16, 17, 18, 19), ov1; + Scalar sc1(20.0, 21.1, 22.2, 23.3), osc1; + Range g1(7, 8), og1; + + FileStorage fs(fname, FileStorage::WRITE); + fs << "mi" << mi; + fs << "mv" << mv; + fs << "mi3" << mi3; + fs << "mv3" << mv3; + fs << "vudt" << vudt; + fs << "vudt3" << vudt3; + fs << "empty" << empty; + fs << "p1" << p1; + fs << "p2" << p2; + fs << "s1" << s1; + fs << "c1" << c1; + fs << "r1" << r1; + fs << "v1" << v1; + fs << "sc1" << sc1; + fs << "g1" << g1; + fs.release(); + + fs.open(fname, FileStorage::READ); + fs["mi"] >> mi2; + fs["mv"] >> mv2; + fs["mi3"] >> mi4; + fs["mv3"] >> mv4; + fs["vudt"] >> vudt2; + fs["vudt3"] >> vudt4; + fs["empty"] >> empty; + fs["p1"] >> op1; + fs["p2"] >> op2; + fs["s1"] >> os1; + fs["c1"] >> oc1; + fs["r1"] >> or1; + fs["v1"] >> ov1; + fs["sc1"] >> osc1; + fs["g1"] >> og1; + CV_Assert( mi2.empty() ); + CV_Assert( mv2.empty() ); + CV_Assert( cvtest::norm(Mat(mi3), Mat(mi4), CV_C) == 0 ); + CV_Assert( mv4.size() == 1 ); + double n = cvtest::norm(mv3[0], mv4[0], CV_C); + CV_Assert( vudt2.empty() ); + CV_Assert( vudt3 == vudt4 ); + CV_Assert( n == 0 ); + CV_Assert( op1 == p1 ); + CV_Assert( op2 == p2 ); + CV_Assert( os1 == s1 ); + CV_Assert( oc1 == c1 ); + CV_Assert( or1 == r1 ); + CV_Assert( ov1 == v1 ); + CV_Assert( osc1 == sc1 ); + CV_Assert( og1 == g1 ); + } + catch(...) + { + ts->set_failed_test_info(cvtest::TS::FAIL_MISMATCH); + } } } }; @@ -618,6 +628,7 @@ TEST(Core_InputOutput, filestorage_base64_basic) char const * filenames[] = { "core_io_base64_basic_test.yml", "core_io_base64_basic_test.xml", + "core_io_base64_basic_test.json", 0 }; @@ -745,15 +756,19 @@ TEST(Core_InputOutput, filestorage_base64_valid_call) char const * filenames[] = { "core_io_base64_other_test.yml", "core_io_base64_other_test.xml", + "core_io_base64_other_test.json", "core_io_base64_other_test.yml?base64", "core_io_base64_other_test.xml?base64", + "core_io_base64_other_test.json?base64", 0 }; char const * real_name[] = { "core_io_base64_other_test.yml", "core_io_base64_other_test.xml", + "core_io_base64_other_test.json", "core_io_base64_other_test.yml", "core_io_base64_other_test.xml", + "core_io_base64_other_test.json", 0 }; @@ -829,6 +844,7 @@ TEST(Core_InputOutput, filestorage_base64_invalid_call) char const * filenames[] = { "core_io_base64_other_test.yml", "core_io_base64_other_test.xml", + "core_io_base64_other_test.json", 0 }; diff --git a/modules/ml/test/test_save_load.cpp b/modules/ml/test/test_save_load.cpp index 7c2a7c4a6c0a2cdd2c1c3dcf839df037662474d0..b97c8c3ab34cf1f39e046fa98690951f49c75d03 100644 --- a/modules/ml/test/test_save_load.cpp +++ b/modules/ml/test/test_save_load.cpp @@ -64,11 +64,11 @@ int CV_SLMLTest::run_test_case( int testCaseIdx ) if( code == cvtest::TS::OK ) { get_test_error( testCaseIdx, &test_resps1 ); - fname1 = tempfile(".yml.gz"); + fname1 = tempfile(".json.gz"); save( (fname1 + "?base64").c_str() ); load( fname1.c_str() ); get_test_error( testCaseIdx, &test_resps2 ); - fname2 = tempfile(".yml.gz"); + fname2 = tempfile(".json.gz"); save( (fname2 + "?base64").c_str() ); } else @@ -279,7 +279,7 @@ TEST(DISABLED_ML_SVM, linear_save_load) svm1 = Algorithm::load("SVM45_X_38-1.xml"); svm2 = Algorithm::load("SVM45_X_38-2.xml"); - string tname = tempfile("a.xml"); + string tname = tempfile("a.json"); svm2->save(tname + "?base64"); svm3 = Algorithm::load(tname);