diff --git a/.buildkite/long_running_tests.yml b/.buildkite/long_running_tests.yml index d4bc7244193bfc3b8a78325d845d35bb9bad7e7b..ed3e2da28502626546796516847d196b8ddd9f33 100644 --- a/.buildkite/long_running_tests.yml +++ b/.buildkite/long_running_tests.yml @@ -107,7 +107,7 @@ steps: - "mongod.log" - "build/genesis.json" - "build/config.ini" - timeout: 60 + timeout: 100 - command: | echo "--- :arrow_down: Downloading build directory" && \ @@ -131,7 +131,7 @@ steps: docker#v1.4.0: image: "eosio/ci:ubuntu" workdir: /data/job - timeout: 60 + timeout: 100 - command: | echo "--- :arrow_down: Downloading build directory" && \ @@ -155,7 +155,7 @@ steps: docker#v1.4.0: image: "eosio/ci:ubuntu18" workdir: /data/job - timeout: 60 + timeout: 100 - command: | echo "--- :arrow_down: Downloading build directory" && \ @@ -179,7 +179,7 @@ steps: docker#v1.4.0: image: "eosio/ci:fedora" workdir: /data/job - timeout: 60 + timeout: 100 - command: | echo "--- :arrow_down: Downloading build directory" && \ @@ -203,7 +203,7 @@ steps: docker#v1.4.0: image: "eosio/ci:centos" workdir: /data/job - timeout: 60 + timeout: 100 - command: | echo "--- :arrow_down: Downloading build directory" && \ @@ -227,4 +227,4 @@ steps: docker#v1.4.0: image: "eosio/ci:amazonlinux" workdir: /data/job - timeout: 60 + timeout: 100 diff --git a/CMakeLists.txt b/CMakeLists.txt index 405a0388606a4ceb85fb8bd8c1d817c27ffacdf0..3b1a16af1484bb83ff0bf50662fe0af86ae7a53c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -232,6 +232,14 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/EosioTester.cmake.in ${C configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/EosioTesterBuild.cmake.in ${CMAKE_BINARY_DIR}/lib/cmake/EosioTester.cmake @ONLY) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/EosioTester.cmake DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR}/cmake/) +install(FILES LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/) +install(FILES libraries/wabt/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.wabt) +install(FILES libraries/softfloat/COPYING.txt DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.softfloat) +install(FILES libraries/wasm-jit/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.wavm) +install(FILES libraries/fc/secp256k1/upstream/COPYING DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.secp256k1) +install(FILES externals/binaryen/LICENSE DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ RENAME LICENSE.binaryen) +install(FILES libraries/fc/src/network/LICENSE.go DESTINATION ${CMAKE_INSTALL_FULL_DATAROOTDIR}/licenses/eosio/ ) + include(installer) include(doxygen) diff --git a/CMakeModules/EosioTester.cmake.in b/CMakeModules/EosioTester.cmake.in index ed177f0f432831935c4c654f5e21d73799469c98..61e25936b825aa11bae7705c830a6ffbe62f96ab 100644 --- a/CMakeModules/EosioTester.cmake.in +++ b/CMakeModules/EosioTester.cmake.in @@ -49,7 +49,6 @@ else() find_library(libsecp256k1 secp256k1 @CMAKE_INSTALL_FULL_LIBDIR@) endif() -find_library(libbinaryen binaryen @CMAKE_INSTALL_FULL_LIBDIR@) find_library(libwasm WASM @CMAKE_INSTALL_FULL_LIBDIR@) find_library(libwast WAST @CMAKE_INSTALL_FULL_LIBDIR@) find_library(libwabt wabt @CMAKE_INSTALL_FULL_LIBDIR@) @@ -76,7 +75,6 @@ macro(add_eosio_test test_name) ${libtester} ${libchain} ${libfc} - ${libbinaryen} ${libwast} ${libwasm} ${libwabt} diff --git a/CMakeModules/EosioTesterBuild.cmake.in b/CMakeModules/EosioTesterBuild.cmake.in index 6f619632fe07c7b4017445100b4c60db9ac6ff81..2c650def39c4abbc46fa453646dfa695dd09014b 100644 --- a/CMakeModules/EosioTesterBuild.cmake.in +++ b/CMakeModules/EosioTesterBuild.cmake.in @@ -50,7 +50,6 @@ else() find_library(libsecp256k1 secp256k1 @CMAKE_BINARY_DIR@/libraries/fc/secp256k1) endif() -find_library(libbinaryen binaryen @CMAKE_BINARY_DIR@/externals/binaryen/lib) find_library(libwasm WASM @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WASM) find_library(libwast WAST @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/WAST) find_library(libir IR @CMAKE_BINARY_DIR@/libraries/wasm-jit/Source/IR) @@ -77,7 +76,6 @@ macro(add_eosio_test test_name) ${libtester} ${libchain} ${libfc} - ${libbinaryen} ${libwast} ${libwasm} ${libwabt} diff --git a/contracts/eosio.bios/eosio.bios.abi b/contracts/eosio.bios/eosio.bios.abi index c81d774c689fab64074cbc83d9e981b436b7f60a..2dd3310fc678c909bcb50c198c5d29984a517dcb 100644 --- a/contracts/eosio.bios/eosio.bios.abi +++ b/contracts/eosio.bios/eosio.bios.abi @@ -53,6 +53,28 @@ {"name":"accounts", "type":"permission_level_weight[]"}, {"name":"waits", "type":"wait_weight[]"} ] + },{ + "name": "blockchain_parameters", + "base": "", + "fields": [ + {"name":"max_block_net_usage", "type":"uint64"}, + {"name":"target_block_net_usage_pct", "type":"uint32"}, + {"name":"max_transaction_net_usage", "type":"uint32"}, + {"name":"base_per_transaction_net_usage", "type":"uint32"}, + {"name":"net_usage_leeway", "type":"uint32"}, + {"name":"context_free_discount_net_usage_num", "type":"uint32"}, + {"name":"context_free_discount_net_usage_den", "type":"uint32"}, + {"name":"max_block_cpu_usage", "type":"uint32"}, + {"name":"target_block_cpu_usage_pct", "type":"uint32"}, + {"name":"max_transaction_cpu_usage", "type":"uint32"}, + {"name":"min_transaction_cpu_usage", "type":"uint32"}, + {"name":"max_transaction_lifetime", "type":"uint32"}, + {"name":"deferred_trx_expiration_window", "type":"uint32"}, + {"name":"max_transaction_delay", "type":"uint32"}, + {"name":"max_inline_action_size", "type":"uint32"}, + {"name":"max_inline_action_depth", "type":"uint16"}, + {"name":"max_authority_depth", "type":"uint16"} + ] },{ "name": "newaccount", "base": "", @@ -160,6 +182,12 @@ "fields": [ {"name":"schedule", "type":"producer_key[]"} ] + },{ + "name": "setparams", + "base": "", + "fields": [ + {"name":"params", "type":"blockchain_parameters"} + ] },{ "name": "require_auth", "base": "", @@ -219,6 +247,10 @@ "name": "setprods", "type": "set_producers", "ricardian_contract": "" + },{ + "name": "setparams", + "type": "setparams", + "ricardian_contract": "" },{ "name": "reqauth", "type": "require_auth", diff --git a/contracts/eosio.bios/eosio.bios.cpp b/contracts/eosio.bios/eosio.bios.cpp index 70279d6e460f3d3c4e56e0db966b05117749bc33..66d70f0c47e1afe184ed37bab710cf32623a57a9 100644 --- a/contracts/eosio.bios/eosio.bios.cpp +++ b/contracts/eosio.bios/eosio.bios.cpp @@ -1,3 +1,3 @@ #include -EOSIO_ABI( eosio::bios, (setpriv)(setalimits)(setglimits)(setprods)(reqauth) ) +EOSIO_ABI( eosio::bios, (setpriv)(setalimits)(setglimits)(setprods)(setparams)(reqauth) ) diff --git a/contracts/eosio.bios/eosio.bios.hpp b/contracts/eosio.bios/eosio.bios.hpp index 99807d811c19c42d346ba0b3ede4c331bc4e9bc6..0abca64c90ef7275100123e7b33caf5c64e87974 100644 --- a/contracts/eosio.bios/eosio.bios.hpp +++ b/contracts/eosio.bios/eosio.bios.hpp @@ -34,6 +34,11 @@ namespace eosio { set_proposed_producers(buffer, size); } + void setparams( const eosio::blockchain_parameters& params ) { + require_auth( _self ); + set_blockchain_parameters( params ); + } + void reqauth( action_name from ) { require_auth( from ); } diff --git a/eosio_build.sh b/eosio_build.sh index f24e80d4600734c3e0292261649b7790ae5e8a85..d775a052a2bed17fd5fc1fd300695951cda26a50 100755 --- a/eosio_build.sh +++ b/eosio_build.sh @@ -265,7 +265,7 @@ -DCMAKE_C_COMPILER="${C_COMPILER}" -DWASM_ROOT="${WASM_ROOT}" -DCORE_SYMBOL_NAME="${CORE_SYMBOL_NAME}" \ -DOPENSSL_ROOT_DIR="${OPENSSL_ROOT_DIR}" -DBUILD_MONGO_DB_PLUGIN=true \ -DENABLE_COVERAGE_TESTING="${ENABLE_COVERAGE_TESTING}" -DBUILD_DOXYGEN="${DOXYGEN}" \ - -DCMAKE_INSTALL_PREFIX="/usr/local/eosio" "${SOURCE_DIR}" + -DCMAKE_INSTALL_PREFIX="/usr/local/eosio" ${LOCAL_CMAKE_FLAGS} "${SOURCE_DIR}" then printf "\\n\\t>>>>>>>>>>>>>>>>>>>> CMAKE building EOSIO has exited with the above error.\\n\\n" exit -1 diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index cf4c1be184d064708b706e4b0a119d54ebc1358c..8d7d9a775c232da9be16edddece5896f41c906a1 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -37,7 +37,6 @@ add_library( eosio_chain asset.cpp webassembly/wavm.cpp - webassembly/binaryen.cpp webassembly/wabt.cpp # get_config.cpp @@ -51,12 +50,11 @@ add_library( eosio_chain ) target_link_libraries( eosio_chain eos_utilities fc chainbase Logging IR WAST WASM Runtime - wasm asmjs passes cfg ast emscripten-optimizer support softfloat builtins wabt + softfloat builtins wabt ) target_include_directories( eosio_chain PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../wasm-jit/Include" - "${CMAKE_CURRENT_SOURCE_DIR}/../../externals/binaryen/src" "${CMAKE_SOURCE_DIR}/libraries/wabt" "${CMAKE_BINARY_DIR}/libraries/wabt" ) diff --git a/libraries/chain/abi_serializer.cpp b/libraries/chain/abi_serializer.cpp index 1c654ddc4d81d8a198d55f9d2824b54c178d2ece..639e2eb430d4104678ae1ff6ca6316b81f5c562f 100644 --- a/libraries/chain/abi_serializer.cpp +++ b/libraries/chain/abi_serializer.cpp @@ -104,7 +104,8 @@ namespace eosio { namespace chain { } void abi_serializer::set_abi(const abi_def& abi, const fc::microseconds& max_serialization_time) { - const fc::time_point deadline = fc::time_point::now() + max_serialization_time; + impl::abi_traverse_context ctx(max_serialization_time); + EOS_ASSERT(starts_with(abi.version, "eosio::abi/1."), unsupported_abi_version_exception, "ABI has an unsupported version"); typedefs.clear(); @@ -118,8 +119,8 @@ namespace eosio { namespace chain { structs[st.name] = st; for( const auto& td : abi.types ) { - EOS_ASSERT(_is_type(td.type, 0, deadline, max_serialization_time), invalid_type_inside_abi, "invalid type", ("type",td.type)); - EOS_ASSERT(!_is_type(td.new_type_name, 0, deadline, max_serialization_time), duplicate_abi_type_def_exception, "type already exists", ("new_type_name",td.new_type_name)); + EOS_ASSERT(_is_type(td.type, ctx), invalid_type_inside_abi, "invalid type", ("type",td.type)); + EOS_ASSERT(!_is_type(td.new_type_name, ctx), duplicate_abi_type_def_exception, "type already exists", ("new_type_name",td.new_type_name)); typedefs[td.new_type_name] = td.type; } @@ -146,7 +147,7 @@ namespace eosio { namespace chain { EOS_ASSERT( error_messages.size() == abi.error_messages.size(), duplicate_abi_err_msg_def_exception, "duplicate error message definition detected" ); EOS_ASSERT( variants.size() == abi.variants.value.size(), duplicate_abi_variant_def_exception, "duplicate variant definition detected" ); - validate(deadline, max_serialization_time); + validate(ctx); } bool abi_serializer::is_builtin_type(const type_name& type)const { @@ -180,6 +181,11 @@ namespace eosio { namespace chain { return ends_with(string(type), "?"); } + bool abi_serializer::is_type(const type_name& type, const fc::microseconds& max_serialization_time)const { + impl::abi_traverse_context ctx(max_serialization_time); + return _is_type(type, ctx); + } + type_name abi_serializer::fundamental_type(const type_name& type)const { if( is_array(type) ) { return type_name(string(type).substr(0, type.size()-2)); @@ -197,12 +203,11 @@ namespace eosio { namespace chain { return type; } - bool abi_serializer::_is_type(const type_name& rtype, size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const { - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); - if( ++recursion_depth > max_recursion_depth) return false; + bool abi_serializer::_is_type(const type_name& rtype, impl::abi_traverse_context& ctx )const { + auto h = ctx.enter_scope(); auto type = fundamental_type(rtype); if( built_in_types.find(type) != built_in_types.end() ) return true; - if( typedefs.find(type) != typedefs.end() ) return _is_type(typedefs.find(type)->second, recursion_depth, deadline, max_serialization_time); + if( typedefs.find(type) != typedefs.end() ) return _is_type(typedefs.find(type)->second, ctx); if( structs.find(type) != structs.end() ) return true; if( variants.find(type) != variants.end() ) return true; return false; @@ -214,26 +219,26 @@ namespace eosio { namespace chain { return itr->second; } - void abi_serializer::validate(const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const { + void abi_serializer::validate( impl::abi_traverse_context& ctx )const { for( const auto& t : typedefs ) { try { vector types_seen{t.first, t.second}; auto itr = typedefs.find(t.second); while( itr != typedefs.end() ) { - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + ctx.check_deadline(); EOS_ASSERT( find(types_seen.begin(), types_seen.end(), itr->second) == types_seen.end(), abi_circular_def_exception, "Circular reference in type ${type}", ("type",t.first) ); types_seen.emplace_back(itr->second); itr = typedefs.find(itr->second); } } FC_CAPTURE_AND_RETHROW( (t) ) } for( const auto& t : typedefs ) { try { - EOS_ASSERT(_is_type(t.second, 0, deadline, max_serialization_time), invalid_type_inside_abi, "", ("type",t.second) ); + EOS_ASSERT(_is_type(t.second, ctx), invalid_type_inside_abi, "", ("type",t.second) ); } FC_CAPTURE_AND_RETHROW( (t) ) } for( const auto& s : structs ) { try { if( s.second.base != type_name() ) { struct_def current = s.second; vector types_seen{current.name}; while( current.base != type_name() ) { - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + ctx.check_deadline(); const auto& base = get_struct(current.base); //<-- force struct to inherit from another struct EOS_ASSERT( find(types_seen.begin(), types_seen.end(), base.name) == types_seen.end(), abi_circular_def_exception, "Circular reference in struct ${type}", ("type",s.second.name) ); types_seen.emplace_back(base.name); @@ -241,24 +246,24 @@ namespace eosio { namespace chain { } } for( const auto& field : s.second.fields ) { try { - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); - EOS_ASSERT(_is_type(_remove_bin_extension(field.type), 0, deadline, max_serialization_time), invalid_type_inside_abi, "", ("type",field.type) ); + ctx.check_deadline(); + EOS_ASSERT(_is_type(_remove_bin_extension(field.type), ctx), invalid_type_inside_abi, "", ("type",field.type) ); } FC_CAPTURE_AND_RETHROW( (field) ) } } FC_CAPTURE_AND_RETHROW( (s) ) } for( const auto& s : variants ) { try { for( const auto& type : s.second.types ) { try { - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); - EOS_ASSERT(_is_type(type, 0, deadline, max_serialization_time), invalid_type_inside_abi, "", ("type",type) ); + ctx.check_deadline(); + EOS_ASSERT(_is_type(type, ctx), invalid_type_inside_abi, "", ("type",type) ); } FC_CAPTURE_AND_RETHROW( (type) ) } } FC_CAPTURE_AND_RETHROW( (s) ) } for( const auto& a : actions ) { try { - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); - EOS_ASSERT(_is_type(a.second, 0, deadline, max_serialization_time), invalid_type_inside_abi, "", ("type",a.second) ); + ctx.check_deadline(); + EOS_ASSERT(_is_type(a.second, ctx), invalid_type_inside_abi, "", ("type",a.second) ); } FC_CAPTURE_AND_RETHROW( (a) ) } for( const auto& t : tables ) { try { - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); - EOS_ASSERT(_is_type(t.second, 0, deadline, max_serialization_time), invalid_type_inside_abi, "", ("type",t.second) ); + ctx.check_deadline(); + EOS_ASSERT(_is_type(t.second, ctx), invalid_type_inside_abi, "", ("type",t.second) ); } FC_CAPTURE_AND_RETHROW( (t) ) } } @@ -275,156 +280,247 @@ namespace eosio { namespace chain { } void abi_serializer::_binary_to_variant( const type_name& type, fc::datastream& stream, - fc::mutable_variant_object& obj, size_t recursion_depth, - const fc::time_point& deadline, const fc::microseconds& max_serialization_time )const + fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const { - EOS_ASSERT( ++recursion_depth < max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); - const auto& st = get_struct(type); + auto h = ctx.enter_scope(); + auto s_itr = structs.find(type); + EOS_ASSERT( s_itr != structs.end(), invalid_type_inside_abi, "Unknown type ${type}", ("type",ctx.maybe_shorten(type)) ); + ctx.hint_struct_type_if_in_array( s_itr ); + const auto& st = s_itr->second; if( st.base != type_name() ) { - _binary_to_variant(resolve_type(st.base), stream, obj, recursion_depth, deadline, max_serialization_time); + _binary_to_variant(resolve_type(st.base), stream, obj, ctx); } - for( const auto& field : st.fields ) { - if( !stream.remaining() && ends_with(field.type, "$") ) - continue; - obj( field.name, _binary_to_variant(resolve_type(_remove_bin_extension(field.type)), stream, recursion_depth, deadline, max_serialization_time) ); + bool encountered_extension = false; + for( uint32_t i = 0; i < st.fields.size(); ++i ) { + const auto& field = st.fields[i]; + bool extension = ends_with(field.type, "$"); + encountered_extension |= extension; + if( !stream.remaining() ) { + if( extension ) { + continue; + } + if( encountered_extension ) { + EOS_THROW( abi_exception, "Encountered field '${f}' without binary extension designation while processing struct '${p}'", + ("f", ctx.maybe_shorten(field.name))("p", ctx.get_path_string()) ); + } + EOS_THROW( unpack_exception, "Stream unexpectedly ended; unable to unpack field '${f}' of struct '${p}'", + ("f", ctx.maybe_shorten(field.name))("p", ctx.get_path_string()) ); + + } + auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } ); + obj( field.name, _binary_to_variant(resolve_type( extension ? _remove_bin_extension(field.type) : field.type ), stream, ctx) ); } } fc::variant abi_serializer::_binary_to_variant( const type_name& type, fc::datastream& stream, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time )const + impl::binary_to_variant_context& ctx )const { - EOS_ASSERT( ++recursion_depth < max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); type_name rtype = resolve_type(type); auto ftype = fundamental_type(rtype); auto btype = built_in_types.find(ftype ); if( btype != built_in_types.end() ) { - return btype->second.first(stream, is_array(rtype), is_optional(rtype)); + try { + return btype->second.first(stream, is_array(rtype), is_optional(rtype)); + } EOS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack ${class} type '${type}' while processing '${p}'", + ("class", is_array(rtype) ? "array of built-in" : is_optional(rtype) ? "optional of built-in" : "built-in") + ("type", ftype)("p", ctx.get_path_string()) ) } if ( is_array(rtype) ) { - fc::unsigned_int size; - fc::raw::unpack(stream, size); - vector vars; - for( decltype(size.value) i = 0; i < size; ++i ) { - auto v = _binary_to_variant(ftype, stream, recursion_depth, deadline, max_serialization_time); - EOS_ASSERT( !v.is_null(), unpack_exception, "Invalid packed array" ); - vars.emplace_back(std::move(v)); - } - EOS_ASSERT( vars.size() == size.value, - unpack_exception, - "packed size does not match unpacked array size, packed size ${p} actual size ${a}", - ("p", size)("a", vars.size()) ); - return fc::variant( std::move(vars) ); + ctx.hint_array_type_if_in_array(); + fc::unsigned_int size; + try { + fc::raw::unpack(stream, size); + } EOS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack size of array '${p}'", ("p", ctx.get_path_string()) ) + vector vars; + auto h1 = ctx.push_to_path( impl::array_index_path_item{} ); + for( decltype(size.value) i = 0; i < size; ++i ) { + ctx.set_array_index_of_path_back(i); + auto v = _binary_to_variant(ftype, stream, ctx); + // QUESTION: Is it actually desired behavior to require the returned variant to not be null? + // This would disallow arrays of optionals in general (though if all optionals in the array were present it would be allowed). + // Is there any scenario in which the returned variant would be null other than in the case of an empty optional? + EOS_ASSERT( !v.is_null(), unpack_exception, "Invalid packed array '${p}'", ("p", ctx.get_path_string()) ); + vars.emplace_back(std::move(v)); + } + // QUESTION: Why would the assert below ever fail? + EOS_ASSERT( vars.size() == size.value, + unpack_exception, + "packed size does not match unpacked array size, packed size ${p} actual size ${a}", + ("p", size)("a", vars.size()) ); + return fc::variant( std::move(vars) ); } else if ( is_optional(rtype) ) { - char flag; - fc::raw::unpack(stream, flag); - return flag ? _binary_to_variant(ftype, stream, recursion_depth, deadline, max_serialization_time) : fc::variant(); + char flag; + try { + fc::raw::unpack(stream, flag); + } EOS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack presence flag of optional '${p}'", ("p", ctx.get_path_string()) ) + return flag ? _binary_to_variant(ftype, stream, ctx) : fc::variant(); } else { - auto v = variants.find(rtype); - if( v != variants.end() ) { + auto v_itr = variants.find(rtype); + if( v_itr != variants.end() ) { + ctx.hint_variant_type_if_in_array( v_itr ); fc::unsigned_int select; - fc::raw::unpack(stream, select); - EOS_ASSERT( (size_t)select < v->second.types.size(), unpack_exception, "Invalid packed variant" ); - return vector{v->second.types[select], _binary_to_variant(v->second.types[select], stream, recursion_depth, deadline, max_serialization_time)}; + try { + fc::raw::unpack(stream, select); + } EOS_RETHROW_EXCEPTIONS( unpack_exception, "Unable to unpack tag of variant '${p}'", ("p", ctx.get_path_string()) ) + EOS_ASSERT( (size_t)select < v_itr->second.types.size(), unpack_exception, + "Unpacked invalid tag (${select}) for variant '${p}'", ("select", select.value)("p",ctx.get_path_string()) ); + auto h1 = ctx.push_to_path( impl::variant_path_item{ .variant_itr = v_itr, .variant_ordinal = static_cast(select) } ); + return vector{v_itr->second.types[select], _binary_to_variant(v_itr->second.types[select], stream, ctx)}; } } fc::mutable_variant_object mvo; - _binary_to_variant(rtype, stream, mvo, recursion_depth, deadline, max_serialization_time); - EOS_ASSERT( mvo.size() > 0, unpack_exception, "Unable to unpack stream ${type}", ("type", type) ); + _binary_to_variant(rtype, stream, mvo, ctx); + // QUESTION: Is this assert actually desired? It disallows unpacking empty structs from datastream. + EOS_ASSERT( mvo.size() > 0, unpack_exception, "Unable to unpack '${p}' from stream", ("p", ctx.get_path_string()) ); return fc::variant( std::move(mvo) ); } - fc::variant abi_serializer::_binary_to_variant( const type_name& type, const bytes& binary, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time )const + fc::variant abi_serializer::_binary_to_variant( const type_name& type, const bytes& binary, impl::binary_to_variant_context& ctx )const { - EOS_ASSERT( ++recursion_depth < max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); fc::datastream ds( binary.data(), binary.size() ); - return _binary_to_variant(type, ds, recursion_depth, deadline, max_serialization_time); + return _binary_to_variant(type, ds, ctx); } - void abi_serializer::_variant_to_binary( const type_name& type, const fc::variant& var, fc::datastream& ds, bool allow_extensions, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time )const + fc::variant abi_serializer::binary_to_variant( const type_name& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path )const { + impl::binary_to_variant_context ctx(*this, max_serialization_time, type); + ctx.short_path = short_path; + return _binary_to_variant(type, binary, ctx); + } + + fc::variant abi_serializer::binary_to_variant( const type_name& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path )const { + impl::binary_to_variant_context ctx(*this, max_serialization_time, type); + ctx.short_path = short_path; + return _binary_to_variant(type, binary, ctx); + } + + void abi_serializer::_variant_to_binary( const type_name& type, const fc::variant& var, fc::datastream& ds, impl::variant_to_binary_context& ctx )const { try { - EOS_ASSERT( ++recursion_depth < max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); auto rtype = resolve_type(type); + auto v_itr = variants.end(); + auto s_itr = structs.end(); + auto btype = built_in_types.find(fundamental_type(rtype)); if( btype != built_in_types.end() ) { btype->second.second(var, ds, is_array(rtype), is_optional(rtype)); } else if ( is_array(rtype) ) { + ctx.hint_array_type_if_in_array(); vector vars = var.get_array(); fc::raw::pack(ds, (fc::unsigned_int)vars.size()); + + auto h1 = ctx.push_to_path( impl::array_index_path_item{} ); + auto h2 = ctx.disallow_extensions_unless(false); + + int64_t i = 0; for (const auto& var : vars) { - _variant_to_binary(fundamental_type(rtype), var, ds, false, recursion_depth, deadline, max_serialization_time); + ctx.set_array_index_of_path_back(i); + _variant_to_binary(fundamental_type(rtype), var, ds, ctx); + ++i; } - } else if ( variants.find(rtype) != variants.end() ) { - EOS_ASSERT( var.is_array() && var.size() == 2 && var[size_t(0)].is_string(), abi_exception, "expected array containing variant" ); - auto& v = variants.find(rtype)->second; - auto it = find(v.types.begin(), v.types.end(), var[size_t(0)].get_string()); - EOS_ASSERT( it != v.types.end(), abi_exception, "type is not valid within this variant" ); + } else if( (v_itr = variants.find(rtype)) != variants.end() ) { + ctx.hint_variant_type_if_in_array( v_itr ); + auto& v = v_itr->second; + EOS_ASSERT( var.is_array() && var.size() == 2, pack_exception, + "Expected input to be an array of two items while processing variant '${p}'", ("p", ctx.get_path_string()) ); + EOS_ASSERT( var[size_t(0)].is_string(), pack_exception, + "Encountered non-string as first item of input array while processing variant '${p}'", ("p", ctx.get_path_string()) ); + auto variant_type_str = var[size_t(0)].get_string(); + auto it = find(v.types.begin(), v.types.end(), variant_type_str); + EOS_ASSERT( it != v.types.end(), pack_exception, + "Specified type '${t}' in input array is not valid within the variant '${p}'", + ("t", ctx.maybe_shorten(variant_type_str))("p", ctx.get_path_string()) ); fc::raw::pack(ds, fc::unsigned_int(it - v.types.begin())); - _variant_to_binary( *it, var[size_t(1)], ds, allow_extensions, recursion_depth, deadline, max_serialization_time ); - } else { - const auto& st = get_struct(rtype); + auto h1 = ctx.push_to_path( impl::variant_path_item{ .variant_itr = v_itr, .variant_ordinal = static_cast(it - v.types.begin()) } ); + _variant_to_binary( *it, var[size_t(1)], ds, ctx ); + } else if( (s_itr = structs.find(rtype)) != structs.end() ) { + ctx.hint_struct_type_if_in_array( s_itr ); + const auto& st = s_itr->second; if( var.is_object() ) { const auto& vo = var.get_object(); if( st.base != type_name() ) { - _variant_to_binary(resolve_type(st.base), var, ds, false, recursion_depth, deadline, max_serialization_time); + auto h2 = ctx.disallow_extensions_unless(false); + _variant_to_binary(resolve_type(st.base), var, ds, ctx); } - bool missing_extension = false; - for( const auto& field : st.fields ) { + bool disallow_additional_fields = false; + for( uint32_t i = 0; i < st.fields.size(); ++i ) { + const auto& field = st.fields[i]; if( vo.contains( string(field.name).c_str() ) ) { - if( missing_extension ) - EOS_THROW( pack_exception, "Unexpected '${f}' in variant object", ("f",field.name) ); - _variant_to_binary(_remove_bin_extension(field.type), vo[field.name], ds, allow_extensions && &field == &st.fields.back(), recursion_depth, deadline, max_serialization_time); - } else if( ends_with(field.type, "$") && allow_extensions ) { - missing_extension = true; + if( disallow_additional_fields ) + EOS_THROW( pack_exception, "Unexpected field '${f}' found in input object while processing struct '${p}'", + ("f", ctx.maybe_shorten(field.name))("p", ctx.get_path_string()) ); + { + auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } ); + auto h2 = ctx.disallow_extensions_unless( &field == &st.fields.back() ); + _variant_to_binary(_remove_bin_extension(field.type), vo[field.name], ds, ctx); + } + } else if( ends_with(field.type, "$") && ctx.extensions_allowed() ) { + disallow_additional_fields = true; + } else if( disallow_additional_fields ) { + EOS_THROW( abi_exception, "Encountered field '${f}' without binary extension designation while processing struct '${p}'", + ("f", ctx.maybe_shorten(field.name))("p", ctx.get_path_string()) ); } else { - EOS_THROW( pack_exception, "Missing '${f}' in variant object", ("f",field.name) ); + EOS_THROW( pack_exception, "Missing field '${f}' in input object while processing struct '${p}'", + ("f", ctx.maybe_shorten(field.name))("p", ctx.get_path_string()) ); } } } else if( var.is_array() ) { const auto& va = var.get_array(); - EOS_ASSERT( st.base == type_name(), invalid_type_inside_abi, "support for base class as array not yet implemented" ); - uint32_t i = 0; - for( const auto& field : st.fields ) { - if( va.size() > i ) - _variant_to_binary(_remove_bin_extension(field.type), va[i], ds, allow_extensions && &field == &st.fields.back(), recursion_depth, deadline, max_serialization_time); - else if( ends_with(field.type, "$") && allow_extensions ) + EOS_ASSERT( st.base == type_name(), invalid_type_inside_abi, + "Using input array to specify the fields of the derived struct '${p}'; input arrays are currently only allowed for structs without a base", + ("p",ctx.get_path_string()) ); + for( uint32_t i = 0; i < st.fields.size(); ++i ) { + const auto& field = st.fields[i]; + if( va.size() > i ) { + auto h1 = ctx.push_to_path( impl::field_path_item{ .parent_struct_itr = s_itr, .field_ordinal = i } ); + auto h2 = ctx.disallow_extensions_unless( &field == &st.fields.back() ); + _variant_to_binary(_remove_bin_extension(field.type), va[i], ds, ctx); + } else if( ends_with(field.type, "$") && ctx.extensions_allowed() ) { break; - else - EOS_THROW( pack_exception, "Early end to array specifying the fields of struct '${t}'; require input for field '${f}'", - ("t", st.name)("f", field.name) ); - ++i; + } else { + EOS_THROW( pack_exception, "Early end to input array specifying the fields of struct '${p}'; require input for field '${f}'", + ("p", ctx.get_path_string())("f", ctx.maybe_shorten(field.name)) ); + } } } else { - EOS_THROW( pack_exception, "Failed to serialize struct '${t}' in variant object", ("t", st.name)); + EOS_THROW( pack_exception, "Unexpected input encountered while processing struct '${p}'", ("p",ctx.get_path_string()) ); } + } else { + EOS_THROW( invalid_type_inside_abi, "Unknown type ${type}", ("type",ctx.maybe_shorten(type)) ); } } FC_CAPTURE_AND_RETHROW( (type)(var) ) } - bytes abi_serializer::_variant_to_binary( const type_name& type, const fc::variant& var, bool allow_extensions, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time )const + bytes abi_serializer::_variant_to_binary( const type_name& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const { try { - EOS_ASSERT( ++recursion_depth < max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); - if( !_is_type(type, recursion_depth, deadline, max_serialization_time) ) { + auto h = ctx.enter_scope(); + if( !_is_type(type, ctx) ) { return var.as(); } bytes temp( 1024*1024 ); fc::datastream ds(temp.data(), temp.size() ); - _variant_to_binary(type, var, ds, allow_extensions, recursion_depth, deadline, max_serialization_time); + _variant_to_binary(type, var, ds, ctx); temp.resize(ds.tellp()); return temp; } FC_CAPTURE_AND_RETHROW( (type)(var) ) } + bytes abi_serializer::variant_to_binary( const type_name& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path )const { + impl::variant_to_binary_context ctx(*this, max_serialization_time, type); + ctx.short_path = short_path; + return _variant_to_binary(type, var, ctx); + } + + void abi_serializer::variant_to_binary( const type_name& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path )const { + impl::variant_to_binary_context ctx(*this, max_serialization_time, type); + ctx.short_path = short_path; + _variant_to_binary(type, var, ds, ctx); + } + type_name abi_serializer::get_action_type(name action)const { auto itr = actions.find(action); if( itr != actions.end() ) return itr->second; @@ -444,4 +540,276 @@ namespace eosio { namespace chain { return itr->second; } + namespace impl { + + void abi_traverse_context::check_deadline()const { + EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, + "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + } + + fc::scoped_exit> abi_traverse_context::enter_scope() { + std::function callback = [old_recursion_depth=recursion_depth, this](){ + recursion_depth = old_recursion_depth; + }; + + ++recursion_depth; + EOS_ASSERT( recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, + "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); + + check_deadline(); + + return {std::move(callback)}; + } + + void abi_traverse_context_with_path::set_path_root( const type_name& type ) { + auto rtype = abis.resolve_type(type); + + if( abis.is_array(rtype) ) { + root_of_path = array_type_path_root{}; + } else { + auto itr1 = abis.structs.find(rtype); + if( itr1 != abis.structs.end() ) { + root_of_path = struct_type_path_root{ .struct_itr = itr1 }; + } else { + auto itr2 = abis.variants.find(rtype); + if( itr2 != abis.variants.end() ) { + root_of_path = variant_type_path_root{ .variant_itr = itr2 }; + } + } + } + } + + fc::scoped_exit> abi_traverse_context_with_path::push_to_path( const path_item& item ) { + std::function callback = [this](){ + EOS_ASSERT( path.size() > 0, abi_exception, + "invariant failure in variant_to_binary_context: path is empty on scope exit" ); + path.pop_back(); + }; + + path.push_back( item ); + + return {std::move(callback)}; + } + + void abi_traverse_context_with_path::set_array_index_of_path_back( uint32_t i ) { + EOS_ASSERT( path.size() > 0, abi_exception, "path is empty" ); + + auto& b = path.back(); + + EOS_ASSERT( b.contains(), abi_exception, "trying to set array index without first pushing new array index item" ); + + b.get().array_index = i; + } + + void abi_traverse_context_with_path::hint_array_type_if_in_array() { + if( path.size() == 0 || !path.back().contains() ) + return; + + path.back().get().type_hint = array_type_path_root{}; + } + + void abi_traverse_context_with_path::hint_struct_type_if_in_array( const map::const_iterator& itr ) { + if( path.size() == 0 || !path.back().contains() ) + return; + + path.back().get().type_hint = struct_type_path_root{ .struct_itr = itr }; + } + + void abi_traverse_context_with_path::hint_variant_type_if_in_array( const map::const_iterator& itr ) { + if( path.size() == 0 || !path.back().contains() ) + return; + + path.back().get().type_hint = variant_type_path_root{ .variant_itr = itr }; + } + + constexpr size_t const_strlen( const char* str ) + { + return (*str == 0) ? 0 : const_strlen(str + 1) + 1; + } + + void output_name( std::ostream& s, const string& str, bool shorten, size_t max_length = 64 ) { + constexpr size_t min_num_characters_at_ends = 4; + constexpr size_t preferred_num_tail_end_characters = 6; + constexpr const char* fill_in = "..."; + + static_assert( min_num_characters_at_ends <= preferred_num_tail_end_characters, + "preferred number of tail end characters cannot be less than the imposed absolute minimum" ); + + constexpr size_t fill_in_length = const_strlen( fill_in ); + constexpr size_t min_length = fill_in_length + 2*min_num_characters_at_ends; + constexpr size_t preferred_min_length = fill_in_length + 2*preferred_num_tail_end_characters; + + max_length = std::max( max_length, min_length ); + + if( !shorten || str.size() <= max_length ) { + s << str; + return; + } + + size_t actual_num_tail_end_characters = preferred_num_tail_end_characters; + if( max_length < preferred_min_length ) { + actual_num_tail_end_characters = min_num_characters_at_ends + (max_length - min_length)/2; + } + + s.write( str.data(), max_length - fill_in_length - actual_num_tail_end_characters ); + s.write( fill_in, fill_in_length ); + s.write( str.data() + (str.size() - actual_num_tail_end_characters), actual_num_tail_end_characters ); + } + + struct generate_path_string_visitor { + using result_type = void; + + generate_path_string_visitor( bool shorten_names, bool track_only ) + : shorten_names(shorten_names), track_only( track_only ) + {} + + std::stringstream s; + bool shorten_names = false; + bool track_only = false; + path_item last_path_item; + + void add_dot() { + s << "."; + } + + void operator()( const empty_path_item& item ) { + } + + void operator()( const array_index_path_item& item ) { + if( track_only ) { + last_path_item = item; + return; + } + + s << "[" << item.array_index << "]"; + } + + void operator()( const field_path_item& item ) { + if( track_only ) { + last_path_item = item; + return; + } + + const auto& str = item.parent_struct_itr->second.fields.at(item.field_ordinal).name; + output_name( s, str, shorten_names ); + } + + void operator()( const variant_path_item& item ) { + if( track_only ) { + last_path_item = item; + return; + } + + s << ""; + } + + void operator()( const empty_path_root& item ) { + } + + void operator()( const array_type_path_root& item ) { + s << "ARRAY"; + } + + void operator()( const struct_type_path_root& item ) { + const auto& str = item.struct_itr->first; + output_name( s, str, shorten_names ); + } + + void operator()( const variant_type_path_root& item ) { + const auto& str = item.variant_itr->first; + output_name( s, str, shorten_names ); + } + }; + + struct path_item_type_visitor { + using result_type = void; + + path_item_type_visitor( std::stringstream& s, bool shorten_names ) + : s(s), shorten_names(shorten_names) + {} + + std::stringstream& s; + bool shorten_names = false; + + void operator()( const empty_path_item& item ) { + } + + void operator()( const array_index_path_item& item ) { + const auto& th = item.type_hint; + if( th.contains() ) { + const auto& str = th.get().struct_itr->first; + output_name( s, str, shorten_names ); + } else if( th.contains() ) { + const auto& str = th.get().variant_itr->first; + output_name( s, str, shorten_names ); + } else if( th.contains() ) { + s << "ARRAY"; + } else { + s << "UNKNOWN"; + } + } + + void operator()( const field_path_item& item ) { + const auto& str = item.parent_struct_itr->second.fields.at(item.field_ordinal).type; + output_name( s, str, shorten_names ); + } + + void operator()( const variant_path_item& item ) { + const auto& str = item.variant_itr->second.types.at(item.variant_ordinal); + output_name( s, str, shorten_names ); + } + }; + + string abi_traverse_context_with_path::get_path_string()const { + bool full_path = !short_path; + bool shorten_names = short_path; + + generate_path_string_visitor visitor(shorten_names, !full_path); + if( full_path ) + root_of_path.visit( visitor ); + for( size_t i = 0, n = path.size(); i < n; ++i ) { + if( full_path && !path[i].contains() ) + visitor.add_dot(); + + path[i].visit( visitor ); + + } + + if( !full_path ) { + if( visitor.last_path_item.contains() ) { + root_of_path.visit( visitor ); + } else { + path_item_type_visitor vis2(visitor.s, shorten_names); + visitor.last_path_item.visit(vis2); + } + } + + return visitor.s.str(); + } + + string abi_traverse_context_with_path::maybe_shorten( const string& str ) { + if( !short_path ) + return str; + + std::stringstream s; + output_name( s, str, true ); + return s.str(); + } + + fc::scoped_exit> variant_to_binary_context::disallow_extensions_unless( bool condition ) { + std::function callback = [old_recursion_depth=recursion_depth, old_allow_extensions=allow_extensions, this](){ + allow_extensions = old_allow_extensions; + }; + + if( !condition ) { + allow_extensions = false; + } + + return {std::move(callback)}; + } + } + } } diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index f7513debf62a945701092cc3e420c533f53ef1e7..9b92b8d2a1f1b84685648b9c9d83eb261686e072 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -272,8 +272,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a } uint32_t trx_size = 0; - auto& d = control.db(); - if ( auto ptr = d.find(boost::make_tuple(receiver, sender_id)) ) { + if ( auto ptr = db.find(boost::make_tuple(receiver, sender_id)) ) { EOS_ASSERT( replace_existing, deferred_tx_duplicate, "deferred transaction with the same sender_id and payer already exists" ); // TODO: Remove the following subjective check when the deferred trx replacement RAM bug has been fixed with a hard fork. @@ -283,7 +282,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a // TODO: The logic of the next line needs to be incorporated into the next hard fork. // add_ram_usage( ptr->payer, -(config::billable_size_v + ptr->packed_trx.size()) ); - d.modify( *ptr, [&]( auto& gtx ) { + db.modify( *ptr, [&]( auto& gtx ) { gtx.sender = receiver; gtx.sender_id = sender_id; gtx.payer = payer; @@ -294,7 +293,7 @@ void apply_context::schedule_deferred_transaction( const uint128_t& sender_id, a trx_size = gtx.set( trx ); }); } else { - d.create( [&]( auto& gtx ) { + db.create( [&]( auto& gtx ) { gtx.trx_id = trx.id(); gtx.sender = receiver; gtx.sender_id = sender_id; diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 8c7e00ec78340a37b888d841d1dc00d790db463c..977df891460f06dd30ee984e5bc34dab7be4317b 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1400,9 +1400,11 @@ void controller::startup() { my->init(); } -chainbase::database& controller::db()const { return my->db; } +const chainbase::database& controller::db()const { return my->db; } -fork_database& controller::fork_db()const { return my->fork_db; } +chainbase::database& controller::mutable_db()const { return my->db; } + +const fork_database& controller::fork_db()const { return my->fork_db; } void controller::start_block( block_timestamp_type when, uint16_t confirm_block_count) { diff --git a/libraries/chain/include/eosio/chain/abi_serializer.hpp b/libraries/chain/include/eosio/chain/abi_serializer.hpp index a8e9ba3e520eda50535e8e2c123bda40e8fccc31..221424e041ed8b9db5251b37fbe36885e29a8a62 100644 --- a/libraries/chain/include/eosio/chain/abi_serializer.hpp +++ b/libraries/chain/include/eosio/chain/abi_serializer.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace eosio { namespace chain { @@ -17,8 +18,13 @@ using std::pair; using namespace fc; namespace impl { - struct abi_from_variant; - struct abi_to_variant; + struct abi_from_variant; + struct abi_to_variant; + + struct abi_traverse_context; + struct abi_traverse_context_with_path; + struct binary_to_variant_context; + struct variant_to_binary_context; } /** @@ -33,9 +39,7 @@ struct abi_serializer { type_name resolve_type(const type_name& t)const; bool is_array(const type_name& type)const; bool is_optional(const type_name& type)const; - bool is_type(const type_name& type, const fc::microseconds& max_serialization_time)const { - return _is_type(type, 0, fc::time_point::now() + max_serialization_time, max_serialization_time); - } + bool is_type(const type_name& type, const fc::microseconds& max_serialization_time)const; bool is_builtin_type(const type_name& type)const; bool is_integer(const type_name& type) const; int get_integer_size(const type_name& type) const; @@ -49,19 +53,11 @@ struct abi_serializer { optional get_error_message( uint64_t error_code )const; - fc::variant binary_to_variant(const type_name& type, const bytes& binary, const fc::microseconds& max_serialization_time)const { - return _binary_to_variant(type, binary, 0, fc::time_point::now() + max_serialization_time, max_serialization_time); - } - bytes variant_to_binary(const type_name& type, const fc::variant& var, const fc::microseconds& max_serialization_time)const { - return _variant_to_binary(type, var, true, 0, fc::time_point::now() + max_serialization_time, max_serialization_time); - } + fc::variant binary_to_variant( const type_name& type, const bytes& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; + fc::variant binary_to_variant( const type_name& type, fc::datastream& binary, const fc::microseconds& max_serialization_time, bool short_path = false )const; - fc::variant binary_to_variant(const type_name& type, fc::datastream& binary, const fc::microseconds& max_serialization_time)const { - return _binary_to_variant(type, binary, 0, fc::time_point::now() + max_serialization_time, max_serialization_time); - } - void variant_to_binary(const type_name& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time)const { - _variant_to_binary(type, var, ds, true, 0, fc::time_point::now() + max_serialization_time, max_serialization_time); - } + bytes variant_to_binary( const type_name& type, const fc::variant& var, const fc::microseconds& max_serialization_time, bool short_path = false )const; + void variant_to_binary( const type_name& type, const fc::variant& var, fc::datastream& ds, const fc::microseconds& max_serialization_time, bool short_path = false )const; template static void to_variant( const T& o, fc::variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time ); @@ -105,29 +101,135 @@ private: map> built_in_types; void configure_built_in_types(); - fc::variant _binary_to_variant(const type_name& type, const bytes& binary, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const; - bytes _variant_to_binary(const type_name& type, const fc::variant& var, bool allow_extensions, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const; - - fc::variant _binary_to_variant(const type_name& type, fc::datastream& binary, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const; - void _variant_to_binary(const type_name& type, const fc::variant& var, fc::datastream& ds, bool allow_extensions, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const; + fc::variant _binary_to_variant( const type_name& type, const bytes& binary, impl::binary_to_variant_context& ctx )const; + fc::variant _binary_to_variant( const type_name& type, fc::datastream& binary, impl::binary_to_variant_context& ctx )const; + void _binary_to_variant( const type_name& type, fc::datastream& stream, + fc::mutable_variant_object& obj, impl::binary_to_variant_context& ctx )const; - void _binary_to_variant(const type_name& type, fc::datastream& stream, fc::mutable_variant_object& obj, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const; + bytes _variant_to_binary( const type_name& type, const fc::variant& var, impl::variant_to_binary_context& ctx )const; + void _variant_to_binary( const type_name& type, const fc::variant& var, + fc::datastream& ds, impl::variant_to_binary_context& ctx )const; static type_name _remove_bin_extension(const type_name& type); - bool _is_type(const type_name& type, size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const; + bool _is_type( const type_name& type, impl::abi_traverse_context& ctx )const; - void validate(const fc::time_point& deadline, const fc::microseconds& max_serialization_time)const; + void validate( impl::abi_traverse_context& ctx )const; friend struct impl::abi_from_variant; friend struct impl::abi_to_variant; + friend struct impl::abi_traverse_context_with_path; }; namespace impl { + + struct abi_traverse_context { + abi_traverse_context( fc::microseconds max_serialization_time ) + : max_serialization_time( max_serialization_time ), deadline( fc::time_point::now() + max_serialization_time ), recursion_depth(0) + {} + + abi_traverse_context( fc::microseconds max_serialization_time, fc::time_point deadline ) + : max_serialization_time( max_serialization_time ), deadline( deadline ), recursion_depth(0) + {} + + void check_deadline()const; + + fc::scoped_exit> enter_scope(); + + protected: + fc::microseconds max_serialization_time; + fc::time_point deadline; + size_t recursion_depth; + }; + + struct empty_path_root {}; + + struct array_type_path_root { + }; + + struct struct_type_path_root { + map::const_iterator struct_itr; + }; + + struct variant_type_path_root { + map::const_iterator variant_itr; + }; + + using path_root = static_variant; + + struct empty_path_item {}; + + struct array_index_path_item { + path_root type_hint; + uint32_t array_index = 0; + }; + + struct field_path_item { + map::const_iterator parent_struct_itr; + uint32_t field_ordinal = 0; + }; + + struct variant_path_item { + map::const_iterator variant_itr; + uint32_t variant_ordinal = 0; + }; + + using path_item = static_variant; + + struct abi_traverse_context_with_path : public abi_traverse_context { + abi_traverse_context_with_path( const abi_serializer& abis, fc::microseconds max_serialization_time, const type_name& type ) + : abi_traverse_context( max_serialization_time ), abis(abis) + { + set_path_root(type); + } + + abi_traverse_context_with_path( const abi_serializer& abis, fc::microseconds max_serialization_time, fc::time_point deadline, const type_name& type ) + : abi_traverse_context( max_serialization_time, deadline ), abis(abis) + { + set_path_root(type); + } + + abi_traverse_context_with_path( const abi_serializer& abis, const abi_traverse_context& ctx, const type_name& type ) + : abi_traverse_context(ctx), abis(abis) + { + set_path_root(type); + } + + void set_path_root( const type_name& type ); + + fc::scoped_exit> push_to_path( const path_item& item ); + + void set_array_index_of_path_back( uint32_t i ); + void hint_array_type_if_in_array(); + void hint_struct_type_if_in_array( const map::const_iterator& itr ); + void hint_variant_type_if_in_array( const map::const_iterator& itr ); + + string get_path_string()const; + + string maybe_shorten( const string& str ); + + protected: + const abi_serializer& abis; + path_root root_of_path; + vector path; + public: + bool short_path = false; + }; + + struct binary_to_variant_context : public abi_traverse_context_with_path { + using abi_traverse_context_with_path::abi_traverse_context_with_path; + }; + + struct variant_to_binary_context : public abi_traverse_context_with_path { + using abi_traverse_context_with_path::abi_traverse_context_with_path; + + fc::scoped_exit> disallow_extensions_unless( bool condition ); + + bool extensions_allowed()const { return allow_extensions; } + + protected: + bool allow_extensions = true; + }; + /** * Determine if a type contains ABI related info, perhaps deeply nested * @tparam T - the type to check @@ -187,11 +289,9 @@ namespace impl { * and can be degraded to the normal ::to_variant(...) processing */ template = 1> - static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); mvo(name,v); } @@ -200,25 +300,22 @@ namespace impl { * for these types we create new ABI aware visitors */ template = 1> - static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ); + static void add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, abi_traverse_context& ctx ); /** * template which overloads add for vectors of types which contain ABI information in their trees * for these members we call ::add in order to trigger further processing */ template = 1> - static void add( mutable_variant_object &mvo, const char* name, const vector& v, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void add( mutable_variant_object &mvo, const char* name, const vector& v, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); vector array; array.reserve(v.size()); for (const auto& iter: v) { mutable_variant_object elem_mvo; - add(elem_mvo, "_", iter, resolver, recursion_depth, deadline, max_serialization_time); + add(elem_mvo, "_", iter, resolver, ctx); array.emplace_back(std::move(elem_mvo["_"])); } mvo(name, std::move(array)); @@ -229,14 +326,12 @@ namespace impl { * for these members we call ::add in order to trigger further processing */ template = 1> - static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr& v, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void add( mutable_variant_object &mvo, const char* name, const std::shared_ptr& v, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); if( !v ) return; mutable_variant_object obj_mvo; - add(obj_mvo, "_", *v, resolver, recursion_depth, deadline, max_serialization_time); + add(obj_mvo, "_", *v, resolver, ctx); mvo(name, std::move(obj_mvo["_"])); } @@ -245,27 +340,24 @@ namespace impl { { mutable_variant_object& obj_mvo; Resolver& resolver; - size_t recursion_depth; - fc::time_point deadline; - fc::microseconds max_serialization_time; - add_static_variant( mutable_variant_object& o, Resolver& r, size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) - :obj_mvo(o), resolver(r), recursion_depth(recursion_depth), deadline(deadline), max_serialization_time(max_serialization_time){} + abi_traverse_context& ctx; + + add_static_variant( mutable_variant_object& o, Resolver& r, abi_traverse_context& ctx ) + :obj_mvo(o), resolver(r), ctx(ctx) {} typedef void result_type; template void operator()( T& v )const { - add(obj_mvo, "_", v, resolver, recursion_depth, deadline, max_serialization_time); + add(obj_mvo, "_", v, resolver, ctx); } }; template - static void add( mutable_variant_object &mvo, const char* name, const fc::static_variant& v, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void add( mutable_variant_object &mvo, const char* name, const fc::static_variant& v, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); mutable_variant_object obj_mvo; - add_static_variant adder(obj_mvo, resolver, recursion_depth, deadline, max_serialization_time); + add_static_variant adder(obj_mvo, resolver, ctx); v.visit(adder); mvo(name, std::move(obj_mvo["_"])); } @@ -278,11 +370,9 @@ namespace impl { * @return */ template - static void add( mutable_variant_object &out, const char* name, const action& act, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void add( mutable_variant_object &out, const char* name, const action& act, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); mutable_variant_object mvo; mvo("account", act.account); mvo("name", act.name); @@ -294,7 +384,9 @@ namespace impl { auto type = abi->get_action_type(act.name); if (!type.empty()) { try { - mvo( "data", abi->_binary_to_variant( type, act.data, recursion_depth, deadline, max_serialization_time )); + binary_to_variant_context _ctx(*abi, ctx, type); + _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place + mvo( "data", abi->_binary_to_variant( type, act.data, _ctx )); mvo("hex_data", act.data); } catch(...) { // any failure to serialize data, then leave as not serailzed @@ -320,11 +412,9 @@ namespace impl { * @return */ template - static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void add( mutable_variant_object &out, const char* name, const packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); mutable_variant_object mvo; auto trx = ptrx.get_transaction(); mvo("id", trx.id()); @@ -333,7 +423,7 @@ namespace impl { mvo("packed_context_free_data", ptrx.packed_context_free_data); mvo("context_free_data", ptrx.get_context_free_data()); mvo("packed_trx", ptrx.packed_trx); - add(mvo, "transaction", trx, resolver, recursion_depth, deadline, max_serialization_time); + add(mvo, "transaction", trx, resolver, ctx); out(name, std::move(mvo)); } @@ -350,14 +440,11 @@ namespace impl { class abi_to_variant_visitor { public: - abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, Resolver _resolver, - size_t _recursion_depth, const fc::time_point& _deadline, const fc::microseconds& max_serialization_time ) + abi_to_variant_visitor( mutable_variant_object& _mvo, const T& _val, Resolver _resolver, abi_traverse_context& _ctx ) :_vo(_mvo) ,_val(_val) ,_resolver(_resolver) - ,_recursion_depth(_recursion_depth) - ,_deadline(_deadline) - ,_max_serialization_time(max_serialization_time) + ,_ctx(_ctx) {} /** @@ -370,16 +457,14 @@ namespace impl { template void operator()( const char* name )const { - abi_to_variant::add( _vo, name, (_val.*member), _resolver, _recursion_depth, _deadline, _max_serialization_time ); + abi_to_variant::add( _vo, name, (_val.*member), _resolver, _ctx ); } private: mutable_variant_object& _vo; const T& _val; Resolver _resolver; - size_t _recursion_depth; - fc::time_point _deadline; - fc::microseconds _max_serialization_time; + abi_traverse_context& _ctx; }; struct abi_from_variant { @@ -388,11 +473,9 @@ namespace impl { * and can be degraded to the normal ::from_variant(...) processing */ template = 1> - static void extract( const variant& v, M& o, Resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void extract( const variant& v, M& o, Resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); from_variant(v, o); } @@ -401,25 +484,22 @@ namespace impl { * for these types we create new ABI aware visitors */ template = 1> - static void extract( const variant& v, M& o, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ); + static void extract( const variant& v, M& o, Resolver resolver, abi_traverse_context& ctx ); /** * template which overloads extract for vectors of types which contain ABI information in their trees * for these members we call ::extract in order to trigger further processing */ template = 1> - static void extract( const variant& v, vector& o, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void extract( const variant& v, vector& o, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); const variants& array = v.get_array(); o.clear(); o.reserve( array.size() ); for( auto itr = array.begin(); itr != array.end(); ++itr ) { M o_iter; - extract(*itr, o_iter, resolver, recursion_depth, deadline, max_serialization_time); + extract(*itr, o_iter, resolver, ctx); o.emplace_back(std::move(o_iter)); } } @@ -429,14 +509,12 @@ namespace impl { * for these members we call ::extract in order to trigger further processing */ template = 1> - static void extract( const variant& v, std::shared_ptr& o, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void extract( const variant& v, std::shared_ptr& o, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); M obj; - extract(vo, obj, resolver, recursion_depth, deadline, max_serialization_time); + extract(vo, obj, resolver, ctx); o = std::make_shared(obj); } @@ -446,11 +524,9 @@ namespace impl { * exploded and processed explicitly */ template - static void extract( const variant& v, action& act, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void extract( const variant& v, action& act, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); EOS_ASSERT(vo.contains("account"), packed_transaction_type_exception, "Missing account"); EOS_ASSERT(vo.contains("name"), packed_transaction_type_exception, "Missing name"); @@ -472,7 +548,9 @@ namespace impl { if (abi.valid()) { auto type = abi->get_action_type(act.name); if (!type.empty()) { - act.data = std::move( abi->_variant_to_binary( type, data, true, recursion_depth, deadline, max_serialization_time )); + variant_to_binary_context _ctx(*abi, ctx, type); + _ctx.short_path = true; // Just to be safe while avoiding the complexity of threading an override boolean all over the place + act.data = std::move( abi->_variant_to_binary( type, data, _ctx )); valid_empty_data = act.data.empty(); } } @@ -493,11 +571,9 @@ namespace impl { } template - static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + static void extract( const variant& v, packed_transaction& ptrx, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); EOS_ASSERT(vo.contains("signatures"), packed_transaction_type_exception, "Missing signatures"); EOS_ASSERT(vo.contains("compression"), packed_transaction_type_exception, "Missing compression"); @@ -519,7 +595,7 @@ namespace impl { EOS_ASSERT(vo.contains("transaction"), packed_transaction_type_exception, "Missing transaction"); transaction trx; vector context_free_data; - extract(vo["transaction"], trx, resolver, recursion_depth, deadline, max_serialization_time); + extract(vo["transaction"], trx, resolver, ctx); if( vo.contains("packed_context_free_data") && vo["packed_context_free_data"].is_string() && !vo["packed_context_free_data"].as_string().empty() ) { from_variant(vo["packed_context_free_data"], ptrx.packed_context_free_data ); context_free_data = ptrx.get_context_free_data(); @@ -542,14 +618,11 @@ namespace impl { class abi_from_variant_visitor : reflector_verifier_visitor { public: - abi_from_variant_visitor( const variant_object& _vo, T& v, Resolver _resolver, - size_t _recursion_depth, const fc::time_point& _deadline, const fc::microseconds& max_serialization_time ) + abi_from_variant_visitor( const variant_object& _vo, T& v, Resolver _resolver, abi_traverse_context& _ctx ) : reflector_verifier_visitor(v) ,_vo(_vo) ,_resolver(_resolver) - ,_recursion_depth(_recursion_depth) - ,_deadline(_deadline) - ,_max_serialization_time(max_serialization_time) + ,_ctx(_ctx) {} /** @@ -564,49 +637,45 @@ namespace impl { { auto itr = _vo.find(name); if( itr != _vo.end() ) - abi_from_variant::extract( itr->value(), this->obj.*member, _resolver, _recursion_depth, _deadline, _max_serialization_time ); + abi_from_variant::extract( itr->value(), this->obj.*member, _resolver, _ctx ); } private: const variant_object& _vo; Resolver _resolver; - size_t _recursion_depth; - fc::time_point _deadline; - fc::microseconds _max_serialization_time; + abi_traverse_context& _ctx; }; template> - void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + void abi_to_variant::add( mutable_variant_object &mvo, const char* name, const M& v, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); mutable_variant_object member_mvo; - fc::reflector::visit( impl::abi_to_variant_visitor( member_mvo, v, resolver, recursion_depth, deadline, max_serialization_time ) ); + fc::reflector::visit( impl::abi_to_variant_visitor( member_mvo, v, resolver, ctx) ); mvo(name, std::move(member_mvo)); } template> - void abi_from_variant::extract( const variant& v, M& o, Resolver resolver, - size_t recursion_depth, const fc::time_point& deadline, const fc::microseconds& max_serialization_time ) + void abi_from_variant::extract( const variant& v, M& o, Resolver resolver, abi_traverse_context& ctx ) { - EOS_ASSERT( ++recursion_depth < abi_serializer::max_recursion_depth, abi_recursion_depth_exception, "recursive definition, max_recursion_depth ${r} ", ("r", abi_serializer::max_recursion_depth) ); - EOS_ASSERT( fc::time_point::now() < deadline, abi_serialization_deadline_exception, "serialization time limit ${t}us exceeded", ("t", max_serialization_time) ); + auto h = ctx.enter_scope(); const variant_object& vo = v.get_object(); - fc::reflector::visit( abi_from_variant_visitor( vo, o, resolver, recursion_depth, deadline, max_serialization_time ) ); + fc::reflector::visit( abi_from_variant_visitor( vo, o, resolver, ctx ) ); } -} +} /// namespace eosio::chain::impl template void abi_serializer::to_variant( const T& o, variant& vo, Resolver resolver, const fc::microseconds& max_serialization_time ) try { mutable_variant_object mvo; - impl::abi_to_variant::add(mvo, "_", o, resolver, 0, fc::time_point::now() + max_serialization_time, max_serialization_time); + impl::abi_traverse_context ctx(max_serialization_time); + impl::abi_to_variant::add(mvo, "_", o, resolver, ctx); vo = std::move(mvo["_"]); } FC_RETHROW_EXCEPTIONS(error, "Failed to serialize type", ("object",o)) template void abi_serializer::from_variant( const variant& v, T& o, Resolver resolver, const fc::microseconds& max_serialization_time ) try { - impl::abi_from_variant::extract(v, o, resolver, 0, fc::time_point::now() + max_serialization_time, max_serialization_time); + impl::abi_traverse_context ctx(max_serialization_time); + impl::abi_from_variant::extract(v, o, resolver, ctx); } FC_RETHROW_EXCEPTIONS(error, "Failed to deserialize variant", ("variant",v)) diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index 8a4f98a7caa3d6bb1cded6353758e1535a593478..70fb198b1e767122c6a955dd77eb501df90fea96 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -453,7 +453,7 @@ class apply_context { public: apply_context(controller& con, transaction_context& trx_ctx, const action& a, uint32_t depth=0) :control(con) - ,db(con.db()) + ,db(con.mutable_db()) ,trx_context(trx_ctx) ,act(a) ,receiver(act.account) diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index 9e1dcd0b073090b77777c1bb2a9347ec1dd3eae3..c0e9806319e3aa08b18028a9dce058c14d1cf38a 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -93,7 +93,7 @@ const static uint32_t setcode_ram_bytes_multiplier = 10; ///< multip const static uint32_t hashing_checktime_block_size = 10*1024; /// call checktime from hashing intrinsic once per this number of bytes -const static eosio::chain::wasm_interface::vm_type default_wasm_runtime = eosio::chain::wasm_interface::vm_type::binaryen; +const static eosio::chain::wasm_interface::vm_type default_wasm_runtime = eosio::chain::wasm_interface::vm_type::wabt; const static uint32_t default_abi_serializer_max_time_ms = 15*1000; ///< default deadline for abi serialization methods /** diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 6b5cf3b613f6c2bdd198c302942e0eef350ccfae..6d2baa9165fcdc3c71cd8e9645ea29c64923b4f7 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -146,9 +146,9 @@ namespace eosio { namespace chain { */ void push_confirmation( const header_confirmation& c ); - chainbase::database& db()const; + const chainbase::database& db()const; - fork_database& fork_db()const; + const fork_database& fork_db()const; const account_object& get_account( account_name n )const; const global_property_object& get_global_properties()const; @@ -286,6 +286,10 @@ namespace eosio { namespace chain { } private: + friend class apply_context; + friend class transaction_context; + + chainbase::database& mutable_db()const; std::unique_ptr my; diff --git a/libraries/chain/include/eosio/chain/name.hpp b/libraries/chain/include/eosio/chain/name.hpp index af5af89ec2367a828eadbe7fa64e015044181db8..81c13145dde6815e90d892c288ddb8805b95b2c7 100644 --- a/libraries/chain/include/eosio/chain/name.hpp +++ b/libraries/chain/include/eosio/chain/name.hpp @@ -91,12 +91,6 @@ namespace eosio { namespace chain { operator unsigned __int128()const { return value; } }; - - inline std::vector sort_names( std::vector&& names ) { - fc::deduplicate(names); - return names; - } - } } // eosio::chain namespace std { diff --git a/libraries/chain/include/eosio/chain/wasm_interface.hpp b/libraries/chain/include/eosio/chain/wasm_interface.hpp index 17ac03fddfed41245baea81a02a38a2759a06596..974ee92e5acebb19bd782f815279aa99221a9245 100644 --- a/libraries/chain/include/eosio/chain/wasm_interface.hpp +++ b/libraries/chain/include/eosio/chain/wasm_interface.hpp @@ -53,7 +53,6 @@ namespace eosio { namespace chain { public: enum class vm_type { wavm, - binaryen, wabt }; @@ -77,4 +76,4 @@ namespace eosio{ namespace chain { std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime); }} -FC_REFLECT_ENUM( eosio::chain::wasm_interface::vm_type, (wavm)(binaryen)(wabt) ) +FC_REFLECT_ENUM( eosio::chain::wasm_interface::vm_type, (wavm)(wabt) ) diff --git a/libraries/chain/include/eosio/chain/wasm_interface_private.hpp b/libraries/chain/include/eosio/chain/wasm_interface_private.hpp index df28d79a21b82502df8e9ab762698ffc2baf466e..c3af34d79eac9089c79d5d5f7c51ce943b6bf26d 100644 --- a/libraries/chain/include/eosio/chain/wasm_interface_private.hpp +++ b/libraries/chain/include/eosio/chain/wasm_interface_private.hpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -27,8 +26,6 @@ namespace eosio { namespace chain { wasm_interface_impl(wasm_interface::vm_type vm) { if(vm == wasm_interface::vm_type::wavm) runtime_interface = std::make_unique(); - else if(vm == wasm_interface::vm_type::binaryen) - runtime_interface = std::make_unique(); else if(vm == wasm_interface::vm_type::wabt) runtime_interface = std::make_unique(); else @@ -98,7 +95,6 @@ namespace eosio { namespace chain { #define _REGISTER_INTRINSIC_EXPLICIT(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ _REGISTER_WAVM_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ - _REGISTER_BINARYEN_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ _REGISTER_WABT_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG) #define _REGISTER_INTRINSIC4(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ diff --git a/libraries/chain/include/eosio/chain/webassembly/binaryen.hpp b/libraries/chain/include/eosio/chain/webassembly/binaryen.hpp deleted file mode 100644 index 51e908edbbd9dffc78c93a08adb628a3532df054..0000000000000000000000000000000000000000 --- a/libraries/chain/include/eosio/chain/webassembly/binaryen.hpp +++ /dev/null @@ -1,701 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - - -namespace eosio { namespace chain { namespace webassembly { namespace binaryen { - -using namespace fc; -using namespace wasm; -using namespace eosio::chain::webassembly::common; - - -using linear_memory_type = fc::array; -using call_indirect_table_type = vector; - -struct interpreter_interface; - -struct intrinsic_registrator { - using intrinsic_fn = Literal(*)(interpreter_interface*, LiteralList&); - - static auto& get_map(){ - static map _map; - return _map; - }; - - intrinsic_registrator(const char* name, intrinsic_fn fn) - { - get_map()[string(name)] = fn; - } -}; - -using import_lut_type = unordered_map; - - -struct interpreter_interface : ModuleInstance::ExternalInterface { - interpreter_interface(linear_memory_type& memory, call_indirect_table_type& table, import_lut_type& import_lut, const unsigned& initial_memory_size, apply_context& context) - :memory(memory),table(table),import_lut(import_lut), current_memory_size(initial_memory_size), context(context) - {} - - void importGlobals(std::map& globals, Module& wasm) override - { - - } - - void init(Module& wasm, ModuleInstance& instance) override { - - } - - Literal callImport(Import *import, LiteralList &args) override - { - auto fn_iter = import_lut.find((uintptr_t)import); - EOS_ASSERT(fn_iter != import_lut.end(), wasm_execution_error, "unknown import ${m}:${n}", ("m", import->module.c_str())("n", import->module.c_str())); - return fn_iter->second(this, args); - } - - Literal callTable(Index index, LiteralList& arguments, WasmType result, ModuleInstance& instance) override - { - EOS_ASSERT(index < table.size(), wasm_execution_error, "callIndirect: bad pointer"); - auto* func = instance.wasm.getFunctionOrNull(table[index]); - EOS_ASSERT(func, wasm_execution_error, "callIndirect: uninitialized element"); - EOS_ASSERT(func->params.size() == arguments.size(), wasm_execution_error, "callIndirect: bad # of arguments"); - - for (size_t i = 0; i < func->params.size(); i++) { - EOS_ASSERT(func->params[i] == arguments[i].type, wasm_execution_error, "callIndirect: bad argument type"); - } - EOS_ASSERT(func->result == result, wasm_execution_error, "callIndirect: bad result type"); - return instance.callFunctionInternal(func->name, arguments); - } - - void trap(const char* why) override { - FC_THROW_EXCEPTION(wasm_execution_error, why); - } - - void assert_memory_is_accessible(uint32_t offset, uint32_t size) { - EOS_ASSERT(offset + size <= current_memory_size && offset + size >= offset, - wasm_execution_error, "access violation"); - } - - char* get_validated_pointer(uint32_t offset, uint32_t size) { - assert_memory_is_accessible(offset, size); - return memory.data + offset; - } - - template - static bool aligned_for(const void* address) { - return 0 == (reinterpret_cast(address) & (std::alignment_of::value - 1)); - } - - template - T load_memory(uint32_t offset) { - char *base = get_validated_pointer(offset, sizeof(T)); - if (aligned_for(base)) { - return *reinterpret_cast(base); - } else { - T temp; - memcpy(&temp, base, sizeof(T)); - return temp; - } - } - - template - void store_memory(uint32_t offset, T value) { - char *base = get_validated_pointer(offset, sizeof(T)); - if (aligned_for(base)) { - *reinterpret_cast(base) = value; - } else { - memcpy(base, &value, sizeof(T)); - } - } - - void growMemory(Address old_size, Address new_size) override { - memset(memory.data + old_size.addr, 0, new_size.addr - old_size.addr); - current_memory_size += new_size.addr - old_size.addr; - } - - int8_t load8s(Address addr) override { return load_memory(addr); } - uint8_t load8u(Address addr) override { return load_memory(addr); } - int16_t load16s(Address addr) override { return load_memory(addr); } - uint16_t load16u(Address addr) override { return load_memory(addr); } - int32_t load32s(Address addr) override { return load_memory(addr); } - uint32_t load32u(Address addr) override { return load_memory(addr); } - int64_t load64s(Address addr) override { return load_memory(addr); } - uint64_t load64u(Address addr) override { return load_memory(addr); } - - void store8(Address addr, int8_t value) override { store_memory(addr, value); } - void store16(Address addr, int16_t value) override { store_memory(addr, value); } - void store32(Address addr, int32_t value) override { store_memory(addr, value); } - void store64(Address addr, int64_t value) override { store_memory(addr, value); } - - linear_memory_type& memory; - call_indirect_table_type& table; - import_lut_type& import_lut; - unsigned current_memory_size; - apply_context& context; -}; - -class binaryen_runtime : public eosio::chain::wasm_runtime_interface { - public: - binaryen_runtime(); - std::unique_ptr instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory) override; - - private: - linear_memory_type _memory __attribute__ ((aligned (4096))); -}; - -/** - * class to represent an in-wasm-memory array - * it is a hint to the transcriber that the next parameter will - * be a size (data bytes length) and that the pair are validated together - * This triggers the template specialization of intrinsic_invoker_impl - * @tparam T - */ -template -inline array_ptr array_ptr_impl (interpreter_interface* interface, uint32_t ptr, uint32_t length) -{ - EOS_ASSERT( length < INT_MAX/(uint32_t)sizeof(T), binaryen_exception, "length will overflow" ); - return array_ptr((T*)(interface->get_validated_pointer(ptr, length * (uint32_t)sizeof(T)))); -} - -/** - * class to represent an in-wasm-memory char array that must be null terminated - */ -inline null_terminated_ptr null_terminated_ptr_impl(interpreter_interface* interface, uint32_t ptr) -{ - char *value = interface->get_validated_pointer(ptr, 1); - const char* p = value; - const char* const top_of_memory = interface->memory.data + interface->current_memory_size; - while(p < top_of_memory) - if(*p++ == '\0') - return null_terminated_ptr(value); - - FC_THROW_EXCEPTION(wasm_execution_error, "unterminated string"); -} - - -template -struct is_reference_from_value { - static constexpr bool value = false; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template<> -struct is_reference_from_value { - static constexpr bool value = true; -}; - -template -constexpr bool is_reference_from_value_v = is_reference_from_value::value; - -template -T convert_literal_to_native(Literal& v); - -template<> -inline double convert_literal_to_native(Literal& v) { - return v.getf64(); -} - -template<> -inline float convert_literal_to_native(Literal& v) { - return v.getf32(); -} - -template<> -inline int64_t convert_literal_to_native(Literal& v) { - return v.geti64(); -} - -template<> -inline uint64_t convert_literal_to_native(Literal& v) { - return v.geti64(); -} - -template<> -inline int32_t convert_literal_to_native(Literal& v) { - return v.geti32(); -} - -template<> -inline uint32_t convert_literal_to_native(Literal& v) { - return v.geti32(); -} - -template<> -inline bool convert_literal_to_native(Literal& v) { - return v.geti32(); -} - -template<> -inline name convert_literal_to_native(Literal& v) { - int64_t val = v.geti64(); - return name(val); -} - -template -inline auto convert_native_to_literal(const interpreter_interface*, T val) { - return Literal(val); -} - -inline auto convert_native_to_literal(const interpreter_interface*, const name &val) { - return Literal(val.value); -} - -inline auto convert_native_to_literal(const interpreter_interface*, const fc::time_point_sec &val) { - return Literal(val.sec_since_epoch()); -} - -inline auto convert_native_to_literal(const interpreter_interface* interface, char* ptr) { - const char* base = interface->memory.data; - const char* top_of_memory = base + interface->current_memory_size; - EOS_ASSERT(ptr >= base && ptr < top_of_memory, wasm_execution_error, "returning pointer not in linear memory"); - return Literal((int)(ptr - base)); -} - -struct void_type { -}; - -/** - * Forward declaration of the invoker type which transcribes arguments to/from a native method - * and injects the appropriate checks - * - * @tparam Ret - the return type of the native function - * @tparam NativeParameters - a std::tuple of the remaining native parameters to transcribe - * @tparam WasmParameters - a std::tuple of the transribed parameters - */ -template -struct intrinsic_invoker_impl; - -/** - * Specialization for the fully transcribed signature - * @tparam Ret - the return type of the native function - */ -template -struct intrinsic_invoker_impl> { - using next_method_type = Ret (*)(interpreter_interface*, LiteralList&, int); - - template - static Literal invoke(interpreter_interface* interface, LiteralList& args) { - return convert_native_to_literal(interface, Method(interface, args, args.size() - 1)); - } - - template - static const auto fn() { - return invoke; - } -}; - -/** - * specialization of the fully transcribed signature for void return values - * @tparam Translated - the arguments to the wasm function - */ -template<> -struct intrinsic_invoker_impl> { - using next_method_type = void_type (*)(interpreter_interface*, LiteralList&, int); - - template - static Literal invoke(interpreter_interface* interface, LiteralList& args) { - Method(interface, args, args.size() - 1); - return Literal(); - } - - template - static const auto fn() { - return invoke; - } -}; - -/** - * Sepcialization for transcribing a simple type in the native method signature - * @tparam Ret - the return type of the native method - * @tparam Input - the type of the native parameter to transcribe - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(interpreter_interface*, Input, Inputs..., LiteralList&, int); - - template - static Ret translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) { - auto& last = args.at(offset); - auto native = convert_literal_to_native(last); - return Then(interface, native, rest..., args, (uint32_t)offset - 1); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a array_ptr type in the native method signature - * This type transcribes into 2 wasm parameters: a pointer and byte length and checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl, size_t, Inputs...>> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(interpreter_interface*, array_ptr, size_t, Inputs..., LiteralList&, int); - - template - static auto translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) -> std::enable_if_t::value, Ret> { - static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); - uint32_t ptr = args.at((uint32_t)offset - 1).geti32(); - size_t length = args.at((uint32_t)offset).geti32(); - T* base = array_ptr_impl(interface, ptr, length); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned array of const values" ); - std::vector > copy(length > 0 ? length : 1); - T* copy_ptr = ©[0]; - memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); - return Then(interface, static_cast>(copy_ptr), length, rest..., args, (uint32_t)offset - 2); - } - return Then(interface, static_cast>(base), length, rest..., args, (uint32_t)offset - 2); - }; - - template - static auto translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) -> std::enable_if_t::value, Ret> { - static_assert(!std::is_pointer::value, "Currently don't support array of pointers"); - uint32_t ptr = args.at((uint32_t)offset - 1).geti32(); - size_t length = args.at((uint32_t)offset).geti32(); - T* base = array_ptr_impl(interface, ptr, length); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned array of values" ); - std::vector > copy(length > 0 ? length : 1); - T* copy_ptr = ©[0]; - memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); - Ret ret = Then(interface, static_cast>(copy_ptr), length, rest..., args, (uint32_t)offset - 2); - memcpy( (void*)base, (void*)copy_ptr, length * sizeof(T) ); - return ret; - } - return Then(interface, static_cast>(base), length, rest..., args, (uint32_t)offset - 2); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a null_terminated_ptr type in the native method signature - * This type transcribes 1 wasm parameters: a char pointer which is validated to contain - * a null value before the end of the allocated memory. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(interpreter_interface*, null_terminated_ptr, Inputs..., LiteralList&, int); - - template - static Ret translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) { - uint32_t ptr = args.at((uint32_t)offset).geti32(); - return Then(interface, null_terminated_ptr_impl(interface, ptr), rest..., args, (uint32_t)offset - 1); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a pair of array_ptr types in the native method signature that share size - * This type transcribes into 3 wasm parameters: 2 pointers and byte length and checks the validity of those memory - * ranges before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl, array_ptr, size_t, Inputs...>> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(interpreter_interface*, array_ptr, array_ptr, size_t, Inputs..., LiteralList&, int); - - template - static Ret translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) { - uint32_t ptr_t = args.at((uint32_t)offset - 2).geti32(); - uint32_t ptr_u = args.at((uint32_t)offset - 1).geti32(); - size_t length = args.at((uint32_t)offset).geti32(); - static_assert(std::is_same, char>::value && std::is_same, char>::value, "Currently only support array of (const)chars"); - return Then(interface, array_ptr_impl(interface, ptr_t, length), array_ptr_impl(interface, ptr_u, length), length, args, (uint32_t)offset - 3); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing memset parameters - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl, int, size_t>> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret(*)(interpreter_interface*, array_ptr, int, size_t, LiteralList&, int); - - template - static Ret translate_one(interpreter_interface* interface, LiteralList& args, int offset) { - uint32_t ptr = args.at((uint32_t)offset - 2).geti32(); - uint32_t value = args.at((uint32_t)offset - 1).geti32(); - size_t length = args.at((uint32_t)offset).geti32(); - return Then(interface, array_ptr_impl(interface, ptr, length), value, length, args, (uint32_t)offset - 3); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a pointer type in the native method signature - * This type transcribes into an int32 pointer checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(interpreter_interface*, T *, Inputs..., LiteralList&, int); - - template - static auto translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) -> std::enable_if_t::value, Ret> { - uint32_t ptr = args.at((uint32_t)offset).geti32(); - T* base = array_ptr_impl(interface, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned const pointer" ); - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); - return Then(interface, copy_ptr, rest..., args, (uint32_t)offset - 1); - } - return Then(interface, base, rest..., args, (uint32_t)offset - 1); - }; - - template - static auto translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) -> std::enable_if_t::value, Ret> { - uint32_t ptr = args.at((uint32_t)offset).geti32(); - T* base = array_ptr_impl(interface, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned pointer" ); - T copy; - memcpy( (void*)©, (void*)base, sizeof(T) ); - Ret ret = Then(interface, ©, rest..., args, (uint32_t)offset - 1); - memcpy( (void*)base, (void*)©, sizeof(T) ); - return ret; - } - return Then(interface, base, rest..., args, (uint32_t)offset - 1); - }; - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a reference to a name which can be passed as a native value - * This type transcribes into a native type which is loaded by value into a - * variable on the stack and then passed by reference to the intrinsic. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(interpreter_interface*, const name&, Inputs..., LiteralList&, int); - - template - static Ret translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) { - uint64_t wasm_value = args.at((uint32_t)offset).geti64(); - auto value = name(wasm_value); - return Then(interface, value, rest..., args, (uint32_t)offset - 1); - } - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * Specialization for transcribing a reference to a fc::time_point_sec which can be passed as a native value - * This type transcribes into a native type which is loaded by value into a - * variable on the stack and then passed by reference to the intrinsic. - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(interpreter_interface*, const fc::time_point_sec&, Inputs..., LiteralList&, int); - - template - static Ret translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) { - uint32_t wasm_value = args.at((uint32_t)offset).geti32(); - auto value = fc::time_point_sec(wasm_value); - return Then(interface, value, rest..., args, (uint32_t)offset - 1); - } - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - - -/** - * Specialization for transcribing a reference type in the native method signature - * This type transcribes into an int32 pointer checks the validity of that memory - * range before dispatching to the native method - * - * @tparam Ret - the return type of the native method - * @tparam Inputs - the remaining native parameters to transcribe - */ -template -struct intrinsic_invoker_impl> { - using next_step = intrinsic_invoker_impl>; - using then_type = Ret (*)(interpreter_interface*, T &, Inputs..., LiteralList&, int); - - template - static auto translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) -> std::enable_if_t::value, Ret> { - // references cannot be created for null pointers - uint32_t ptr = args.at((uint32_t)offset).geti32(); - EOS_ASSERT(ptr != 0, binaryen_exception, "references cannot be created for null pointers"); - T* base = array_ptr_impl(interface, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned const reference" ); - std::remove_const_t copy; - T* copy_ptr = © - memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); - return Then(interface, *copy_ptr, rest..., args, (uint32_t)offset - 1); - } - return Then(interface, *base, rest..., args, (uint32_t)offset - 1); - } - - template - static auto translate_one(interpreter_interface* interface, Inputs... rest, LiteralList& args, int offset) -> std::enable_if_t::value, Ret> { - // references cannot be created for null pointers - uint32_t ptr = args.at((uint32_t)offset).geti32(); - EOS_ASSERT(ptr != 0, binaryen_exception, "references cannot be created for null pointers"); - T* base = array_ptr_impl(interface, ptr, 1); - if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned reference" ); - T copy; - memcpy( (void*)©, (void*)base, sizeof(T) ); - Ret ret = Then(interface, copy, rest..., args, (uint32_t)offset - 1); - memcpy( (void*)base, (void*)©, sizeof(T) ); - return ret; - } - return Then(interface, *base, rest..., args, (uint32_t)offset - 1); - } - - - template - static const auto fn() { - return next_step::template fn>(); - } -}; - -/** - * forward declaration of a wrapper class to call methods of the class - */ -template -struct intrinsic_function_invoker { - using impl = intrinsic_invoker_impl>; - - template - static Ret wrapper(interpreter_interface* interface, Params... params, LiteralList&, int) { - class_from_wasm::value(interface->context).checktime(); - return (class_from_wasm::value(interface->context).*Method)(params...); - } - - template - static const intrinsic_registrator::intrinsic_fn fn() { - return impl::template fn>(); - } -}; - -template -struct intrinsic_function_invoker { - using impl = intrinsic_invoker_impl>; - - template - static void_type wrapper(interpreter_interface* interface, Params... params, LiteralList& args, int offset) { - class_from_wasm::value(interface->context).checktime(); - (class_from_wasm::value(interface->context).*Method)(params...); - return void_type(); - } - - template - static const intrinsic_registrator::intrinsic_fn fn() { - return impl::template fn>(); - } - -}; - -template -struct intrinsic_function_invoker_wrapper; - -template -struct intrinsic_function_invoker_wrapper { - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - using type = intrinsic_function_invoker; -}; - -template -struct intrinsic_function_invoker_wrapper { - using type = intrinsic_function_invoker; -}; - -#define _ADD_PAREN_1(...) ((__VA_ARGS__)) _ADD_PAREN_2 -#define _ADD_PAREN_2(...) ((__VA_ARGS__)) _ADD_PAREN_1 -#define _ADD_PAREN_1_END -#define _ADD_PAREN_2_END -#define _WRAPPED_SEQ(SEQ) BOOST_PP_CAT(_ADD_PAREN_1 SEQ, _END) - -#define __INTRINSIC_NAME(LABEL, SUFFIX) LABEL##SUFFIX -#define _INTRINSIC_NAME(LABEL, SUFFIX) __INTRINSIC_NAME(LABEL,SUFFIX) - -#define _REGISTER_BINARYEN_INTRINSIC(CLS, MOD, METHOD, WASM_SIG, NAME, SIG)\ - static eosio::chain::webassembly::binaryen::intrinsic_registrator _INTRINSIC_NAME(__binaryen_intrinsic_fn, __COUNTER__) (\ - MOD "." NAME,\ - eosio::chain::webassembly::binaryen::intrinsic_function_invoker_wrapper::type::fn<&CLS::METHOD>()\ - );\ - - -} } } }// eosio::chain::webassembly::wavm diff --git a/libraries/chain/include/eosio/chain/webassembly/wabt.hpp b/libraries/chain/include/eosio/chain/webassembly/wabt.hpp index 5be568d4b01758f6ff5c53871df563f330bac4cf..bf33448a7872c3e4f7bb610bb4bc5f10a2b6cb02 100644 --- a/libraries/chain/include/eosio/chain/webassembly/wabt.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/wabt.hpp @@ -371,7 +371,8 @@ struct intrinsic_invoker_impl, size_t, Inputs...>> size_t length = args.at((uint32_t)offset).get_i32(); T* base = array_ptr_impl(vars, ptr, length); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned array of const values" ); + if(vars.ctx.control.contracts_console()) + wlog( "misaligned array of const values" ); std::vector > copy(length > 0 ? length : 1); T* copy_ptr = ©[0]; memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); @@ -387,7 +388,8 @@ struct intrinsic_invoker_impl, size_t, Inputs...>> size_t length = args.at((uint32_t)offset).get_i32(); T* base = array_ptr_impl(vars, ptr, length); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned array of values" ); + if(vars.ctx.control.contracts_console()) + wlog( "misaligned array of values" ); std::vector > copy(length > 0 ? length : 1); T* copy_ptr = ©[0]; memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); @@ -500,7 +502,8 @@ struct intrinsic_invoker_impl> { uint32_t ptr = args.at((uint32_t)offset).get_i32(); T* base = array_ptr_impl(vars, ptr, 1); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned const pointer" ); + if(vars.ctx.control.contracts_console()) + wlog( "misaligned const pointer" ); std::remove_const_t copy; T* copy_ptr = © memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); @@ -514,7 +517,8 @@ struct intrinsic_invoker_impl> { uint32_t ptr = args.at((uint32_t)offset).get_i32(); T* base = array_ptr_impl(vars, ptr, 1); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned pointer" ); + if(vars.ctx.control.contracts_console()) + wlog( "misaligned pointer" ); T copy; memcpy( (void*)©, (void*)base, sizeof(T) ); Ret ret = Then(vars, ©, rest..., args, (uint32_t)offset - 1); @@ -603,7 +607,8 @@ struct intrinsic_invoker_impl> { EOS_ASSERT(ptr != 0, binaryen_exception, "references cannot be created for null pointers"); T* base = array_ptr_impl(vars, ptr, 1); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned const reference" ); + if(vars.ctx.control.contracts_console()) + wlog( "misaligned const reference" ); std::remove_const_t copy; T* copy_ptr = © memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); @@ -619,7 +624,8 @@ struct intrinsic_invoker_impl> { EOS_ASSERT(ptr != 0, binaryen_exception, "references cannot be created for null pointers"); T* base = array_ptr_impl(vars, ptr, 1); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned reference" ); + if(vars.ctx.control.contracts_console()) + wlog( "misaligned reference" ); T copy; memcpy( (void*)©, (void*)base, sizeof(T) ); Ret ret = Then(vars, copy, rest..., args, (uint32_t)offset - 1); diff --git a/libraries/chain/include/eosio/chain/webassembly/wavm.hpp b/libraries/chain/include/eosio/chain/webassembly/wavm.hpp index 4674c97e2a8fd1f49d24da1af9bf5dc07bd58b29..2bffe3f621fe626fe55aa32ab8b5ebbf805df2cf 100644 --- a/libraries/chain/include/eosio/chain/webassembly/wavm.hpp +++ b/libraries/chain/include/eosio/chain/webassembly/wavm.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "Runtime/Runtime.h" #include "IR/Types.h" @@ -382,7 +383,8 @@ struct intrinsic_invoker_impl, size_t, Inputs...>, const auto length = size_t(size); T* base = array_ptr_impl(ctx, (U32)ptr, length); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned array of const values" ); + if(ctx.apply_ctx->control.contracts_console()) + wlog( "misaligned array of const values" ); std::vector > copy(length > 0 ? length : 1); T* copy_ptr = ©[0]; memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); @@ -397,7 +399,8 @@ struct intrinsic_invoker_impl, size_t, Inputs...>, const auto length = size_t(size); T* base = array_ptr_impl(ctx, (U32)ptr, length); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned array of values" ); + if(ctx.apply_ctx->control.contracts_console()) + wlog( "misaligned array of values" ); std::vector > copy(length > 0 ? length : 1); T* copy_ptr = ©[0]; memcpy( (void*)copy_ptr, (void*)base, length * sizeof(T) ); @@ -508,7 +511,8 @@ struct intrinsic_invoker_impl, std::tuple std::enable_if_t::value, Ret> { T* base = array_ptr_impl(ctx, (U32)ptr, 1); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned const pointer" ); + if(ctx.apply_ctx->control.contracts_console()) + wlog( "misaligned const pointer" ); std::remove_const_t copy; T* copy_ptr = © memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); @@ -521,7 +525,8 @@ struct intrinsic_invoker_impl, std::tuple std::enable_if_t::value, Ret> { T* base = array_ptr_impl(ctx, (U32)ptr, 1); if ( reinterpret_cast(base) % alignof(T) != 0 ) { - wlog( "misaligned pointer" ); + if(ctx.apply_ctx->control.contracts_console()) + wlog( "misaligned pointer" ); std::remove_const_t copy; T* copy_ptr = © memcpy( (void*)copy_ptr, (void*)base, sizeof(T) ); @@ -587,7 +592,8 @@ struct intrinsic_invoker_impl, std::tuple(&base) % alignof(T) != 0 ) { - wlog( "misaligned const reference" ); + if(ctx.apply_ctx->control.contracts_console()) + wlog( "misaligned const reference" ); std::remove_const_t copy; T* copy_ptr = © memcpy( (void*)copy_ptr, (void*)&base, sizeof(T) ); @@ -605,7 +611,8 @@ struct intrinsic_invoker_impl, std::tuple(&base) % alignof(T) != 0 ) { - wlog( "misaligned reference" ); + if(ctx.apply_ctx->control.contracts_console()) + wlog( "misaligned reference" ); std::remove_const_t copy; T* copy_ptr = © memcpy( (void*)copy_ptr, (void*)&base, sizeof(T) ); diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index dd58f0364ec1e2c115d76887e16ad2dd52f03eac..806ce5d3f8ebda8c180928542a295201e3e73f41 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -23,7 +23,7 @@ namespace eosio { namespace chain { ,pseudo_start(s) { if (!c.skip_db_sessions()) { - undo_session = c.db().start_undo_session(true); + undo_session = c.mutable_db().start_undo_session(true); } trace->id = id; trace->block_num = c.pending_block_state()->block_num; @@ -451,7 +451,7 @@ namespace eosio { namespace chain { auto first_auth = trx.first_authorizor(); uint32_t trx_size = 0; - const auto& cgto = control.db().create( [&]( auto& gto ) { + const auto& cgto = control.mutable_db().create( [&]( auto& gto ) { gto.trx_id = id; gto.payer = first_auth; gto.sender = account_name(); /// delayed transactions have no sender @@ -467,7 +467,7 @@ namespace eosio { namespace chain { void transaction_context::record_transaction( const transaction_id_type& id, fc::time_point_sec expire ) { try { - control.db().create([&](transaction_object& transaction) { + control.mutable_db().create([&](transaction_object& transaction) { transaction.trx_id = id; transaction.expiration = expire; }); diff --git a/libraries/chain/wasm_interface.cpp b/libraries/chain/wasm_interface.cpp index ac58004527794fbe9bc27526c41fc2950cae31c1..67bf07430b1f8ce9f24959a457f4b7872321e3db 100644 --- a/libraries/chain/wasm_interface.cpp +++ b/libraries/chain/wasm_interface.cpp @@ -1911,8 +1911,6 @@ std::istream& operator>>(std::istream& in, wasm_interface::vm_type& runtime) { in >> s; if (s == "wavm") runtime = eosio::chain::wasm_interface::vm_type::wavm; - else if (s == "binaryen") - runtime = eosio::chain::wasm_interface::vm_type::binaryen; else if (s == "wabt") runtime = eosio::chain::wasm_interface::vm_type::wabt; else diff --git a/libraries/chain/webassembly/binaryen.cpp b/libraries/chain/webassembly/binaryen.cpp deleted file mode 100644 index ca3138db806ecf62fc9769af444a4b8c92cd2d6d..0000000000000000000000000000000000000000 --- a/libraries/chain/webassembly/binaryen.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#include -#include - -#include - - -namespace eosio { namespace chain { namespace webassembly { namespace binaryen { - -class binaryen_instantiated_module : public wasm_instantiated_module_interface { - public: - binaryen_instantiated_module(linear_memory_type& shared_linear_memory, - std::vector initial_memory, - call_indirect_table_type table, - import_lut_type import_lut, - unique_ptr&& module) : - _shared_linear_memory(shared_linear_memory), - _initial_memory(initial_memory), - _table(forward(table)), - _import_lut(forward(import_lut)), - _module(forward(module)) { - - } - - void apply(apply_context& context) override { - LiteralList args = {Literal(uint64_t(context.receiver)), - Literal(uint64_t(context.act.account)), - Literal(uint64_t(context.act.name))}; - call("apply", args, context); - } - - private: - linear_memory_type& _shared_linear_memory; - std::vector _initial_memory; - call_indirect_table_type _table; - import_lut_type _import_lut; - unique_ptr _module; - - void call(const string& entry_point, LiteralList& args, apply_context& context){ - const unsigned initial_memory_size = _module->memory.initial*Memory::kPageSize; - interpreter_interface local_interface(_shared_linear_memory, _table, _import_lut, initial_memory_size, context); - - //zero out the initial pages - memset(_shared_linear_memory.data, 0, initial_memory_size); - //copy back in the initial data - memcpy(_shared_linear_memory.data, _initial_memory.data(), _initial_memory.size()); - - //be aware that construction of the ModuleInstance implictly fires the start function - ModuleInstance instance(*_module.get(), &local_interface); - instance.callExport(Name(entry_point), args); - } -}; - -binaryen_runtime::binaryen_runtime() { - -} - -std::unique_ptr binaryen_runtime::instantiate_module(const char* code_bytes, size_t code_size, std::vector initial_memory) { - try { - vector code(code_bytes, code_bytes + code_size); - unique_ptr module(new Module()); - WasmBinaryBuilder builder(*module, code, false); - builder.read(); - - EOS_ASSERT(module->memory.initial * Memory::kPageSize <= wasm_constraints::maximum_linear_memory, binaryen_exception, "exceeds maximum linear memory"); - - // create a temporary globals to use - TrivialGlobalManager globals; - for (auto& global : module->globals) { - globals[global->name] = ConstantExpressionRunner(globals).visit(global->init).value; - } - - call_indirect_table_type table; - table.resize(module->table.initial); - for (auto& segment : module->table.segments) { - Address offset = ConstantExpressionRunner(globals).visit(segment.offset).value.geti32(); - EOS_ASSERT( uint64_t(offset) + segment.data.size() <= module->table.initial, binaryen_exception, ""); - for (size_t i = 0; i != segment.data.size(); ++i) { - table[offset + i] = segment.data[i]; - } - } - - // initialize the import lut - import_lut_type import_lut; - import_lut.reserve(module->imports.size()); - for (auto& import : module->imports) { - std::string full_name = string(import->module.c_str()) + "." + string(import->base.c_str()); - if (import->kind == ExternalKind::Function) { - auto& intrinsic_map = intrinsic_registrator::get_map(); - auto intrinsic_itr = intrinsic_map.find(full_name); - if (intrinsic_itr != intrinsic_map.end()) { - import_lut.emplace(make_pair((uintptr_t)import.get(), intrinsic_itr->second)); - continue; - } - } - - EOS_ASSERT( !"unresolvable", wasm_exception, "${module}.${export} unresolveable", ("module",import->module.c_str())("export",import->base.c_str()) ); - } - - return std::make_unique(_memory, initial_memory, move(table), move(import_lut), move(module)); - } catch (const ParseException &e) { - FC_THROW_EXCEPTION(wasm_execution_error, "Error building interpreter: ${s}", ("s", e.text)); - } -} - -}}}} diff --git a/libraries/chainbase b/libraries/chainbase index 4724baf2095cdc1bb1722254874b51070adf0e74..8ca96ad6b18709d65a7d1f67f8893978f25babcf 160000 --- a/libraries/chainbase +++ b/libraries/chainbase @@ -1 +1 @@ -Subproject commit 4724baf2095cdc1bb1722254874b51070adf0e74 +Subproject commit 8ca96ad6b18709d65a7d1f67f8893978f25babcf diff --git a/libraries/fc b/libraries/fc index f085773d29ecc457894170fae740b726465f1382..29cd7df702e79954076461af0eadad2e9d745d44 160000 --- a/libraries/fc +++ b/libraries/fc @@ -1 +1 @@ -Subproject commit f085773d29ecc457894170fae740b726465f1382 +Subproject commit 29cd7df702e79954076461af0eadad2e9d745d44 diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index 2e609616615e7b281f3837beef47ed3e13b72924..772ca726e317ac809edc9770eca072c59a06e1a0 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -332,9 +332,7 @@ namespace eosio { namespace testing { vcfg.genesis.initial_key = get_public_key( config::system_account_name, "active" ); for(int i = 0; i < boost::unit_test::framework::master_test_suite().argc; ++i) { - if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--binaryen")) - vcfg.wasm_runtime = chain::wasm_interface::vm_type::binaryen; - else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wavm")) + if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wavm")) vcfg.wasm_runtime = chain::wasm_interface::vm_type::wavm; else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wabt")) vcfg.wasm_runtime = chain::wasm_interface::vm_type::wabt; diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 4e9470837635f58c3e03566022dd6de1cecfb2c2..69c9d8f47e4a5959de950ad98918b62649490c9c 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -96,14 +96,10 @@ namespace eosio { namespace testing { cfg.genesis.initial_key = get_public_key( config::system_account_name, "active" ); for(int i = 0; i < boost::unit_test::framework::master_test_suite().argc; ++i) { - if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--binaryen")) - cfg.wasm_runtime = chain::wasm_interface::vm_type::binaryen; - else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wavm")) + if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wavm")) cfg.wasm_runtime = chain::wasm_interface::vm_type::wavm; else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wabt")) cfg.wasm_runtime = chain::wasm_interface::vm_type::wabt; - else - cfg.wasm_runtime = chain::wasm_interface::vm_type::binaryen; } open(); diff --git a/plugins/COMMUNITY.md b/plugins/COMMUNITY.md index 9d568adc9a19cd1c7d2678a99784f45196840de2..87f80792c1a7e20d77f1b8447af7daa1b1d0b2e3 100644 --- a/plugins/COMMUNITY.md +++ b/plugins/COMMUNITY.md @@ -6,10 +6,13 @@ Third parties are encouraged to make pull requests to this file (`develop` branc | Description | URL | | ----------- | --- | -| Watch for specific actions and send them to an HTTP URL | https://github.com/eosauthority/eosio-watcher-plugin | +| BP Heartbeat | https://github.com/bancorprotocol/eos-producer-heartbeat-plugin | +| ElasticSearch | https://github.com/EOSLaoMao/elasticsearch_plugin | | Kafka | https://github.com/TP-Lab/kafka_plugin | +| MySQL | https://github.com/eosBLACK/eosio_mysqldb_plugin | | SQL | https://github.com/asiniscalchi/eosio_sql_plugin | -| ElasticSearch | https://github.com/EOSLaoMao/elasticsearch_plugin | +| Watch for specific actions and send them to an HTTP URL | https://github.com/eosauthority/eosio-watcher-plugin | +| ZeroMQ | https://github.com/cc32d9/eos_zmq_plugin | ## DISCLAIMER: diff --git a/plugins/chain_api_plugin/chain_api_plugin.cpp b/plugins/chain_api_plugin/chain_api_plugin.cpp index 31e576ae4d87ed02db76e63e6fb9bb96816227cd..f07e48a7f0476c7fab8b45341420bf866ff479ac 100644 --- a/plugins/chain_api_plugin/chain_api_plugin.cpp +++ b/plugins/chain_api_plugin/chain_api_plugin.cpp @@ -37,7 +37,7 @@ struct async_result_visitor : public fc::visitor { #define CALL(api_name, api_handle, api_namespace, call_name, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ - [this, api_handle](string, string body, url_response_callback cb) mutable { \ + [api_handle](string, string body, url_response_callback cb) mutable { \ api_handle.validate(); \ try { \ if (body.empty()) body = "{}"; \ @@ -50,7 +50,7 @@ struct async_result_visitor : public fc::visitor { #define CALL_ASYNC(api_name, api_handle, api_namespace, call_name, call_result, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ - [this, api_handle](string, string body, url_response_callback cb) mutable { \ + [api_handle](string, string body, url_response_callback cb) mutable { \ if (body.empty()) body = "{}"; \ api_handle.validate(); \ api_handle.call_name(fc::json::from_string(body).as(),\ @@ -79,7 +79,10 @@ void chain_api_plugin::plugin_startup() { auto ro_api = app().get_plugin().get_read_only_api(); auto rw_api = app().get_plugin().get_read_write_api(); - app().get_plugin().add_api({ + auto& _http_plugin = app().get_plugin(); + ro_api.set_shorten_abi_errors( !_http_plugin.verbose_errors() ); + + _http_plugin.add_api({ CHAIN_RO_CALL(get_info, 200l), CHAIN_RO_CALL(get_block, 200), CHAIN_RO_CALL(get_block_header_state, 200), diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 2b48d979f6d5476a77147a760f622d4cbfa1fd99..a9d4ed063926400c70440187a194c71233939a15 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -212,7 +212,7 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip ("blocks-dir", bpo::value()->default_value("blocks"), "the location of the blocks directory (absolute path or relative to application data dir)") ("checkpoint", bpo::value>()->composing(), "Pairs of [BLOCK_NUM,BLOCK_ID] that should be enforced as checkpoints.") - ("wasm-runtime", bpo::value()->value_name("wavm/binaryen/wabt"), "Override default WASM runtime") + ("wasm-runtime", bpo::value()->value_name("wavm/wabt"), "Override default WASM runtime") ("abi-serializer-max-time-ms", bpo::value()->default_value(config::default_abi_serializer_max_time_ms), "Override default maximum ABI serialization time allowed in ms") ("chain-state-db-size-mb", bpo::value()->default_value(config::default_state_size / (1024 * 1024)), "Maximum size (in MiB) of the chain state database") @@ -1214,7 +1214,7 @@ static float64_t to_softfloat64( double d ) { return *reinterpret_cast(&d); } -static fc::variant get_global_row( const database& db, const abi_def& abi, const abi_serializer& abis, const fc::microseconds& abi_serializer_max_time_ms ) { +fc::variant get_global_row( const database& db, const abi_def& abi, const abi_serializer& abis, const fc::microseconds& abi_serializer_max_time_ms, bool shorten_abi_errors ) { const auto table_type = get_table_type(abi, N(global)); EOS_ASSERT(table_type == read_only::KEYi64, chain::contract_table_query_exception, "Invalid table type ${type} for table global", ("type",table_type)); @@ -1227,7 +1227,7 @@ static fc::variant get_global_row( const database& db, const abi_def& abi, const vector data; read_only::copy_inline_row(*it, data); - return abis.binary_to_variant(abis.get_table_type(N(global)), data, abi_serializer_max_time_ms); + return abis.binary_to_variant(abis.get_table_type(N(global)), data, abi_serializer_max_time_ms, shorten_abi_errors ); } read_only::get_producers_result read_only::get_producers( const read_only::get_producers_params& p ) const { @@ -1272,12 +1272,12 @@ read_only::get_producers_result read_only::get_producers( const read_only::get_p } copy_inline_row(*kv_index.find(boost::make_tuple(table_id->id, it->primary_key)), data); if (p.json) - result.rows.emplace_back(abis.binary_to_variant(abis.get_table_type(N(producers)), data, abi_serializer_max_time)); + result.rows.emplace_back( abis.binary_to_variant( abis.get_table_type(N(producers)), data, abi_serializer_max_time, shorten_abi_errors ) ); else result.rows.emplace_back(fc::variant(data)); } - result.total_producer_vote_weight = get_global_row(d, abi, abis, abi_serializer_max_time)["total_producer_vote_weight"].as_double(); + result.total_producer_vote_weight = get_global_row(d, abi, abis, abi_serializer_max_time, shorten_abi_errors)["total_producer_vote_weight"].as_double(); return result; } @@ -1665,7 +1665,7 @@ read_only::get_account_results read_only::get_account( const get_account_params& if ( it != idx.end() ) { vector data; copy_inline_row(*it, data); - result.total_resources = abis.binary_to_variant( "user_resources", data, abi_serializer_max_time ); + result.total_resources = abis.binary_to_variant( "user_resources", data, abi_serializer_max_time, shorten_abi_errors ); } } @@ -1676,7 +1676,7 @@ read_only::get_account_results read_only::get_account( const get_account_params& if ( it != idx.end() ) { vector data; copy_inline_row(*it, data); - result.self_delegated_bandwidth = abis.binary_to_variant( "delegated_bandwidth", data, abi_serializer_max_time ); + result.self_delegated_bandwidth = abis.binary_to_variant( "delegated_bandwidth", data, abi_serializer_max_time, shorten_abi_errors ); } } @@ -1687,7 +1687,7 @@ read_only::get_account_results read_only::get_account( const get_account_params& if ( it != idx.end() ) { vector data; copy_inline_row(*it, data); - result.refund_request = abis.binary_to_variant( "refund_request", data, abi_serializer_max_time ); + result.refund_request = abis.binary_to_variant( "refund_request", data, abi_serializer_max_time, shorten_abi_errors ); } } @@ -1698,7 +1698,7 @@ read_only::get_account_results read_only::get_account( const get_account_params& if ( it != idx.end() ) { vector data; copy_inline_row(*it, data); - result.voter_info = abis.binary_to_variant( "voter_info", data, abi_serializer_max_time ); + result.voter_info = abis.binary_to_variant( "voter_info", data, abi_serializer_max_time, shorten_abi_errors ); } } } @@ -1724,7 +1724,7 @@ read_only::abi_json_to_bin_result read_only::abi_json_to_bin( const read_only::a auto action_type = abis.get_action_type(params.action); EOS_ASSERT(!action_type.empty(), action_validate_exception, "Unknown action ${action} in contract ${contract}", ("action", params.action)("contract", params.code)); try { - result.binargs = abis.variant_to_binary(action_type, params.args, abi_serializer_max_time); + result.binargs = abis.variant_to_binary( action_type, params.args, abi_serializer_max_time, shorten_abi_errors ); } EOS_RETHROW_EXCEPTIONS(chain::invalid_action_args_exception, "'${args}' is invalid args for action '${action}' code '${code}'. expected '${proto}'", ("args", params.args)("action", params.action)("code", params.code)("proto", action_abi_to_variant(abi, action_type))) @@ -1741,7 +1741,7 @@ read_only::abi_bin_to_json_result read_only::abi_bin_to_json( const read_only::a abi_def abi; if( abi_serializer::to_abi(code_account.abi, abi) ) { abi_serializer abis( abi, abi_serializer_max_time ); - result.args = abis.binary_to_variant( abis.get_action_type( params.action ), params.binargs, abi_serializer_max_time ); + result.args = abis.binary_to_variant( abis.get_action_type( params.action ), params.binargs, abi_serializer_max_time, shorten_abi_errors ); } else { EOS_ASSERT(false, abi_not_found_exception, "No ABI found for ${contract}", ("contract", params.code)); } diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index 7997aad5b1b22130820e24f23294ae93983c52a0..d94d48d0785bfc82639dc8836779318124416ff8 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -68,6 +68,7 @@ uint64_t convert_to_type(const string& str, const string& desc); class read_only { const controller& db; const fc::microseconds abi_serializer_max_time; + bool shorten_abi_errors = true; public: static const string KEYi64; @@ -77,6 +78,8 @@ public: void validate() const {} + void set_shorten_abi_errors( bool f ) { shorten_abi_errors = f; } + using get_info_params = empty; struct get_info_results { @@ -434,7 +437,7 @@ public: copy_inline_row(*itr2, data); if (p.json) { - result.rows.emplace_back(abis.binary_to_variant(abis.get_table_type(p.table), data, abi_serializer_max_time)); + result.rows.emplace_back( abis.binary_to_variant( abis.get_table_type(p.table), data, abi_serializer_max_time, shorten_abi_errors ) ); } else { result.rows.emplace_back(fc::variant(data)); } @@ -495,7 +498,7 @@ public: copy_inline_row(*itr, data); if (p.json) { - result.rows.emplace_back(abis.binary_to_variant(abis.get_table_type(p.table), data, abi_serializer_max_time)); + result.rows.emplace_back( abis.binary_to_variant( abis.get_table_type(p.table), data, abi_serializer_max_time, shorten_abi_errors ) ); } else { result.rows.emplace_back(fc::variant(data)); } diff --git a/plugins/db_size_api_plugin/db_size_api_plugin.cpp b/plugins/db_size_api_plugin/db_size_api_plugin.cpp index 8dd5b50f48e60acd0ca6960d2cf2e6eb4245543a..13b717c0789c85244932526df760dd75ba8c19f3 100644 --- a/plugins/db_size_api_plugin/db_size_api_plugin.cpp +++ b/plugins/db_size_api_plugin/db_size_api_plugin.cpp @@ -36,7 +36,7 @@ void db_size_api_plugin::plugin_startup() { } db_size_stats db_size_api_plugin::get() { - chainbase::database& db = app().get_plugin().chain().db(); + const chainbase::database& db = app().get_plugin().chain().db(); db_size_stats ret; ret.free_bytes = db.get_segment_manager()->get_free_memory(); diff --git a/plugins/history_api_plugin/history_api_plugin.cpp b/plugins/history_api_plugin/history_api_plugin.cpp index 71f4370172833d4b4cb5cf879f59ded2508c4f63..bd78dede08612ea97b92c8f34ba6f40f089f2b4c 100644 --- a/plugins/history_api_plugin/history_api_plugin.cpp +++ b/plugins/history_api_plugin/history_api_plugin.cpp @@ -21,7 +21,7 @@ void history_api_plugin::plugin_initialize(const variables_map&) {} #define CALL(api_name, api_handle, api_namespace, call_name) \ {std::string("/v1/" #api_name "/" #call_name), \ - [this, api_handle](string, string body, url_response_callback cb) mutable { \ + [api_handle](string, string body, url_response_callback cb) mutable { \ try { \ if (body.empty()) body = "{}"; \ auto result = api_handle.call_name(fc::json::from_string(body).as()); \ diff --git a/plugins/history_plugin/history_plugin.cpp b/plugins/history_plugin/history_plugin.cpp index 3c3650b9e599f3d4c812e4ed266aa81aa17cb6f5..e3292672e446c6fee69c87f4d57e190db443cbbe 100644 --- a/plugins/history_plugin/history_plugin.cpp +++ b/plugins/history_plugin/history_plugin.cpp @@ -206,7 +206,7 @@ namespace eosio { void record_account_action( account_name n, const base_action_trace& act ) { auto& chain = chain_plug->chain(); - auto& db = chain.db(); + chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) const auto& idx = db.get_index(); auto itr = idx.lower_bound( boost::make_tuple( name(n.value+1), 0 ) ); @@ -227,7 +227,7 @@ namespace eosio { void on_system_action( const action_trace& at ) { auto& chain = chain_plug->chain(); - auto& db = chain.db(); + chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) if( at.act.name == N(newaccount) ) { const auto create = at.act.data_as(); @@ -256,7 +256,7 @@ namespace eosio { if( filter( at ) ) { //idump((fc::json::to_pretty_string(at))); auto& chain = chain_plug->chain(); - auto& db = chain.db(); + chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) db.create( [&]( auto& aho ) { auto ps = fc::raw::pack_size( at ); @@ -344,10 +344,12 @@ namespace eosio { EOS_ASSERT( my->chain_plug, chain::missing_chain_plugin_exception, "" ); auto& chain = my->chain_plug->chain(); - chain.db().add_index(); - chain.db().add_index(); - chain.db().add_index(); - chain.db().add_index(); + chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) + // TODO: Use separate chainbase database for managing the state of the history_plugin (or remove deprecated history_plugin entirely) + db.add_index(); + db.add_index(); + db.add_index(); + db.add_index(); my->applied_transaction_connection.emplace( chain.applied_transaction.connect( [&]( const transaction_trace_ptr& p ) { diff --git a/plugins/http_plugin/http_plugin.cpp b/plugins/http_plugin/http_plugin.cpp index 6a380da0d03d79750dc4c700f80e5e4ce638f694..d9be006bb45ba3f1e3a98927eaa2c5f1bd3099c0 100644 --- a/plugins/http_plugin/http_plugin.cpp +++ b/plugins/http_plugin/http_plugin.cpp @@ -606,4 +606,8 @@ namespace eosio { return (!my->listen_endpoint || my->listen_endpoint->address().is_loopback()); } + bool http_plugin::verbose_errors()const { + return verbose_http_errors; + } + } diff --git a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp index e78300c62405439ca4ca4053b96289c55bceaddf..7f9aedb01e4be43d2282dc74243ed719dbcc909b 100644 --- a/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp +++ b/plugins/http_plugin/include/eosio/http_plugin/http_plugin.hpp @@ -60,12 +60,12 @@ namespace eosio { * called with the response code and body. * * The handler will be called from the appbase application io_service - * thread. The callback can be called from any thread and will + * thread. The callback can be called from any thread and will * automatically propagate the call to the http thread. * * The HTTP service will run in its own thread with its own io_service to * make sure that HTTP request processing does not interfer with other - * plugins. + * plugins. */ class http_plugin : public appbase::plugin { @@ -85,7 +85,7 @@ namespace eosio { void add_handler(const string& url, const url_handler&); void add_api(const api_description& api) { - for (const auto& call : api) + for (const auto& call : api) add_handler(call.first, call.second); } @@ -95,6 +95,8 @@ namespace eosio { bool is_on_loopback() const; bool is_secure() const; + bool verbose_errors()const; + private: std::unique_ptr my; }; diff --git a/plugins/mongo_db_plugin/mongo_db_plugin.cpp b/plugins/mongo_db_plugin/mongo_db_plugin.cpp index f0e27401dfe6212559534210a9ba69cc4f621683..a23eeda311f4e524a19d883df650373b43163d5f 100644 --- a/plugins/mongo_db_plugin/mongo_db_plugin.cpp +++ b/plugins/mongo_db_plugin/mongo_db_plugin.cpp @@ -111,7 +111,8 @@ public: void remove_account_control( const account_name& name, const permission_name& permission ); /// @return true if act should be added to mongodb, false to skip it - bool filter_include( const chain::action_trace& action_trace ) const; + bool filter_include( const account_name& receiver, const action_name& act_name, + const vector& authorization ) const; void init(); void wipe_database(); @@ -127,6 +128,7 @@ public: bool filter_on_star = true; std::set filter_on; std::set filter_out; + bool update_blocks_via_block_num = false; bool store_blocks = true; bool store_block_states = true; bool store_transactions = true; @@ -217,20 +219,22 @@ const std::string mongo_db_plugin_impl::accounts_col = "accounts"; const std::string mongo_db_plugin_impl::pub_keys_col = "pub_keys"; const std::string mongo_db_plugin_impl::account_controls_col = "account_controls"; -bool mongo_db_plugin_impl::filter_include( const chain::action_trace& action_trace ) const { +bool mongo_db_plugin_impl::filter_include( const account_name& receiver, const action_name& act_name, + const vector& authorization ) const +{ bool include = false; if( filter_on_star ) { include = true; } else { - auto itr = std::find_if( filter_on.cbegin(), filter_on.cend(), [&action_trace]( const auto& filter ) { - return filter.match( action_trace.receipt.receiver, action_trace.act.name, 0 ); + auto itr = std::find_if( filter_on.cbegin(), filter_on.cend(), [&receiver, &act_name]( const auto& filter ) { + return filter.match( receiver, act_name, 0 ); } ); if( itr != filter_on.cend() ) { include = true; } else { - for( const auto& a : action_trace.act.authorization ) { - auto itr = std::find_if( filter_on.cbegin(), filter_on.cend(), [&action_trace, &a]( const auto& filter ) { - return filter.match( action_trace.receipt.receiver, action_trace.act.name, a.actor ); + for( const auto& a : authorization ) { + auto itr = std::find_if( filter_on.cbegin(), filter_on.cend(), [&receiver, &act_name, &a]( const auto& filter ) { + return filter.match( receiver, act_name, a.actor ); } ); if( itr != filter_on.cend() ) { include = true; @@ -241,15 +245,16 @@ bool mongo_db_plugin_impl::filter_include( const chain::action_trace& action_tra } if( !include ) { return false; } + if( filter_out.empty() ) { return true; } - auto itr = std::find_if( filter_out.cbegin(), filter_out.cend(), [&action_trace]( const auto& filter ) { - return filter.match( action_trace.receipt.receiver, action_trace.act.name, 0 ); + auto itr = std::find_if( filter_out.cbegin(), filter_out.cend(), [&receiver, &act_name]( const auto& filter ) { + return filter.match( receiver, act_name, 0 ); } ); if( itr != filter_out.cend() ) { return false; } - for( const auto& a : action_trace.act.authorization ) { - auto itr = std::find_if( filter_out.cbegin(), filter_out.cend(), [&action_trace, &a]( const auto& filter ) { - return filter.match( action_trace.receipt.receiver, action_trace.act.name, a.actor ); + for( const auto& a : authorization ) { + auto itr = std::find_if( filter_out.cbegin(), filter_out.cend(), [&receiver, &act_name, &a]( const auto& filter ) { + return filter.match( receiver, act_name, a.actor ); } ); if( itr != filter_out.cend() ) { return false; } } @@ -694,6 +699,27 @@ void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transacti using bsoncxx::builder::basic::make_array; namespace bbb = bsoncxx::builder::basic; + const auto& trx = t->trx; + + if( !filter_on_star || !filter_out.empty() ) { + bool include = false; + for( const auto& a : trx.actions ) { + if( filter_include( a.account, a.name, a.authorization ) ) { + include = true; + break; + } + } + if( !include ) { + for( const auto& a : trx.context_free_actions ) { + if( filter_include( a.account, a.name, a.authorization ) ) { + include = true; + break; + } + } + } + if( !include ) return; + } + auto trans_doc = bsoncxx::builder::basic::document{}; auto now = std::chrono::duration_cast( @@ -701,7 +727,6 @@ void mongo_db_plugin_impl::_process_accepted_transaction( const chain::transacti const auto& trx_id = t->id; const auto trx_id_str = trx_id.str(); - const auto& trx = t->trx; trans_doc.append( kvp( "trx_id", trx_id_str ) ); @@ -776,7 +801,8 @@ mongo_db_plugin_impl::add_action_trace( mongocxx::bulk_write& bulk_action_traces } bool added = false; - if( start_block_reached && store_action_traces && filter_include( atrace ) ) { + if( start_block_reached && store_action_traces && + filter_include( atrace.receipt.receiver, atrace.act.name, atrace.act.authorization ) ) { auto action_traces_doc = bsoncxx::builder::basic::document{}; const chain::base_action_trace& base = atrace; // without inline action traces @@ -930,9 +956,16 @@ void mongo_db_plugin_impl::_process_accepted_block( const chain::block_state_ptr block_state_doc.append( kvp( "createdAt", b_date{now} ) ); try { - if( !_block_states.update_one( make_document( kvp( "block_id", block_id_str ) ), - make_document( kvp( "$set", block_state_doc.view() ) ), update_opts ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block_state ${bid}", ("bid", block_id) ); + if( update_blocks_via_block_num ) { + if( !_block_states.update_one( make_document( kvp( "block_num", b_int32{static_cast(block_num)} ) ), + make_document( kvp( "$set", block_state_doc.view() ) ), update_opts ) ) { + EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block_state ${num}", ("num", block_num) ); + } + } else { + if( !_block_states.update_one( make_document( kvp( "block_id", block_id_str ) ), + make_document( kvp( "$set", block_state_doc.view() ) ), update_opts ) ) { + EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block_state ${bid}", ("bid", block_id) ); + } } } catch( ... ) { handle_mongo_exception( "block_states insert: " + json, __LINE__ ); @@ -963,9 +996,16 @@ void mongo_db_plugin_impl::_process_accepted_block( const chain::block_state_ptr block_doc.append( kvp( "createdAt", b_date{now} ) ); try { - if( !_blocks.update_one( make_document( kvp( "block_id", block_id_str ) ), - make_document( kvp( "$set", block_doc.view() ) ), update_opts ) ) { - EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block ${bid}", ("bid", block_id) ); + if( update_blocks_via_block_num ) { + if( !_blocks.update_one( make_document( kvp( "block_num", b_int32{static_cast(block_num)} ) ), + make_document( kvp( "$set", block_doc.view() ) ), update_opts ) ) { + EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block ${num}", ("num", block_num) ); + } + } else { + if( !_blocks.update_one( make_document( kvp( "block_id", block_id_str ) ), + make_document( kvp( "$set", block_doc.view() ) ), update_opts ) ) { + EOS_ASSERT( false, chain::mongo_db_insert_fail, "Failed to insert block ${bid}", ("bid", block_id) ); + } } } catch( ... ) { handle_mongo_exception( "blocks insert: " + json, __LINE__ ); @@ -1427,6 +1467,8 @@ void mongo_db_plugin::set_program_options(options_description& cli, options_desc "MongoDB URI connection string, see: https://docs.mongodb.com/master/reference/connection-string/." " If not specified then plugin is disabled. Default database 'EOS' is used if not specified in URI." " Example: mongodb://127.0.0.1:27017/EOS") + ("mongodb-update-via-block-num", bpo::value()->default_value(false), + "Update blocks/block_state with latest via block number so that duplicates are overwritten.") ("mongodb-store-blocks", bpo::value()->default_value(true), "Enables storing blocks in mongodb.") ("mongodb-store-block-states", bpo::value()->default_value(true), @@ -1476,6 +1518,9 @@ void mongo_db_plugin::plugin_initialize(const variables_map& options) if( options.count( "mongodb-block-start" )) { my->start_block_num = options.at( "mongodb-block-start" ).as(); } + if( options.count( "mongodb-update-via-block-num" )) { + my->update_blocks_via_block_num = options.at( "mongodb-update-via-block-num" ).as(); + } if( options.count( "mongodb-store-blocks" )) { my->store_blocks = options.at( "mongodb-store-blocks" ).as(); } diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 28ee6f47d83996ddff5e62d67a6a17dfd717f772..6ba10db2752eae8e7986ecbcb2ccf35c1aa68570 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -1556,7 +1556,7 @@ namespace eosio { else { c->last_handshake_recv.last_irreversible_block_num = msg.known_trx.pending; reset_lib_num (c); - start_sync(c, msg.known_blocks.pending); + start_sync(c, msg.known_trx.pending); } } diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index b9fbd0489d090a0b57fabd88bcc3c4fb9fb89998..85f1d4615e74cfef74ed92d5288a7ebf1ba452aa 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -514,9 +514,9 @@ void producer_plugin::set_program_options( ("greylist-account", boost::program_options::value>()->composing()->multitoken(), "account that can not access to extended CPU/NET virtual resources") ("produce-time-offset-us", boost::program_options::value()->default_value(0), - "offset of non last block producing time in micro second. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later") + "offset of non last block producing time in microseconds. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later") ("last-block-time-offset-us", boost::program_options::value()->default_value(0), - "offset of last block producing time in micro second. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later") + "offset of last block producing time in microseconds. Negative number results in blocks to go out sooner, and positive number results in blocks to go out later") ("incoming-defer-ratio", bpo::value()->default_value(1.0), "ratio between incoming transations and deferred transactions when both are exhausted") ; diff --git a/plugins/test_control_api_plugin/test_control_api_plugin.cpp b/plugins/test_control_api_plugin/test_control_api_plugin.cpp index 91d5535c7965d011aab7e1fc942352543c3f19ca..16510b0646058a0f0a8d03e8c9a9c562f2ca06ed 100644 --- a/plugins/test_control_api_plugin/test_control_api_plugin.cpp +++ b/plugins/test_control_api_plugin/test_control_api_plugin.cpp @@ -37,7 +37,7 @@ struct async_result_visitor : public fc::visitor { #define CALL(api_name, api_handle, api_namespace, call_name, http_response_code) \ {std::string("/v1/" #api_name "/" #call_name), \ - [this, api_handle](string, string body, url_response_callback cb) mutable { \ + [api_handle](string, string body, url_response_callback cb) mutable { \ try { \ if (body.empty()) body = "{}"; \ auto result = api_handle.call_name(fc::json::from_string(body).as()); \ diff --git a/plugins/wallet_plugin/se_wallet.cpp b/plugins/wallet_plugin/se_wallet.cpp index 6f612a1701383c93009df37a77e1998dd19f4d4a..6e1a4fe0e1799528727d54ac3296a4fb6df0d8e1 100644 --- a/plugins/wallet_plugin/se_wallet.cpp +++ b/plugins/wallet_plugin/se_wallet.cpp @@ -192,7 +192,6 @@ struct se_wallet_impl { return optional{}; fc::ecdsa_sig sig = ECDSA_SIG_new(); - BIGNUM *r = BN_new(), *s = BN_new(); CFErrorRef error = nullptr; CFDataRef digestData = CFDataCreateWithBytesNoCopy(nullptr, (UInt8*)d.data(), d.data_size(), kCFAllocatorNull); @@ -205,10 +204,8 @@ struct se_wallet_impl { } const UInt8* der_bytes = CFDataGetBytePtr(signature); - - BN_bin2bn(der_bytes+4, der_bytes[3], r); - BN_bin2bn(der_bytes+6+der_bytes[3], der_bytes[4+der_bytes[3]+1], s); - ECDSA_SIG_set0(sig, r, s); + long derSize = CFDataGetLength(signature); + d2i_ECDSA_SIG(&sig.obj, &der_bytes, derSize); public_key_data kd; compact_signature compact_sig; @@ -303,7 +300,7 @@ se_wallet::se_wallet() : my(new detail::se_wallet_impl()) { } unsigned int major, minor; if(sscanf(model, "MacBookPro%u,%u", &major, &minor) == 2) { - if(major >= 13 && minor >= 2) { + if((major >= 15) || (major >= 13 && minor >= 2)) { my->populate_existing_keys(); return; } diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 3457b5679b2e80123c242f72e9f092a82de913e0..b3a656c3b1063677b39256c59a6fc37d9e971dfc 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -3,3 +3,4 @@ add_subdirectory( cleos ) add_subdirectory( keosd ) add_subdirectory( eosio-launcher ) add_subdirectory( eosio-abigen ) +add_subdirectory( eosio-blocklog ) diff --git a/programs/cleos/httpc.cpp b/programs/cleos/httpc.cpp index 26b260b1a95848b2c59fa33cb7bfb3c36a01287f..b0b8acd574e90d64580f9caf2ab74487509b083a 100644 --- a/programs/cleos/httpc.cpp +++ b/programs/cleos/httpc.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -230,14 +231,7 @@ namespace eosio { namespace client { namespace http { } else { //https boost::asio::ssl::context ssl_context(boost::asio::ssl::context::sslv23_client); -#if defined( __APPLE__ ) - //TODO: this is undocumented/not supported; fix with keychain based approach - ssl_context.load_verify_file("/private/etc/ssl/cert.pem"); -#elif defined( _WIN32 ) - EOS_THROW(http_exception, "HTTPS on Windows not supported"); -#else - ssl_context.set_default_verify_paths(); -#endif + fc::add_platform_root_cas_to_context(ssl_context); boost::asio::ssl::stream socket(cp.context->ios, ssl_context); SSL_set_tlsext_host_name(socket.native_handle(), url.server.c_str()); diff --git a/programs/cleos/main.cpp b/programs/cleos/main.cpp index 474f160f4ef7ac6dbad358605da96ca911832328..1428c4d1a38cbc5ec0ae076e90e17264d63e36b7 100644 --- a/programs/cleos/main.cpp +++ b/programs/cleos/main.cpp @@ -166,7 +166,8 @@ bfs::path determine_home_directory() } string url = "http://127.0.0.1:8888/"; -string wallet_url = "http://127.0.0.1:8900/"; +string default_wallet_url = "unix://" + (determine_home_directory() / "eosio-wallet" / (string(key_store_executable_name) + ".sock")).string(); +string wallet_url; //to be set to default_wallet_url in main bool no_verify = false; vector headers; @@ -768,25 +769,22 @@ struct set_action_permission_subcommand { }; -bool local_port_used(const string& lo_address, uint16_t port) { +bool local_port_used() { using namespace boost::asio; io_service ios; - boost::asio::ip::tcp::endpoint endpoint(boost::asio::ip::address::from_string(lo_address), port); - boost::asio::ip::tcp::socket socket(ios); - boost::system::error_code ec = error::would_block; - //connecting/failing to connect to localhost should be always fast - don't care about timeouts - socket.async_connect(endpoint, [&](const boost::system::error_code& error) { ec = error; } ); - do { - ios.run_one(); - } while (ec == error::would_block); + local::stream_protocol::endpoint endpoint(wallet_url.substr(strlen("unix://"))); + local::stream_protocol::socket socket(ios); + boost::system::error_code ec; + socket.connect(endpoint, ec); + return !ec; } -void try_local_port( const string& lo_address, uint16_t port, uint32_t duration ) { +void try_local_port(uint32_t duration) { using namespace std::chrono; auto start_time = duration_cast( system_clock::now().time_since_epoch() ).count(); - while ( !local_port_used(lo_address, port)) { + while ( !local_port_used()) { if (duration_cast( system_clock::now().time_since_epoch()).count() - start_time > duration ) { std::cerr << "Unable to connect to keosd, if keosd is running please kill the process and try again.\n"; throw connection_exception(fc::log_messages{FC_LOG_MESSAGE(error, "Unable to connect to keosd")}); @@ -806,16 +804,11 @@ void ensure_keosd_running(CLI::App* app) { if (subapp->got_subcommand("listproducers") || subapp->got_subcommand("listbw") || subapp->got_subcommand("bidnameinfo")) // system list* do not require wallet return; } + if (wallet_url != default_wallet_url) + return; - auto parsed_url = parse_url(wallet_url); - auto resolved_url = resolve_url(context, parsed_url); - - if (!resolved_url.is_loopback) - return; - - for (const auto& addr: resolved_url.resolved_addresses) - if (local_port_used(addr, resolved_url.resolved_port)) // Hopefully taken by keosd - return; + if (local_port_used()) + return; boost::filesystem::path binPath = boost::dll::program_location(); binPath.remove_filename(); @@ -827,13 +820,15 @@ void ensure_keosd_running(CLI::App* app) { binPath.remove_filename().remove_filename().append("keosd").append(key_store_executable_name); } - const auto& lo_address = resolved_url.resolved_addresses.front(); if (boost::filesystem::exists(binPath)) { namespace bp = boost::process; binPath = boost::filesystem::canonical(binPath); vector pargs; - pargs.push_back("--http-server-address=" + lo_address + ":" + std::to_string(resolved_url.resolved_port)); + pargs.push_back("--http-server-address"); + pargs.push_back(""); + pargs.push_back("--https-server-address"); + pargs.push_back(""); ::boost::process::child keos(binPath, pargs, bp::std_in.close(), @@ -842,13 +837,12 @@ void ensure_keosd_running(CLI::App* app) { if (keos.running()) { std::cerr << binPath << " launched" << std::endl; keos.detach(); - try_local_port(lo_address, resolved_url.resolved_port, 2000); + try_local_port(2000); } else { - std::cerr << "No wallet service listening on " << lo_address << ":" - << std::to_string(resolved_url.resolved_port) << ". Failed to launch " << binPath << std::endl; + std::cerr << "No wallet service listening on " << wallet_url << ". Failed to launch " << binPath << std::endl; } } else { - std::cerr << "No wallet service listening on " << lo_address << ":" << std::to_string(resolved_url.resolved_port) + std::cerr << "No wallet service listening on " << ". Cannot automatically start keosd because keosd was not found." << std::endl; } } @@ -896,6 +890,7 @@ struct create_account_subcommand { string stake_net; string stake_cpu; uint32_t buy_ram_bytes_in_kbytes = 0; + uint32_t buy_ram_bytes = 0; string buy_ram_eos; bool transfer; bool simple; @@ -913,7 +908,9 @@ struct create_account_subcommand { createAccount->add_option("--stake-cpu", stake_cpu, (localized("The amount of EOS delegated for CPU bandwidth")))->required(); createAccount->add_option("--buy-ram-kbytes", buy_ram_bytes_in_kbytes, - (localized("The amount of RAM bytes to purchase for the new account in kibibytes (KiB), default is 8 KiB"))); + (localized("The amount of RAM bytes to purchase for the new account in kibibytes (KiB)"))); + createAccount->add_option("--buy-ram-bytes", buy_ram_bytes, + (localized("The amount of RAM bytes to purchase for the new account in bytes"))); createAccount->add_option("--buy-ram", buy_ram_eos, (localized("The amount of RAM bytes to purchase for the new account in EOS"))); createAccount->add_flag("--transfer", transfer, @@ -934,12 +931,10 @@ struct create_account_subcommand { } EOS_RETHROW_EXCEPTIONS(public_key_type_exception, "Invalid active public key: ${public_key}", ("public_key", active_key_str)); auto create = create_newaccount(creator, account_name, owner_key, active_key); if (!simple) { - if ( buy_ram_eos.empty() && buy_ram_bytes_in_kbytes == 0) { - std::cerr << "ERROR: Either --buy-ram or --buy-ram-kbytes with non-zero value is required" << std::endl; - return; - } + EOSC_ASSERT( buy_ram_eos.size() || buy_ram_bytes_in_kbytes || buy_ram_bytes, "ERROR: One of --buy-ram, --buy-ram-kbytes or --buy-ram-bytes should have non-zero value" ); + EOSC_ASSERT( !buy_ram_bytes_in_kbytes || !buy_ram_bytes, "ERROR: --buy-ram-kbytes and --buy-ram-bytes cannot be set at the same time" ); action buyram = !buy_ram_eos.empty() ? create_buyram(creator, account_name, to_asset(buy_ram_eos)) - : create_buyrambytes(creator, account_name, buy_ram_bytes_in_kbytes * 1024); + : create_buyrambytes(creator, account_name, (buy_ram_bytes_in_kbytes) ? (buy_ram_bytes_in_kbytes * 1024) : buy_ram_bytes); auto net = to_asset(stake_net); auto cpu = to_asset(stake_cpu); if ( net.get_amount() != 0 || cpu.get_amount() != 0 ) { @@ -1200,6 +1195,7 @@ struct delegate_bandwidth_subcommand { string stake_cpu_amount; string stake_storage_amount; string buy_ram_amount; + uint32_t buy_ram_bytes = 0; bool transfer = false; delegate_bandwidth_subcommand(CLI::App* actionRoot) { @@ -1209,6 +1205,7 @@ struct delegate_bandwidth_subcommand { delegate_bandwidth->add_option("stake_net_quantity", stake_net_amount, localized("The amount of EOS to stake for network bandwidth"))->required(); delegate_bandwidth->add_option("stake_cpu_quantity", stake_cpu_amount, localized("The amount of EOS to stake for CPU bandwidth"))->required(); delegate_bandwidth->add_option("--buyram", buy_ram_amount, localized("The amount of EOS to buyram")); + delegate_bandwidth->add_option("--buy-ram-bytes", buy_ram_bytes, localized("The amount of RAM to buy in number of bytes")); delegate_bandwidth->add_flag("--transfer", transfer, localized("Transfer voting power and right to unstake EOS to receiver")); add_standard_transaction_options(delegate_bandwidth); @@ -1220,12 +1217,11 @@ struct delegate_bandwidth_subcommand { ("stake_cpu_quantity", to_asset(stake_cpu_amount)) ("transfer", transfer); std::vector acts{create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(delegatebw), act_payload)}; - if (buy_ram_amount.length()) { - fc::variant act_payload2 = fc::mutable_variant_object() - ("payer", from_str) - ("receiver", receiver_str) - ("quant", to_asset(buy_ram_amount)); - acts.push_back(create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(buyram), act_payload2)); + EOSC_ASSERT( !(buy_ram_amount.size()) || !buy_ram_bytes, "ERROR: --buyram and --buy-ram-bytes cannot be set at the same time" ); + if (buy_ram_amount.size()) { + acts.push_back( create_buyram(from_str, receiver_str, to_asset(buy_ram_amount)) ); + } else if (buy_ram_bytes) { + acts.push_back( create_buyrambytes(from_str, receiver_str, buy_ram_bytes) ); } send_actions(std::move(acts)); }); @@ -1353,27 +1349,22 @@ struct buyram_subcommand { string receiver_str; string amount; bool kbytes = false; + bool bytes = false; buyram_subcommand(CLI::App* actionRoot) { auto buyram = actionRoot->add_subcommand("buyram", localized("Buy RAM")); buyram->add_option("payer", from_str, localized("The account paying for RAM"))->required(); buyram->add_option("receiver", receiver_str, localized("The account receiving bought RAM"))->required(); - buyram->add_option("amount", amount, localized("The amount of EOS to pay for RAM, or number of kbytes of RAM if --kbytes is set"))->required(); - buyram->add_flag("--kbytes,-k", kbytes, localized("buyram in number of kbytes")); + buyram->add_option("amount", amount, localized("The amount of EOS to pay for RAM, or number of bytes/kibibytes of RAM if --bytes/--kbytes is set"))->required(); + buyram->add_flag("--kbytes,-k", kbytes, localized("buyram in number of kibibytes (KiB)")); + buyram->add_flag("--bytes,-b", bytes, localized("buyram in number of bytes")); add_standard_transaction_options(buyram); buyram->set_callback([this] { - if (kbytes) { - fc::variant act_payload = fc::mutable_variant_object() - ("payer", from_str) - ("receiver", receiver_str) - ("bytes", fc::to_uint64(amount) * 1024ull); - send_actions({create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(buyrambytes), act_payload)}); + EOSC_ASSERT( !kbytes || !bytes, "ERROR: --kbytes and --bytes cannot be set at the same time" ); + if (kbytes || bytes) { + send_actions( { create_buyrambytes(from_str, receiver_str, fc::to_uint64(amount) * ((kbytes) ? 1024ull : 1ull)) } ); } else { - fc::variant act_payload = fc::mutable_variant_object() - ("payer", from_str) - ("receiver", receiver_str) - ("quant", to_asset(amount)); - send_actions({create_action({permission_level{from_str,config::active_name}}, config::system_account_name, N(buyram), act_payload)}); + send_actions( { create_buyram(from_str, receiver_str, to_asset(amount)) } ); } }); } @@ -1745,6 +1736,7 @@ int main( int argc, char** argv ) { bindtextdomain(locale_domain, locale_path); textdomain(locale_domain); context = eosio::client::http::create_http_context(); + wallet_url = default_wallet_url; CLI::App app{"Command Line Interface to EOSIO Client"}; app.require_subcommand(); diff --git a/programs/eosio-blocklog/CMakeLists.txt b/programs/eosio-blocklog/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..b883e493f858f1d25c3166d9eee0df01812b0a28 --- /dev/null +++ b/programs/eosio-blocklog/CMakeLists.txt @@ -0,0 +1,25 @@ +add_executable( eosio-blocklog main.cpp ) + +if( UNIX AND NOT APPLE ) + set(rt_library rt ) +endif() + +find_package( Gperftools QUIET ) +if( GPERFTOOLS_FOUND ) + message( STATUS "Found gperftools; compiling eosio-blocklog with TCMalloc") + list( APPEND PLATFORM_SPECIFIC_LIBS tcmalloc ) +endif() + +target_include_directories(eosio-blocklog PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) + +target_link_libraries( eosio-blocklog + PRIVATE appbase + PRIVATE eosio_chain fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) + +install( TARGETS + eosio-blocklog + + RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} +) diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bc2de200ed54779d3370b070c0a9b4a2a3426948 --- /dev/null +++ b/programs/eosio-blocklog/main.cpp @@ -0,0 +1,192 @@ +/** + * @file + * @copyright defined in eosio/LICENSE.txt + */ +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +using namespace eosio::chain; +namespace bfs = boost::filesystem; +namespace bpo = boost::program_options; +using bpo::options_description; +using bpo::variables_map; + +struct blocklog { + blocklog() + {} + + void read_log(); + void set_program_options(options_description& cli); + void initialize(const variables_map& options); + + bfs::path blocks_dir; + bfs::path output_file; + uint32_t first_block; + uint32_t last_block; + bool no_pretty_print; +}; + +void blocklog::read_log() { + block_log block_logger(blocks_dir); + const auto end = block_logger.read_head(); + EOS_ASSERT( end, block_log_exception, "No blocks found in block log" ); + EOS_ASSERT( end->block_num() > 1, block_log_exception, "Only one block found in block log" ); + + ilog( "existing block log contains block num 1 through block num ${n}", ("n",end->block_num()) ); + + optional reversible_blocks; + try { + reversible_blocks.emplace(blocks_dir / config::reversible_blocks_dir_name, chainbase::database::read_only, config::default_reversible_cache_size); + reversible_blocks->add_index(); + const auto& idx = reversible_blocks->get_index(); + auto first = idx.lower_bound(end->block_num()); + auto last = idx.rbegin(); + if (first != idx.end() && last != idx.rend()) + ilog( "existing reversible block num ${first} through block num ${last} ", ("first",first->get_block()->block_num())("last",last->get_block()->block_num()) ); + else { + elog( "no blocks available in reversible block database: only block_log blocks are available" ); + reversible_blocks.reset(); + } + } catch( const std::runtime_error& e ) { + if( std::string(e.what()) == "database dirty flag set" ) { + elog( "database dirty flag set (likely due to unclean shutdown): only block_log blocks are available" ); + } else if( std::string(e.what()) == "database metadata dirty flag set" ) { + elog( "database metadata dirty flag set (likely due to unclean shutdown): only block_log blocks are available" ); + } else { + throw; + } + } + + std::ofstream output_blocks; + std::ostream* out; + if (!output_file.empty()) { + output_blocks.open(output_file.generic_string().c_str()); + if (output_blocks.fail()) { + std::ostringstream ss; + ss << "Unable to open file '" << output_file.string() << "'"; + throw std::runtime_error(ss.str()); + } + out = &output_blocks; + } + else + out = &std::cout; + + uint32_t block_num = (first_block < 1) ? 1 : first_block; + signed_block_ptr next; + fc::variant pretty_output; + const fc::microseconds deadline = fc::seconds(10); + auto print_block = [&](signed_block_ptr& next) { + abi_serializer::to_variant(*next, + pretty_output, + []( account_name n ) { return optional(); }, + deadline); + const auto block_id = next->id(); + const uint32_t ref_block_prefix = block_id._hash[1]; + const auto enhanced_object = fc::mutable_variant_object + ("block_num",next->block_num()) + ("id", block_id) + ("ref_block_prefix", ref_block_prefix) + (pretty_output.get_object()); + fc::variant v(std::move(enhanced_object)); + if (no_pretty_print) + fc::json::to_stream(*out, v, fc::json::stringify_large_ints_and_doubles); + else + *out << fc::json::to_pretty_string(v) << "\n"; + }; + while((block_num <= last_block) && (next = block_logger.read_block_by_num( block_num ))) { + print_block(next); + ++block_num; + out->flush(); + } + if (!reversible_blocks) { + return; + } + const reversible_block_object* obj = nullptr; + while( (block_num <= last_block) && (obj = reversible_blocks->find(block_num)) ) { + auto next = obj->get_block(); + print_block(next); + ++block_num; + } +} + +void blocklog::set_program_options(options_description& cli) +{ + cli.add_options() + ("blocks-dir", bpo::value()->default_value("blocks"), + "the location of the blocks directory (absolute path or relative to the current directory)") + ("output-file,o", bpo::value(), + "the file to write the block log output to (absolute or relative path). If not specified then output is to stdout.") + ("first", bpo::value(&first_block)->default_value(1), + "the first block number to log") + ("last", bpo::value(&last_block)->default_value(std::numeric_limits::max()), + "the last block number (inclusive) to log") + ("no-pretty-print", bpo::bool_switch(&no_pretty_print)->default_value(false), + "Do not pretty print the output. Useful if piping to jq to improve performance.") + ("help", "Print this help message and exit.") + ; + +} + +void blocklog::initialize(const variables_map& options) { + try { + auto bld = options.at( "blocks-dir" ).as(); + if( bld.is_relative()) + blocks_dir = bfs::current_path() / bld; + else + blocks_dir = bld; + + if (options.count( "output-file" )) { + bld = options.at( "output-file" ).as(); + if( bld.is_relative()) + output_file = bfs::current_path() / bld; + else + output_file = bld; + } + } FC_LOG_AND_RETHROW() + +} + + +int main(int argc, char** argv) +{ + std::ios::sync_with_stdio(false); // for potential performance boost for large block log files + options_description cli ("eosio-blocklog command line options"); + try { + blocklog blog; + blog.set_program_options(cli); + variables_map vmap; + bpo::store(bpo::parse_command_line(argc, argv, cli), vmap); + bpo::notify(vmap); + if (vmap.count("help") > 0) { + cli.print(std::cerr); + return 0; + } + blog.initialize(vmap); + blog.read_log(); + } catch( const fc::exception& e ) { + elog( "${e}", ("e", e.to_detail_string())); + return -1; + } catch( const boost::exception& e ) { + elog("${e}", ("e",boost::diagnostic_information(e))); + return -1; + } catch( const std::exception& e ) { + elog("${e}", ("e",e.what())); + return -1; + } catch( ... ) { + elog("unknown exception"); + return -1; + } + + return 0; +} diff --git a/programs/eosio-launcher/main.cpp b/programs/eosio-launcher/main.cpp index 73cba43f33738080c540e429c0f3d2e7d739c031..597e473b6773f31be3608c3b5ce898dd27cf5309 100644 --- a/programs/eosio-launcher/main.cpp +++ b/programs/eosio-launcher/main.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -387,63 +388,65 @@ string producer_names::producer_name(unsigned int producer_number) { } struct launcher_def { - bool force_overwrite; - size_t total_nodes; - size_t prod_nodes; - size_t producers; - size_t next_node; - string shape; - p2p_plugin p2p; - allowed_connection allowed_connections = PC_NONE; - bfs::path genesis; - bfs::path output; - bfs::path host_map_file; - bfs::path server_ident_file; - bfs::path stage; - - string erd; - bfs::path config_dir_base; - bfs::path data_dir_base; - bool skip_transaction_signatures = false; - string eosd_extra_args; - std::map specific_nodeos_args; - testnet_def network; - string gelf_endpoint; - vector aliases; - vector bindings; - int per_host = 0; - last_run_def last_run; - int start_delay = 0; - bool gelf_enabled; - bool nogen; - bool boot; - bool add_enable_stale_production = false; - string launch_name; - string launch_time; - server_identities servers; - producer_set_def producer_set; - vector genesis_block; + bool force_overwrite; + size_t total_nodes; + size_t prod_nodes; + size_t producers; + size_t next_node; + string shape; + p2p_plugin p2p; + allowed_connection allowed_connections = PC_NONE; + bfs::path genesis; + bfs::path output; + bfs::path host_map_file; + bfs::path server_ident_file; + bfs::path stage; + + string erd; + bfs::path config_dir_base; + bfs::path data_dir_base; + bool skip_transaction_signatures = false; + string eosd_extra_args; + std::map specific_nodeos_args; + testnet_def network; + string gelf_endpoint; + vector aliases; + vector bindings; + int per_host = 0; + last_run_def last_run; + int start_delay = 0; + bool gelf_enabled; + bool nogen; + bool boot; + bool add_enable_stale_production = false; + string launch_name; + string launch_time; + server_identities servers; + producer_set_def producer_set; string start_temp; string start_script; + fc::optional max_block_cpu_usage; + fc::optional max_transaction_cpu_usage; + eosio::chain::genesis_state genesis_from_file; void assign_name (eosd_def &node, bool is_bios); - void set_options (bpo::options_description &cli); - void initialize (const variables_map &vmap); + void set_options (bpo::options_description &cli); + void initialize (const variables_map &vmap); void init_genesis (); - void load_servers (); - bool generate (); - void define_network (); - void bind_nodes (); - host_def *find_host (const string &name); - host_def *find_host_by_name_or_address (const string &name); - host_def *deploy_config_files (tn_node_def &node); - string compose_scp_command (const host_def &host, const bfs::path &source, - const bfs::path &destination); - void write_config_file (tn_node_def &node); - void write_logging_config_file (tn_node_def &node); - void write_genesis_file (tn_node_def &node); - void write_setprods_file (); + void load_servers (); + bool generate (); + void define_network (); + void bind_nodes (); + host_def *find_host (const string &name); + host_def *find_host_by_name_or_address (const string &name); + host_def *deploy_config_files (tn_node_def &node); + string compose_scp_command (const host_def &host, const bfs::path &source, + const bfs::path &destination); + void write_config_file (tn_node_def &node); + void write_logging_config_file (tn_node_def &node); + void write_genesis_file (tn_node_def &node); + void write_setprods_file (); void write_bios_boot (); bool is_bios_ndx (size_t ndx); @@ -451,25 +454,25 @@ struct launcher_def { bool next_ndx(size_t &ndx); size_t skip_ndx (size_t from, size_t offset); - void make_ring (); - void make_star (); - void make_mesh (); - void make_custom (); - void write_dot_file (); - void format_ssh (const string &cmd, const string &host_name, string &ssh_cmd_line); - void do_command(const host_def& host, const string& name, vector> env_pairs, const string& cmd); - bool do_ssh (const string &cmd, const string &host_name); - void prep_remote_config_dir (eosd_def &node, host_def *host); - void launch (eosd_def &node, string >s); - void kill (launch_modes mode, string sig_opt); - static string get_node_num(uint16_t node_num); - pair find_node(uint16_t node_num); - vector> get_nodes(const string& node_number_list); - void bounce (const string& node_numbers); - void down (const string& node_numbers); - void roll (const string& host_names); - void start_all (string >s, launch_modes mode); - void ignite (); + void make_ring (); + void make_star (); + void make_mesh (); + void make_custom (); + void write_dot_file (); + void format_ssh (const string &cmd, const string &host_name, string &ssh_cmd_line); + void do_command(const host_def& host, const string& name, vector> env_pairs, const string& cmd); + bool do_ssh (const string &cmd, const string &host_name); + void prep_remote_config_dir (eosd_def &node, host_def *host); + void launch (eosd_def &node, string >s); + void kill (launch_modes mode, string sig_opt); + static string get_node_num(uint16_t node_num); + pair find_node(uint16_t node_num); + vector> get_nodes(const string& node_number_list); + void bounce (const string& node_numbers); + void down (const string& node_numbers); + void roll (const string& host_names); + void start_all (string >s, launch_modes mode); + void ignite (); }; void @@ -482,7 +485,7 @@ launcher_def::set_options (bpo::options_description &cfg) { ("mode,m",bpo::value>()->multitoken()->default_value({"any"}, "any"),"connection mode, combination of \"any\", \"producers\", \"specified\", \"none\"") ("shape,s",bpo::value(&shape)->default_value("star"),"network topology, use \"star\" \"mesh\" or give a filename for custom") ("p2p-plugin", bpo::value()->default_value("net"),"select a p2p plugin to use (either net or bnet). Defaults to net.") - ("genesis,g",bpo::value(&genesis)->default_value("./genesis.json"),"set the path to genesis.json") + ("genesis,g",bpo::value()->default_value("./genesis.json"),"set the path to genesis.json") ("skip-signature", bpo::bool_switch(&skip_transaction_signatures)->default_value(false), "nodeos does not require transaction signatures.") ("nodeos", bpo::value(&eosd_extra_args), "forward nodeos command line argument(s) to each instance of nodeos, enclose arg(s) in quotes") ("specific-num", bpo::value>()->composing(), "forward nodeos command line argument(s) (using \"--specific-nodeos\" flag) to this specific instance of nodeos. This parameter can be entered multiple times and requires a paired \"--specific-nodeos\" flag") @@ -490,14 +493,16 @@ launcher_def::set_options (bpo::options_description &cfg) { ("delay,d",bpo::value(&start_delay)->default_value(0),"seconds delay before starting each node after the first") ("boot",bpo::bool_switch(&boot)->default_value(false),"After deploying the nodes and generating a boot script, invoke it.") ("nogen",bpo::bool_switch(&nogen)->default_value(false),"launch nodes without writing new config files") - ("host-map",bpo::value(&host_map_file)->default_value(""),"a file containing mapping specific nodes to hosts. Used to enhance the custom shape argument") - ("servers",bpo::value(&server_ident_file)->default_value(""),"a file containing ip addresses and names of individual servers to deploy as producers or non-producers ") + ("host-map",bpo::value(),"a file containing mapping specific nodes to hosts. Used to enhance the custom shape argument") + ("servers",bpo::value(),"a file containing ip addresses and names of individual servers to deploy as producers or non-producers ") ("per-host",bpo::value(&per_host)->default_value(0),"specifies how many nodeos instances will run on a single host. Use 0 to indicate all on one.") ("network-name",bpo::value(&network.name)->default_value("testnet_"),"network name prefix used in GELF logging source") ("enable-gelf-logging",bpo::value(&gelf_enabled)->default_value(true),"enable gelf logging appender in logging configuration file") ("gelf-endpoint",bpo::value(&gelf_endpoint)->default_value("10.160.11.21:12201"),"hostname:port or ip:port of GELF endpoint") ("template",bpo::value(&start_temp)->default_value("testnet.template"),"the startup script template") ("script",bpo::value(&start_script)->default_value("bios_boot.sh"),"the generated startup script name") + ("max-block-cpu-usage",bpo::value(),"Provide the \"max-block-cpu-usage\" value to use in the genesis.json file") + ("max-transaction-cpu-usage",bpo::value(),"Provide the \"max-transaction-cpu-usage\" value to use in the genesis.json file") ; } @@ -529,6 +534,22 @@ launcher_def::initialize (const variables_map &vmap) { } } + if (vmap.count("max-block-cpu-usage")) { + max_block_cpu_usage = vmap["max-block-cpu-usage"].as(); + } + + if (vmap.count("max-transaction-cpu-usage")) { + max_transaction_cpu_usage = vmap["max-transaction-cpu-usage"].as(); + } + + genesis = vmap["genesis"].as(); + if (vmap.count("host-map")) { + host_map_file = vmap["host-map"].as(); + } + if (vmap.count("servers")) { + server_ident_file = vmap["servers"].as(); + } + if (vmap.count("specific-num")) { const auto specific_nums = vmap["specific-num"].as>(); const auto specific_args = vmap["specific-nodeos"].as>(); @@ -1157,27 +1178,20 @@ launcher_def::write_logging_config_file(tn_node_def &node) { void launcher_def::init_genesis () { - bfs::path genesis_path = bfs::current_path() / "genesis.json"; - bfs::ifstream src(genesis_path); - if (!src.good()) { + const bfs::path genesis_path = genesis.is_complete() ? genesis : bfs::current_path() / genesis; + if (!bfs::exists(genesis_path)) { cout << "generating default genesis file " << genesis_path << endl; eosio::chain::genesis_state default_genesis; fc::json::save_to_file( default_genesis, genesis_path, true ); - src.open(genesis_path); } string bioskey = string(network.nodes["bios"].keys[0].get_public_key()); - string str; - string prefix("initial_key"); - while(getline(src,str)) { - size_t pos = str.find(prefix); - if (pos != string::npos) { - size_t cut = str.find("EOS",pos); - genesis_block.push_back(str.substr(0,cut) + bioskey + "\","); - } - else { - genesis_block.push_back(str); - } - } + + fc::json::from_file(genesis_path).as(genesis_from_file); + genesis_from_file.initial_key = public_key_type(bioskey); + if (max_block_cpu_usage) + genesis_from_file.initial_configuration.max_block_cpu_usage = *max_block_cpu_usage; + if (max_transaction_cpu_usage) + genesis_from_file.initial_configuration.max_transaction_cpu_usage = *max_transaction_cpu_usage; } void @@ -1191,10 +1205,7 @@ launcher_def::write_genesis_file(tn_node_def &node) { } filename = dd / "genesis.json"; - bfs::ofstream gf ( dd / "genesis.json"); - for (auto &line : genesis_block) { - gf << line << "\n"; - } + fc::json::save_to_file( genesis_from_file, dd / "genesis.json", true ); } void @@ -1707,6 +1718,13 @@ launcher_def::bounce (const string& node_numbers) { const string node_num = node.get_node_num(); cout << "Bouncing " << node.name << endl; string cmd = "./scripts/eosio-tn_bounce.sh " + eosd_extra_args; + if (node_num != "bios" && !specific_nodeos_args.empty()) { + const auto node_num_i = boost::lexical_cast(node_num); + if (specific_nodeos_args.count(node_num_i)) { + cmd += " " + specific_nodeos_args[node_num_i]; + } + } + do_command(host, node.name, { { "EOSIO_HOME", host.eosio_home }, { "EOSIO_NODE", node_num } }, cmd); } } diff --git a/programs/keosd/main.cpp b/programs/keosd/main.cpp index 58a42d96b3075685bf82e7198f5d49bae43718cd..c457af3c0e398ac80a62fccf2f38a29a721e96f4 100644 --- a/programs/keosd/main.cpp +++ b/programs/keosd/main.cpp @@ -43,7 +43,7 @@ int main(int argc, char** argv) http_plugin::set_defaults({ .address_config_prefix = "", .default_unix_socket_path = keosd::config::key_store_executable_name + ".sock", - .default_http_port = 8900 + .default_http_port = 0 }); app().register_plugin(); if(!app().initialize(argc, argv)) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 417eb35f07c1f22f08283721eb4232cc50327685..cc9eec4e538063684132e2f0ebf76f9089546a40 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -59,7 +59,7 @@ add_test(NAME p2p_dawn515_test COMMAND tests/p2p_tests/dawn_515/test.sh WORKING_ set_property(TEST p2p_dawn515_test PROPERTY LABELS nonparallelizable_tests) if(BUILD_MONGO_DB_PLUGIN) add_test(NAME nodeos_run_test-mongodb COMMAND tests/nodeos_run_test.py --mongodb -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - set_property(TEST nodeos_run_test PROPERTY LABELS nonparallelizable_tests) + set_property(TEST nodeos_run_test-mongodb PROPERTY LABELS nonparallelizable_tests) endif() add_test(NAME distributed-transactions-test COMMAND tests/distributed-transactions-test.py -d 2 -p 1 -n 4 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) @@ -81,20 +81,22 @@ add_test(NAME nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity set_property(TEST nodeos_sanity_lr_test PROPERTY LABELS long_running_tests) add_test(NAME bnet_nodeos_sanity_lr_test COMMAND tests/nodeos_run_test.py -v --sanity-test --p2p-plugin bnet --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST bnet_nodeos_sanity_lr_test PROPERTY LABELS long_running_tests) +add_test(NAME nodeos_run_check_lr_test COMMAND tests/nodeos_run_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +set_property(TEST nodeos_run_check_lr_test PROPERTY LABELS long_running_tests) #add_test(NAME distributed_transactions_lr_test COMMAND tests/distributed-transactions-test.py -d 2 -p 21 -n 21 -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) #set_property(TEST distributed_transactions_lr_test PROPERTY LABELS long_running_tests) -add_test(NAME nodeos_forked_chain_lr_test COMMAND tests/nodeos_forked_chain_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME nodeos_forked_chain_lr_test COMMAND tests/nodeos_forked_chain_test.py -v --wallet-port 9901 --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_forked_chain_lr_test PROPERTY LABELS long_running_tests) -add_test(NAME nodeos_voting_lr_test COMMAND tests/nodeos_voting_test.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME nodeos_voting_lr_test COMMAND tests/nodeos_voting_test.py -v --wallet-port 9902 --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_voting_lr_test PROPERTY LABELS long_running_tests) -add_test(NAME bnet_nodeos_voting_lr_test COMMAND tests/nodeos_voting_test.py -v --p2p-plugin bnet --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME bnet_nodeos_voting_lr_test COMMAND tests/nodeos_voting_test.py -v --wallet-port 9903 --p2p-plugin bnet --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST bnet_nodeos_voting_lr_test PROPERTY LABELS long_running_tests) -add_test(NAME nodeos_under_min_avail_ram_lr_test COMMAND tests/nodeos_under_min_avail_ram.py -v --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_test(NAME nodeos_under_min_avail_ram_lr_test COMMAND tests/nodeos_under_min_avail_ram.py -v --wallet-port 9904 --clean-run --dump-error-detail WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST nodeos_under_min_avail_ram_lr_test PROPERTY LABELS long_running_tests) diff --git a/tests/Cluster.py b/tests/Cluster.py index 8a9a1cb4b8ff3c848dcb82c15217c888855fb3a6..b8041227c3f02e3756682522f42bc9b52ce5aa33 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -4,7 +4,6 @@ import time import glob import shutil import os -import platform import re import string import signal @@ -12,8 +11,6 @@ import datetime import sys import random import json -import socket -import errno from core_symbol import CORE_SYMBOL from testUtils import Utils @@ -32,6 +29,7 @@ class Cluster(object): __BiosHost="localhost" __BiosPort=8788 __LauncherCmdArr=[] + __bootlog="eosio-ignition-wd/bootlog.txt" # pylint: disable=too-many-arguments # walletd [True|False] Is keosd running. If not load the wallet plugin @@ -64,9 +62,6 @@ class Cluster(object): self.port=port self.walletHost=walletHost self.walletPort=walletPort - self.walletEndpointArgs="" - if self.walletd: - self.walletEndpointArgs += " --wallet-url http://%s:%d" % (self.walletHost, self.walletPort) self.mongoEndpointArgs="" self.mongoUri="" if self.enableMongo: @@ -84,6 +79,9 @@ class Cluster(object): self.defproducerbAccount.ownerPrivateKey=defproducerbPrvtKey self.defproducerbAccount.activePrivateKey=defproducerbPrvtKey + self.useBiosBootFile=False + self.filesToCleanup=[] + def setChainStrategy(self, chainSyncStrategy=Utils.SyncReplayTag): self.__chainSyncStrategy=self.__chainSyncStrategies.get(chainSyncStrategy) @@ -98,8 +96,8 @@ class Cluster(object): # pylint: disable=too-many-return-statements # pylint: disable=too-many-branches # pylint: disable=too-many-statements - def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", p2pPlugin="net", delay=1, onlyBios=False, dontKill=False - , dontBootstrap=False, totalProducers=None, extraNodeosArgs=None, useBiosBootFile=True, specificExtraNodeosArgs=None): + def launch(self, pnodes=1, totalNodes=1, prodCount=1, topo="mesh", p2pPlugin="net", delay=1, onlyBios=False, dontBootstrap=False, + totalProducers=None, extraNodeosArgs=None, useBiosBootFile=True, specificExtraNodeosArgs=None): """Launch cluster. pnodes: producer nodes count totalNodes: producer + non-producer nodes count @@ -125,13 +123,16 @@ class Cluster(object): if len(self.nodes) > 0: raise RuntimeError("Cluster already running.") + if self.walletMgr is None: + self.walletMgr=WalletMgr(True) + producerFlag="" if totalProducers: assert(isinstance(totalProducers, (str,int))) producerFlag="--producers %s" % (totalProducers) tries = 30 - while not Cluster.arePortsAvailable(set(range(self.port, self.port+totalNodes+1))): + while not Utils.arePortsAvailable(set(range(self.port, self.port+totalNodes+1))): Utils.Print("ERROR: Another process is listening on nodeos default port. wait...") if tries == 0: return False @@ -145,7 +146,7 @@ class Cluster(object): if self.staging: cmdArr.append("--nogen") - nodeosArgs="--max-transaction-time 990000 --abi-serializer-max-time-ms 990000 --filter-on * --p2p-max-nodes-per-host %d" % (totalNodes) + nodeosArgs="--max-transaction-time -1 --abi-serializer-max-time-ms 990000 --filter-on * --p2p-max-nodes-per-host %d" % (totalNodes) if not self.walletd: nodeosArgs += " --plugin eosio::wallet_api_plugin" if self.enableMongo: @@ -170,6 +171,11 @@ class Cluster(object): cmdArr.append("--specific-nodeos") cmdArr.append(arg) + cmdArr.append("--max-block-cpu-usage") + cmdArr.append(str(160000000)) + cmdArr.append("--max-transaction-cpu-usage") + cmdArr.append(str(150000000)) + # must be last cmdArr.append before subprocess.call, so that everything is on the command line # before constructing the shape.json file for "bridge" if topo=="bridge": @@ -310,8 +316,7 @@ class Cluster(object): self.nodes=nodes if onlyBios: - biosNode=Node(Cluster.__BiosHost, Cluster.__BiosPort) - biosNode.setWalletEndpointArgs(self.walletEndpointArgs) + biosNode=Node(Cluster.__BiosHost, Cluster.__BiosPort, walletMgr=self.walletMgr) if not biosNode.checkPulse(): Utils.Print("ERROR: Bios node doesn't appear to be running...") return False @@ -330,12 +335,13 @@ class Cluster(object): Utils.Print("Bootstrap cluster.") if onlyBios or not useBiosBootFile: - self.biosNode=Cluster.bootstrap(totalNodes, prodCount, totalProducers, Cluster.__BiosHost, Cluster.__BiosPort, dontKill, onlyBios) + self.biosNode=Cluster.bootstrap(totalNodes, prodCount, totalProducers, Cluster.__BiosHost, Cluster.__BiosPort, self.walletMgr, onlyBios) if self.biosNode is None: Utils.Print("ERROR: Bootstrap failed.") return False else: - self.biosNode=Cluster.bios_bootstrap(totalNodes, Cluster.__BiosHost, Cluster.__BiosPort, dontKill) + self.useBiosBootFile=True + self.biosNode=Cluster.bios_bootstrap(totalNodes, Cluster.__BiosHost, Cluster.__BiosPort, self.walletMgr) if self.biosNode is None: Utils.Print("ERROR: Bootstrap failed.") return False @@ -366,40 +372,11 @@ class Cluster(object): return True - @staticmethod - def arePortsAvailable(ports): - """Check if specified ports are available for listening on.""" - assert(ports) - assert(isinstance(ports, set)) - - for port in ports: - if Utils.Debug: Utils.Print("Checking if port %d is available." % (port)) - assert(isinstance(port, int)) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - try: - s.bind(("127.0.0.1", port)) - except socket.error as e: - if e.errno == errno.EADDRINUSE: - Utils.Print("ERROR: Port %d is already in use" % (port)) - else: - # something else raised the socket.error exception - Utils.Print("ERROR: Unknown exception while trying to listen on port %d" % (port)) - Utils.Print(e) - return False - finally: - s.close() - - return True - - # Initialize the default nodes (at present just the root node) def initializeNodes(self, defproduceraPrvtKey=None, defproducerbPrvtKey=None, onlyBios=False): port=Cluster.__BiosPort if onlyBios else self.port host=Cluster.__BiosHost if onlyBios else self.host - node=Node(host, port, enableMongo=self.enableMongo, mongoHost=self.mongoHost, mongoPort=self.mongoPort, mongoDb=self.mongoDb) - node.setWalletEndpointArgs(self.walletEndpointArgs) + node=Node(host, port, walletMgr=self.walletMgr, enableMongo=self.enableMongo, mongoHost=self.mongoHost, mongoPort=self.mongoPort, mongoDb=self.mongoDb) if Utils.Debug: Utils.Print("Node: %s", str(node)) node.checkPulse(exitOnError=True) @@ -438,8 +415,7 @@ class Cluster(object): for n in nArr: port=n["port"] host=n["host"] - node=Node(host, port) - node.setWalletEndpointArgs(self.walletEndpointArgs) + node=Node(host, port, walletMgr=self.walletMgr) if Utils.Debug: Utils.Print("Node:", node) node.checkPulse(exitOnError=True) @@ -847,11 +823,11 @@ class Cluster(object): return producerKeys @staticmethod - def bios_bootstrap(totalNodes, biosHost, biosPort, dontKill=False): + def bios_bootstrap(totalNodes, biosHost, biosPort, walletMgr): """Bootstrap cluster using the bios_boot.sh script generated by eosio-launcher.""" Utils.Print("Starting cluster bootstrap.") - biosNode=Node(biosHost, biosPort) + biosNode=Node(biosHost, biosPort, walletMgr=walletMgr) if not biosNode.checkPulse(): Utils.Print("ERROR: Bios node doesn't appear to be running...") return None @@ -863,11 +839,10 @@ class Cluster(object): return None p = re.compile('error', re.IGNORECASE) - bootlog="eosio-ignition-wd/bootlog.txt" - with open(bootlog) as bootFile: + with open(Cluster.__bootlog) as bootFile: for line in bootFile: if p.search(line): - Utils.Print("ERROR: bios_boot.sh script resulted in errors. See %s" % (bootlog)) + Utils.Print("ERROR: bios_boot.sh script resulted in errors. See %s" % (Cluster.__bootlog)) Utils.Print(line) return None @@ -877,66 +852,59 @@ class Cluster(object): Utils.Print("ERROR: Failed to parse private keys from cluster config files.") return None - walletMgr=WalletMgr(True) walletMgr.killall() walletMgr.cleanup() if not walletMgr.launch(): Utils.Print("ERROR: Failed to launch bootstrap wallet.") return None - biosNode.setWalletEndpointArgs(walletMgr.walletEndpointArgs) - try: - ignWallet=walletMgr.create("ignition") - if ignWallet is None: - Utils.Print("ERROR: Failed to create ignition wallet.") - return None + ignWallet=walletMgr.create("ignition") + if ignWallet is None: + Utils.Print("ERROR: Failed to create ignition wallet.") + return None - eosioName="eosio" - eosioKeys=producerKeys[eosioName] - eosioAccount=Account(eosioName) - eosioAccount.ownerPrivateKey=eosioKeys["private"] - eosioAccount.ownerPublicKey=eosioKeys["public"] - eosioAccount.activePrivateKey=eosioKeys["private"] - eosioAccount.activePublicKey=eosioKeys["public"] - producerKeys.pop(eosioName) - - if not walletMgr.importKey(eosioAccount, ignWallet): - Utils.Print("ERROR: Failed to import %s account keys into ignition wallet." % (eosioName)) - return None + eosioName="eosio" + eosioKeys=producerKeys[eosioName] + eosioAccount=Account(eosioName) + eosioAccount.ownerPrivateKey=eosioKeys["private"] + eosioAccount.ownerPublicKey=eosioKeys["public"] + eosioAccount.activePrivateKey=eosioKeys["private"] + eosioAccount.activePublicKey=eosioKeys["public"] + producerKeys.pop(eosioName) + + if not walletMgr.importKey(eosioAccount, ignWallet): + Utils.Print("ERROR: Failed to import %s account keys into ignition wallet." % (eosioName)) + return None - initialFunds="1000000.0000 {0}".format(CORE_SYMBOL) - Utils.Print("Transfer initial fund %s to individual accounts." % (initialFunds)) - trans=None - contract="eosio.token" - action="transfer" - for name, keys in producerKeys.items(): - data="{\"from\":\"eosio\",\"to\":\"%s\",\"quantity\":\"%s\",\"memo\":\"%s\"}" % (name, initialFunds, "init transfer") - opts="--permission eosio@active" - if name != "eosio": - trans=biosNode.pushMessage(contract, action, data, opts) - if trans is None or not trans[0]: - Utils.Print("ERROR: Failed to transfer funds from eosio.token to %s." % (name)) - return None + initialFunds="1000000.0000 {0}".format(CORE_SYMBOL) + Utils.Print("Transfer initial fund %s to individual accounts." % (initialFunds)) + trans=None + contract="eosio.token" + action="transfer" + for name, keys in producerKeys.items(): + data="{\"from\":\"eosio\",\"to\":\"%s\",\"quantity\":\"%s\",\"memo\":\"%s\"}" % (name, initialFunds, "init transfer") + opts="--permission eosio@active" + if name != "eosio": + trans=biosNode.pushMessage(contract, action, data, opts) + if trans is None or not trans[0]: + Utils.Print("ERROR: Failed to transfer funds from eosio.token to %s." % (name)) + return None - Node.validateTransaction(trans[1]) + Node.validateTransaction(trans[1]) - Utils.Print("Wait for last transfer transaction to become finalized.") - transId=Node.getTransId(trans[1]) - if not biosNode.waitForTransInBlock(transId): - Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) - return None + Utils.Print("Wait for last transfer transaction to become finalized.") + transId=Node.getTransId(trans[1]) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return None - Utils.Print("Cluster bootstrap done.") - finally: - if not dontKill: - walletMgr.killall() - walletMgr.cleanup() + Utils.Print("Cluster bootstrap done.") return biosNode @staticmethod - def bootstrap(totalNodes, prodCount, totalProducers, biosHost, biosPort, dontKill=False, onlyBios=False): + def bootstrap(totalNodes, prodCount, totalProducers, biosHost, biosPort, walletMgr, onlyBios=False): """Create 'prodCount' init accounts and deposits 10000000000 SYS in each. If prodCount is -1 will initialize all possible producers. Ensure nodes are inter-connected prior to this call. One way to validate this will be to check if every node has block 1.""" @@ -944,7 +912,7 @@ class Cluster(object): if totalProducers is None: totalProducers=totalNodes - biosNode=Node(biosHost, biosPort) + biosNode=Node(biosHost, biosPort, walletMgr=walletMgr) if not biosNode.checkPulse(): Utils.Print("ERROR: Bios node doesn't appear to be running...") return None @@ -958,258 +926,246 @@ class Cluster(object): Utils.Print("ERROR: Failed to parse %d producer keys from cluster config files, only found %d." % (totalProducers+1,len(producerKeys))) return None - walletMgr=WalletMgr(True) walletMgr.killall() walletMgr.cleanup() if not walletMgr.launch(): Utils.Print("ERROR: Failed to launch bootstrap wallet.") return None - biosNode.setWalletEndpointArgs(walletMgr.walletEndpointArgs) - try: - ignWallet=walletMgr.create("ignition") - - eosioName="eosio" - eosioKeys=producerKeys[eosioName] - eosioAccount=Account(eosioName) - eosioAccount.ownerPrivateKey=eosioKeys["private"] - eosioAccount.ownerPublicKey=eosioKeys["public"] - eosioAccount.activePrivateKey=eosioKeys["private"] - eosioAccount.activePublicKey=eosioKeys["public"] - - if not walletMgr.importKey(eosioAccount, ignWallet): - Utils.Print("ERROR: Failed to import %s account keys into ignition wallet." % (eosioName)) - return None + ignWallet=walletMgr.create("ignition") + + eosioName="eosio" + eosioKeys=producerKeys[eosioName] + eosioAccount=Account(eosioName) + eosioAccount.ownerPrivateKey=eosioKeys["private"] + eosioAccount.ownerPublicKey=eosioKeys["public"] + eosioAccount.activePrivateKey=eosioKeys["private"] + eosioAccount.activePublicKey=eosioKeys["public"] + + if not walletMgr.importKey(eosioAccount, ignWallet): + Utils.Print("ERROR: Failed to import %s account keys into ignition wallet." % (eosioName)) + return None + + contract="eosio.bios" + contractDir="contracts/%s" % (contract) + wasmFile="%s.wasm" % (contract) + abiFile="%s.abi" % (contract) + Utils.Print("Publish %s contract" % (contract)) + trans=biosNode.publishContract(eosioAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True) + if trans is None: + Utils.Print("ERROR: Failed to publish contract %s." % (contract)) + return None + + Node.validateTransaction(trans) - contract="eosio.bios" - contractDir="contracts/%s" % (contract) - wasmFile="%s.wasm" % (contract) - abiFile="%s.abi" % (contract) - Utils.Print("Publish %s contract" % (contract)) - trans=biosNode.publishContract(eosioAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True) + Utils.Print("Creating accounts: %s " % ", ".join(producerKeys.keys())) + producerKeys.pop(eosioName) + accounts=[] + for name, keys in producerKeys.items(): + initx = None + initx = Account(name) + initx.ownerPrivateKey=keys["private"] + initx.ownerPublicKey=keys["public"] + initx.activePrivateKey=keys["private"] + initx.activePublicKey=keys["public"] + trans=biosNode.createAccount(initx, eosioAccount, 0) if trans is None: - Utils.Print("ERROR: Failed to publish contract %s." % (contract)) + Utils.Print("ERROR: Failed to create account %s" % (name)) return None - Node.validateTransaction(trans) + accounts.append(initx) + + transId=Node.getTransId(trans) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return None - Utils.Print("Creating accounts: %s " % ", ".join(producerKeys.keys())) - producerKeys.pop(eosioName) - accounts=[] - for name, keys in producerKeys.items(): - initx = None - initx = Account(name) - initx.ownerPrivateKey=keys["private"] - initx.ownerPublicKey=keys["public"] - initx.activePrivateKey=keys["private"] - initx.activePublicKey=keys["public"] - trans=biosNode.createAccount(initx, eosioAccount, 0) - if trans is None: - Utils.Print("ERROR: Failed to create account %s" % (name)) + Utils.Print("Validating system accounts within bootstrap") + biosNode.validateAccounts(accounts) + + if not onlyBios: + if prodCount == -1: + setProdsFile="setprods.json" + if Utils.Debug: Utils.Print("Reading in setprods file %s." % (setProdsFile)) + with open(setProdsFile, "r") as f: + setProdsStr=f.read() + + Utils.Print("Setting producers.") + opts="--permission eosio@active" + myTrans=biosNode.pushMessage("eosio", "setprods", setProdsStr, opts) + if myTrans is None or not myTrans[0]: + Utils.Print("ERROR: Failed to set producers.") + return None + else: + counts=dict.fromkeys(range(totalNodes), 0) #initialize node prods count to 0 + setProdsStr='{"schedule": [' + firstTime=True + prodNames=[] + for name, keys in producerKeys.items(): + if counts[keys["node"]] >= prodCount: + continue + if firstTime: + firstTime = False + else: + setProdsStr += ',' + + setProdsStr += ' { "producer_name": "%s", "block_signing_key": "%s" }' % (keys["name"], keys["public"]) + prodNames.append(keys["name"]) + counts[keys["node"]] += 1 + + setProdsStr += ' ] }' + if Utils.Debug: Utils.Print("setprods: %s" % (setProdsStr)) + Utils.Print("Setting producers: %s." % (", ".join(prodNames))) + opts="--permission eosio@active" + # pylint: disable=redefined-variable-type + trans=biosNode.pushMessage("eosio", "setprods", setProdsStr, opts) + if trans is None or not trans[0]: + Utils.Print("ERROR: Failed to set producer %s." % (keys["name"])) return None - Node.validateTransaction(trans) - accounts.append(initx) + trans=trans[1] transId=Node.getTransId(trans) if not biosNode.waitForTransInBlock(transId): Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) return None - Utils.Print("Validating system accounts within bootstrap") - biosNode.validateAccounts(accounts) - - if not onlyBios: - if prodCount == -1: - setProdsFile="setprods.json" - if Utils.Debug: Utils.Print("Reading in setprods file %s." % (setProdsFile)) - with open(setProdsFile, "r") as f: - setProdsStr=f.read() - - Utils.Print("Setting producers.") - opts="--permission eosio@active" - myTrans=biosNode.pushMessage("eosio", "setprods", setProdsStr, opts) - if myTrans is None or not myTrans[0]: - Utils.Print("ERROR: Failed to set producers.") - return None - else: - counts=dict.fromkeys(range(totalNodes), 0) #initialize node prods count to 0 - setProdsStr='{"schedule": [' - firstTime=True - prodNames=[] - for name, keys in producerKeys.items(): - if counts[keys["node"]] >= prodCount: - continue - if firstTime: - firstTime = False - else: - setProdsStr += ',' + # wait for block production handover (essentially a block produced by anyone but eosio). + lam = lambda: biosNode.getInfo(exitOnError=True)["head_block_producer"] != "eosio" + ret=Utils.waitForBool(lam) + if not ret: + Utils.Print("ERROR: Block production handover failed.") + return None - setProdsStr += ' { "producer_name": "%s", "block_signing_key": "%s" }' % (keys["name"], keys["public"]) - prodNames.append(keys["name"]) - counts[keys["node"]] += 1 + eosioTokenAccount=copy.deepcopy(eosioAccount) + eosioTokenAccount.name="eosio.token" + trans=biosNode.createAccount(eosioTokenAccount, eosioAccount, 0) + if trans is None: + Utils.Print("ERROR: Failed to create account %s" % (eosioTokenAccount.name)) + return None - setProdsStr += ' ] }' - if Utils.Debug: Utils.Print("setprods: %s" % (setProdsStr)) - Utils.Print("Setting producers: %s." % (", ".join(prodNames))) - opts="--permission eosio@active" - # pylint: disable=redefined-variable-type - trans=biosNode.pushMessage("eosio", "setprods", setProdsStr, opts) - if trans is None or not trans[0]: - Utils.Print("ERROR: Failed to set producer %s." % (keys["name"])) - return None + eosioRamAccount=copy.deepcopy(eosioAccount) + eosioRamAccount.name="eosio.ram" + trans=biosNode.createAccount(eosioRamAccount, eosioAccount, 0) + if trans is None: + Utils.Print("ERROR: Failed to create account %s" % (eosioRamAccount.name)) + return None - trans=trans[1] - transId=Node.getTransId(trans) - if not biosNode.waitForTransInBlock(transId): - Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) - return None + eosioRamfeeAccount=copy.deepcopy(eosioAccount) + eosioRamfeeAccount.name="eosio.ramfee" + trans=biosNode.createAccount(eosioRamfeeAccount, eosioAccount, 0) + if trans is None: + Utils.Print("ERROR: Failed to create account %s" % (eosioRamfeeAccount.name)) + return None - # wait for block production handover (essentially a block produced by anyone but eosio). - lam = lambda: biosNode.getInfo(exitOnError=True)["head_block_producer"] != "eosio" - ret=Utils.waitForBool(lam) - if not ret: - Utils.Print("ERROR: Block production handover failed.") - return None + eosioStakeAccount=copy.deepcopy(eosioAccount) + eosioStakeAccount.name="eosio.stake" + trans=biosNode.createAccount(eosioStakeAccount, eosioAccount, 0) + if trans is None: + Utils.Print("ERROR: Failed to create account %s" % (eosioStakeAccount.name)) + return None - eosioTokenAccount=copy.deepcopy(eosioAccount) - eosioTokenAccount.name="eosio.token" - trans=biosNode.createAccount(eosioTokenAccount, eosioAccount, 0) - if trans is None: - Utils.Print("ERROR: Failed to create account %s" % (eosioTokenAccount.name)) - return None + Node.validateTransaction(trans) + transId=Node.getTransId(trans) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return None - eosioRamAccount=copy.deepcopy(eosioAccount) - eosioRamAccount.name="eosio.ram" - trans=biosNode.createAccount(eosioRamAccount, eosioAccount, 0) - if trans is None: - Utils.Print("ERROR: Failed to create account %s" % (eosioRamAccount.name)) - return None + contract="eosio.token" + contractDir="contracts/%s" % (contract) + wasmFile="%s.wasm" % (contract) + abiFile="%s.abi" % (contract) + Utils.Print("Publish %s contract" % (contract)) + trans=biosNode.publishContract(eosioTokenAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True) + if trans is None: + Utils.Print("ERROR: Failed to publish contract %s." % (contract)) + return None - eosioRamfeeAccount=copy.deepcopy(eosioAccount) - eosioRamfeeAccount.name="eosio.ramfee" - trans=biosNode.createAccount(eosioRamfeeAccount, eosioAccount, 0) - if trans is None: - Utils.Print("ERROR: Failed to create account %s" % (eosioRamfeeAccount.name)) - return None + # Create currency0000, followed by issue currency0000 + contract=eosioTokenAccount.name + Utils.Print("push create action to %s contract" % (contract)) + action="create" + data="{\"issuer\":\"%s\",\"maximum_supply\":\"1000000000.0000 %s\",\"can_freeze\":\"0\",\"can_recall\":\"0\",\"can_whitelist\":\"0\"}" % (eosioTokenAccount.name, CORE_SYMBOL) + opts="--permission %s@active" % (contract) + trans=biosNode.pushMessage(contract, action, data, opts) + if trans is None or not trans[0]: + Utils.Print("ERROR: Failed to push create action to eosio contract.") + return None - eosioStakeAccount=copy.deepcopy(eosioAccount) - eosioStakeAccount.name="eosio.stake" - trans=biosNode.createAccount(eosioStakeAccount, eosioAccount, 0) - if trans is None: - Utils.Print("ERROR: Failed to create account %s" % (eosioStakeAccount.name)) - return None + Node.validateTransaction(trans[1]) + transId=Node.getTransId(trans[1]) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return None - Node.validateTransaction(trans) - transId=Node.getTransId(trans) - if not biosNode.waitForTransInBlock(transId): - Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) - return None + contract=eosioTokenAccount.name + Utils.Print("push issue action to %s contract" % (contract)) + action="issue" + data="{\"to\":\"%s\",\"quantity\":\"1000000000.0000 %s\",\"memo\":\"initial issue\"}" % (eosioAccount.name, CORE_SYMBOL) + opts="--permission %s@active" % (contract) + trans=biosNode.pushMessage(contract, action, data, opts) + if trans is None or not trans[0]: + Utils.Print("ERROR: Failed to push issue action to eosio contract.") + return None - contract="eosio.token" - contractDir="contracts/%s" % (contract) - wasmFile="%s.wasm" % (contract) - abiFile="%s.abi" % (contract) - Utils.Print("Publish %s contract" % (contract)) - trans=biosNode.publishContract(eosioTokenAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True) - if trans is None: - Utils.Print("ERROR: Failed to publish contract %s." % (contract)) - return None + Node.validateTransaction(trans[1]) + Utils.Print("Wait for issue action transaction to become finalized.") + transId=Node.getTransId(trans[1]) + # biosNode.waitForTransInBlock(transId) + # guesstimating block finalization timeout. Two production rounds of 12 blocks per node, plus 60 seconds buffer + timeout = .5 * 12 * 2 * len(producerKeys) + 60 + if not biosNode.waitForTransFinalization(transId, timeout=timeout): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a finalized block on server port %d." % (transId, biosNode.port)) + return None - # Create currency0000, followed by issue currency0000 - contract=eosioTokenAccount.name - Utils.Print("push create action to %s contract" % (contract)) - action="create" - data="{\"issuer\":\"%s\",\"maximum_supply\":\"1000000000.0000 %s\",\"can_freeze\":\"0\",\"can_recall\":\"0\",\"can_whitelist\":\"0\"}" % (eosioTokenAccount.name, CORE_SYMBOL) - opts="--permission %s@active" % (contract) - trans=biosNode.pushMessage(contract, action, data, opts) - if trans is None or not trans[0]: - Utils.Print("ERROR: Failed to push create action to eosio contract.") - return None + expectedAmount="1000000000.0000 {0}".format(CORE_SYMBOL) + Utils.Print("Verify eosio issue, Expected: %s" % (expectedAmount)) + actualAmount=biosNode.getAccountEosBalanceStr(eosioAccount.name) + if expectedAmount != actualAmount: + Utils.Print("ERROR: Issue verification failed. Excepted %s, actual: %s" % + (expectedAmount, actualAmount)) + return None - Node.validateTransaction(trans[1]) - transId=Node.getTransId(trans[1]) - if not biosNode.waitForTransInBlock(transId): - Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) - return None + contract="eosio.system" + contractDir="contracts/%s" % (contract) + wasmFile="%s.wasm" % (contract) + abiFile="%s.abi" % (contract) + Utils.Print("Publish %s contract" % (contract)) + trans=biosNode.publishContract(eosioAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True) + if trans is None: + Utils.Print("ERROR: Failed to publish contract %s." % (contract)) + return None + + Node.validateTransaction(trans) - contract=eosioTokenAccount.name - Utils.Print("push issue action to %s contract" % (contract)) - action="issue" - data="{\"to\":\"%s\",\"quantity\":\"1000000000.0000 %s\",\"memo\":\"initial issue\"}" % (eosioAccount.name, CORE_SYMBOL) - opts="--permission %s@active" % (contract) + initialFunds="1000000.0000 {0}".format(CORE_SYMBOL) + Utils.Print("Transfer initial fund %s to individual accounts." % (initialFunds)) + trans=None + contract=eosioTokenAccount.name + action="transfer" + for name, keys in producerKeys.items(): + data="{\"from\":\"%s\",\"to\":\"%s\",\"quantity\":\"%s\",\"memo\":\"%s\"}" % (eosioAccount.name, name, initialFunds, "init transfer") + opts="--permission %s@active" % (eosioAccount.name) trans=biosNode.pushMessage(contract, action, data, opts) if trans is None or not trans[0]: - Utils.Print("ERROR: Failed to push issue action to eosio contract.") + Utils.Print("ERROR: Failed to transfer funds from %s to %s." % (eosioTokenAccount.name, name)) return None Node.validateTransaction(trans[1]) - Utils.Print("Wait for issue action transaction to become finalized.") - transId=Node.getTransId(trans[1]) - # biosNode.waitForTransInBlock(transId) - # guesstimating block finalization timeout. Two production rounds of 12 blocks per node, plus 60 seconds buffer - timeout = .5 * 12 * 2 * len(producerKeys) + 60 - if not biosNode.waitForTransFinalization(transId, timeout=timeout): - Utils.Print("ERROR: Failed to validate transaction %s got rolled into a finalized block on server port %d." % (transId, biosNode.port)) - return None - - expectedAmount="1000000000.0000 {0}".format(CORE_SYMBOL) - Utils.Print("Verify eosio issue, Expected: %s" % (expectedAmount)) - actualAmount=biosNode.getAccountEosBalanceStr(eosioAccount.name) - if expectedAmount != actualAmount: - Utils.Print("ERROR: Issue verification failed. Excepted %s, actual: %s" % - (expectedAmount, actualAmount)) - return None - - contract="eosio.system" - contractDir="contracts/%s" % (contract) - wasmFile="%s.wasm" % (contract) - abiFile="%s.abi" % (contract) - Utils.Print("Publish %s contract" % (contract)) - trans=biosNode.publishContract(eosioAccount.name, contractDir, wasmFile, abiFile, waitForTransBlock=True) - if trans is None: - Utils.Print("ERROR: Failed to publish contract %s." % (contract)) - return None - - Node.validateTransaction(trans) - initialFunds="1000000.0000 {0}".format(CORE_SYMBOL) - Utils.Print("Transfer initial fund %s to individual accounts." % (initialFunds)) - trans=None - contract=eosioTokenAccount.name - action="transfer" - for name, keys in producerKeys.items(): - data="{\"from\":\"%s\",\"to\":\"%s\",\"quantity\":\"%s\",\"memo\":\"%s\"}" % (eosioAccount.name, name, initialFunds, "init transfer") - opts="--permission %s@active" % (eosioAccount.name) - trans=biosNode.pushMessage(contract, action, data, opts) - if trans is None or not trans[0]: - Utils.Print("ERROR: Failed to transfer funds from %s to %s." % (eosioTokenAccount.name, name)) - return None - - Node.validateTransaction(trans[1]) - - Utils.Print("Wait for last transfer transaction to become finalized.") - transId=Node.getTransId(trans[1]) - if not biosNode.waitForTransInBlock(transId): - Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) - return None + Utils.Print("Wait for last transfer transaction to become finalized.") + transId=Node.getTransId(trans[1]) + if not biosNode.waitForTransInBlock(transId): + Utils.Print("ERROR: Failed to validate transaction %s got rolled into a block on server port %d." % (transId, biosNode.port)) + return None - Utils.Print("Cluster bootstrap done.") - finally: - if not dontKill: - walletMgr.killall() - walletMgr.cleanup() + Utils.Print("Cluster bootstrap done.") return biosNode @staticmethod def pgrepEosServers(timeout=None): - pgrepOpts="-fl" - # pylint: disable=deprecated-method - if platform.linux_distribution()[0] in ["Ubuntu", "LinuxMint", "Fedora","CentOS Linux","arch"]: - pgrepOpts="-a" - - cmd="pgrep %s %s" % (pgrepOpts, Utils.EosServerName) + cmd=Utils.pgrepCmd(Utils.EosServerName) def myFunc(): psOut=None @@ -1253,8 +1209,7 @@ class Cluster(object): if m is None: Utils.Print("ERROR: Failed to find %s pid. Pattern %s" % (Utils.EosServerName, pattern)) break - instance=Node(self.host, self.port + i, pid=int(m.group(1)), cmd=m.group(2), enableMongo=self.enableMongo, mongoHost=self.mongoHost, mongoPort=self.mongoPort, mongoDb=self.mongoDb) - instance.setWalletEndpointArgs(self.walletEndpointArgs) + instance=Node(self.host, self.port + i, pid=int(m.group(1)), cmd=m.group(2), walletMgr=self.walletMgr, enableMongo=self.enableMongo, mongoHost=self.mongoHost, mongoPort=self.mongoPort, mongoDb=self.mongoDb) if Utils.Debug: Utils.Print("Node>", instance) nodes.append(instance) @@ -1321,9 +1276,14 @@ class Cluster(object): for i in range(0, len(self.nodes)): fileName="etc/eosio/node_%02d/config.ini" % (i) Cluster.dumpErrorDetailImpl(fileName) + fileName="etc/eosio/node_%02d/genesis.json" % (i) + Cluster.dumpErrorDetailImpl(fileName) fileName="var/lib/node_%02d/stderr.txt" % (i) Cluster.dumpErrorDetailImpl(fileName) + if self.useBiosBootFile: + Cluster.dumpErrorDetailImpl(Cluster.__bootlog) + def killall(self, silent=True, allInstances=False): """Kill cluster nodeos instances. allInstances will kill all nodeos instances running on the system.""" cmd="%s -k 9" % (Utils.EosLauncherPath) @@ -1395,6 +1355,9 @@ class Cluster(object): for f in glob.glob("etc/eosio/node_*"): shutil.rmtree(f) + for f in self.filesToCleanup: + os.remove(f) + if self.enableMongo: cmd="%s %s" % (Utils.MongoPath, self.mongoEndpointArgs) subcommand="db.dropDatabase()" diff --git a/tests/Node.py b/tests/Node.py index 2795ba63c5efba75fd18c9c1100ad9ebe1c01877..93ab850255518d7dc6e87e66c27e0264a28e91b5 100644 --- a/tests/Node.py +++ b/tests/Node.py @@ -31,7 +31,7 @@ class Node(object): # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments - def __init__(self, host, port, pid=None, cmd=None, enableMongo=False, mongoHost="localhost", mongoPort=27017, mongoDb="EOStest"): + def __init__(self, host, port, pid=None, cmd=None, walletMgr=None, enableMongo=False, mongoHost="localhost", mongoPort=27017, mongoDb="EOStest"): self.host=host self.port=port self.pid=pid @@ -44,16 +44,18 @@ class Node(object): self.mongoDb=mongoDb self.endpointHttp="http://%s:%d" % (self.host, self.port) self.endpointArgs="--url %s" % (self.endpointHttp) - self.miscEosClientArgs="--no-auto-keosd" self.mongoEndpointArgs="" self.infoValid=None self.lastRetrievedHeadBlockNum=None self.lastRetrievedLIB=None + self.transCache={} + self.walletMgr=walletMgr if self.enableMongo: self.mongoEndpointArgs += "--host %s --port %d %s" % (mongoHost, mongoPort, mongoDb) def eosClientArgs(self): - return self.endpointArgs + " " + self.miscEosClientArgs + walletArgs=" " + self.walletMgr.getWalletEndpointArgs() if self.walletMgr is not None else "" + return self.endpointArgs + walletArgs + " " + Utils.MiscEosClientArgs def __str__(self): #return "Host: %s, Port:%d, Pid:%s, Cmd:\"%s\"" % (self.host, self.port, self.pid, self.cmd) @@ -64,11 +66,80 @@ class Node(object): assert trans assert isinstance(trans, dict), print("Input type is %s" % type(trans)) - def printTrans(trans): - Utils.Print("ERROR: Failure in transaction validation.") + executed="executed" + def printTrans(trans, status): + Utils.Print("ERROR: Valid transaction should be \"%s\" but it was \"%s\"." % (executed, status)) Utils.Print("Transaction: %s" % (json.dumps(trans, indent=1))) - assert trans["processed"]["receipt"]["status"] == "executed", printTrans(trans) + transStatus=Node.getTransStatus(trans) + assert transStatus == executed, printTrans(trans, transStatus) + + @staticmethod + def __printTransStructureError(trans, context): + Utils.Print("ERROR: Failure in expected transaction structure. Missing trans%s." % (context)) + Utils.Print("Transaction: %s" % (json.dumps(trans, indent=1))) + + class Context: + def __init__(self, obj, desc): + self.obj=obj + self.sections=[obj] + self.keyContext=[] + self.desc=desc + + def __json(self): + return "%s=\n%s" % (self.desc, json.dumps(self.obj, indent=1)) + + def __keyContext(self): + msg="" + for key in self.keyContext: + if msg=="": + msg="[" + else: + msg+="][" + msg+=key + if msg!="": + msg+="]" + return msg + + def __contextDesc(self): + return "%s%s" % (self.desc, self.__keyContext()) + + def add(self, newKey): + assert isinstance(newKey, str), print("ERROR: Trying to use %s as a key" % (newKey)) + subSection=self.sections[-1] + assert isinstance(subSection, dict), print("ERROR: Calling \"add\" method when context is not a dictionary. %s in %s" % (self.__contextDesc(), self.__json())) + assert newKey in subSection, print("ERROR: %s%s does not contain key \"%s\". %s" % (self.__contextDesc(), key, self.__json())) + current=subSection[newKey] + self.sections.append(current) + self.keyContext.append(newKey) + return current + + def index(self, i): + assert isinstance(i, int), print("ERROR: Trying to use \"%s\" as a list index" % (i)) + cur=self.getCurrent() + assert isinstance(cur, list), print("ERROR: Calling \"index\" method when context is not a list. %s in %s" % (self.__contextDesc(), self.__json())) + listLen=len(cur) + assert i < listLen, print("ERROR: Index %s is beyond the size of the current list (%s). %s in %s" % (i, listLen, self.__contextDesc(), self.__json())) + return self.sections.append(cur[i]) + + def getCurrent(self): + return self.sections[-1] + + @staticmethod + def getTransStatus(trans): + cntxt=Node.Context(trans, "trans") + cntxt.add("processed") + cntxt.add("receipt") + return cntxt.add("status") + + @staticmethod + def getTransBlockNum(trans): + cntxt=Node.Context(trans, "trans") + cntxt.add("processed") + cntxt.add("action_traces") + cntxt.index(0) + return cntxt.add("block_num") + @staticmethod def stdinAndCheckOutput(cmd, subcommand): @@ -140,17 +211,22 @@ class Node(object): assert trans assert isinstance(trans, dict), print("Input type is %s" % type(trans)) - #Utils.Print("%s" % trans) + assert "transaction_id" in trans, print("trans does not contain key %s. trans={%s}" % ("transaction_id", json.dumps(trans, indent=2, sort_keys=True))) transId=trans["transaction_id"] return transId + @staticmethod + def isTrans(obj): + """Identify if this is a transaction dictionary.""" + if obj is None or not isinstance(obj, dict): + return False + + return True if "transaction_id" in obj else False + @staticmethod def byteArrToStr(arr): return arr.decode("utf-8") - def setWalletEndpointArgs(self, args): - self.endpointArgs="--url http://%s:%d %s" % (self.host, self.port, args) - def validateAccounts(self, accounts): assert(accounts) assert(isinstance(accounts, list)) @@ -249,8 +325,10 @@ class Node(object): return self.isBlockPresent(blockNum, blockType=BlockType.lib) class BlockWalker: - def __init__(self, node, trans, startBlockNum=None, endBlockNum=None): - self.trans=trans + def __init__(self, node, transId, startBlockNum=None, endBlockNum=None): + assert(isinstance(transId, str)) + self.trans=None + self.transId=transId self.node=node self.startBlockNum=startBlockNum self.endBlockNum=endBlockNum @@ -258,32 +336,45 @@ class Node(object): def walkBlocks(self): start=None end=None - blockNum=self.trans["processed"]["action_traces"][0]["block_num"] + if self.trans is None and self.transId in self.transCache.keys(): + self.trans=self.transCache[self.transId] + if self.trans is not None: + cntxt=Node.Context(self.trans, "trans") + cntxt.add("processed") + cntxt.add("action_traces") + cntxt.index(0) + blockNum=cntxt.add("block_num") + else: + blockNum=None # it should be blockNum or later, but just in case the block leading up have any clues... + start=None if self.startBlockNum is not None: start=self.startBlockNum - else: + elif blockNum is not None: start=blockNum-5 if self.endBlockNum is not None: end=self.endBlockNum else: info=self.node.getInfo() end=info["head_block_num"] - msg="Original transaction=\n%s\nExpected block_num=%s\n" % (json.dumps(trans, indent=2, sort_keys=True), blockNum) + if start is None: + if end > 100: + start=end-100 + else: + start=0 + transDesc=" id =%s" % (self.transId) + if self.trans is not None: + transDesc="=%s" % (json.dumps(self.trans, indent=2, sort_keys=True)) + msg="Original transaction%s\nExpected block_num=%s\n" % (transDesc, blockNum) for blockNum in range(start, end+1): block=self.node.getBlock(blockNum) msg+=json.dumps(block, indent=2, sort_keys=True)+"\n" + return msg + # pylint: disable=too-many-branches - def getTransaction(self, transOrTransId, silentErrors=False, exitOnError=False, delayedRetry=True): - transId=None - trans=None - assert(isinstance(transOrTransId, (str,dict))) - if isinstance(transOrTransId, str): - transId=transOrTransId - else: - trans=transOrTransId - transId=Node.getTransId(trans) + def getTransaction(self, transId, silentErrors=False, exitOnError=False, delayedRetry=True): + assert(isinstance(transId, str)) exitOnErrorForDelayed=not delayedRetry and exitOnError timeout=3 blockWalker=None @@ -296,7 +387,7 @@ class Node(object): if trans is not None or not delayedRetry: return trans if blockWalker is None: - blockWalker=Node.BlockWalker(self, trans) + blockWalker=Node.BlockWalker(self, transId) if Utils.Debug: Utils.Print("Could not find transaction with id %s, delay and retry" % (transId)) time.sleep(timeout) @@ -367,16 +458,11 @@ class Node(object): return False - def getBlockIdByTransId(self, transOrTransId, delayedRetry=True): - """Given a transaction (dictionary) or transaction Id (string), will return the actual block id (int) containing the transaction""" - assert(transOrTransId) - transId=None - assert(isinstance(transOrTransId, (str,dict))) - if isinstance(transOrTransId, str): - transId=transOrTransId - else: - transId=Node.getTransId(transOrTransId) - trans=self.getTransaction(transOrTransId, exitOnError=True, delayedRetry=delayedRetry) + def getBlockIdByTransId(self, transId, delayedRetry=True): + """Given a transaction Id (string), will return the actual block id (int) containing the transaction""" + assert(transId) + assert(isinstance(transId, str)) + trans=self.getTransaction(transId, exitOnError=True, delayedRetry=delayedRetry) refBlockNum=None key="" @@ -466,6 +552,7 @@ class Node(object): account.activePublicKey, stakeNet, CORE_SYMBOL, stakeCPU, CORE_SYMBOL, buyRAM, CORE_SYMBOL) msg="(creator account=%s, account=%s)" % (creatorAccount.name, account.name); trans=self.processCleosCmd(cmd, cmdDesc, silentErrors=False, exitOnError=exitOnError, exitMsg=msg) + self.trackCmdTransaction(trans) transId=Node.getTransId(trans) if stakedDeposit > 0: @@ -483,11 +570,13 @@ class Node(object): cmdDesc, creatorAccount.name, account.name, account.ownerPublicKey, account.activePublicKey) msg="(creator account=%s, account=%s)" % (creatorAccount.name, account.name); trans=self.processCleosCmd(cmd, cmdDesc, silentErrors=False, exitOnError=exitOnError, exitMsg=msg) + self.trackCmdTransaction(trans) transId=Node.getTransId(trans) if stakedDeposit > 0: self.waitForTransInBlock(transId) # seems like account creation needs to be finlized before transfer can happen trans = self.transferFunds(creatorAccount, account, "%0.04f %s" % (stakedDeposit/10000, CORE_SYMBOL), "init") + self.trackCmdTransaction(trans) transId=Node.getTransId(trans) return self.waitForTransBlockIfNeeded(trans, waitForTransBlock, exitOnError=exitOnError) @@ -638,6 +727,7 @@ class Node(object): trans=None try: trans=Utils.runCmdArrReturnJson(cmdArr) + self.trackCmdTransaction(trans) except subprocess.CalledProcessError as ex: msg=ex.output.decode("utf-8") Utils.Print("ERROR: Exception during funds transfer. %s" % (msg)) @@ -819,6 +909,7 @@ class Node(object): trans=None try: trans=Utils.runCmdReturnJson(cmd, trace=False) + self.trackCmdTransaction(trans) except subprocess.CalledProcessError as ex: if not shouldFail: msg=ex.output.decode("utf-8") @@ -876,6 +967,7 @@ class Node(object): if Utils.Debug: Utils.Print("cmd: %s" % (cmdArr)) try: trans=Utils.runCmdArrReturnJson(cmdArr) + self.trackCmdTransaction(trans, ignoreNonTrans=True) return (True, trans) except subprocess.CalledProcessError as ex: msg=ex.output.decode("utf-8") @@ -887,6 +979,7 @@ class Node(object): cmdDesc="set action permission" cmd="%s -j %s %s %s %s" % (cmdDesc, account, code, pType, requirement) trans=self.processCleosCmd(cmd, cmdDesc, silentErrors=False, exitOnError=exitOnError) + self.trackCmdTransaction(trans) return self.waitForTransBlockIfNeeded(trans, waitForTransBlock, exitOnError=exitOnError) @@ -895,11 +988,12 @@ class Node(object): toAccount=fromAccount cmdDesc="system delegatebw" - transferStr="--transfer" if transferTo else "" + transferStr="--transfer" if transferTo else "" cmd="%s -j %s %s \"%s %s\" \"%s %s\" %s" % ( cmdDesc, fromAccount.name, toAccount.name, netQuantity, CORE_SYMBOL, cpuQuantity, CORE_SYMBOL, transferStr) msg="fromAccount=%s, toAccount=%s" % (fromAccount.name, toAccount.name); trans=self.processCleosCmd(cmd, cmdDesc, exitOnError=exitOnError, exitMsg=msg) + self.trackCmdTransaction(trans) return self.waitForTransBlockIfNeeded(trans, waitForTransBlock, exitOnError=exitOnError) @@ -909,6 +1003,7 @@ class Node(object): cmdDesc, producer.name, producer.activePublicKey, url, location) msg="producer=%s" % (producer.name); trans=self.processCleosCmd(cmd, cmdDesc, exitOnError=exitOnError, exitMsg=msg) + self.trackCmdTransaction(trans) return self.waitForTransBlockIfNeeded(trans, waitForTransBlock, exitOnError=exitOnError) @@ -918,6 +1013,7 @@ class Node(object): cmdDesc, account.name, " ".join(producers)) msg="account=%s, producers=[ %s ]" % (account.name, ", ".join(producers)); trans=self.processCleosCmd(cmd, cmdDesc, exitOnError=exitOnError, exitMsg=msg) + self.trackCmdTransaction(trans) return self.waitForTransBlockIfNeeded(trans, waitForTransBlock, exitOnError=exitOnError) @@ -949,7 +1045,7 @@ class Node(object): return None if exitOnError and trans is None: - Utils.cmdError("could not \"%s\" - %s" % (cmdDesc,exitMsg)) + Utils.cmdError("could not \"%s\". %s" % (cmdDesc,exitMsg)) errorExit("Failed to \"%s\"" % (cmdDesc)) return trans @@ -963,12 +1059,12 @@ class Node(object): cmd="curl %s/v1/test_control/kill_node_on_producer -d '{ \"producer\":\"%s\", \"where_in_sequence\":%d, \"based_on_lib\":\"%s\" }' -X POST -H \"Content-Type: application/json\"" % \ (self.endpointHttp, producer, whereInSequence, basedOnLib) if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) - trans=None + rtn=None try: if returnType==ReturnType.json: - trans=Utils.runCmdReturnJson(cmd, silentErrors=silentErrors) + rtn=Utils.runCmdReturnJson(cmd, silentErrors=silentErrors) elif returnType==ReturnType.raw: - trans=Utils.runCmdReturnStr(cmd) + rtn=Utils.runCmdReturnStr(cmd) else: unhandledEnumType(returnType) except subprocess.CalledProcessError as ex: @@ -986,11 +1082,11 @@ class Node(object): exitMsg=": " + exitMsg else: exitMsg="" - if exitOnError and trans is None: + if exitOnError and rtn is None: Utils.cmdError("could not \"%s\" - %s" % (cmd,exitMsg)) Utils.errorExit("Failed to \"%s\"" % (cmd)) - return trans + return rtn def waitForTransBlockIfNeeded(self, trans, waitForTransBlock, exitOnError=False): if not waitForTransBlock: @@ -1166,7 +1262,7 @@ class Node(object): cmdArr=[] myCmd=self.cmd - toAddOrSwap=copy.deepcopy(addOrSwapFlags) if addOrSwapFlags is not None else {} + toAddOrSwap=copy.deepcopy(addOrSwapFlags) if addOrSwapFlags is not None else {} if not newChain: skip=False swapValue=None @@ -1227,6 +1323,26 @@ class Node(object): self.killed=False return True + def trackCmdTransaction(self, trans, ignoreNonTrans=False): + if trans is None: + if Utils.Debug: Utils.Print(" cmd returned transaction: %s" % (trans)) + return + + if ignoreNonTrans and not Node.isTrans(trans): + if Utils.Debug: Utils.Print(" cmd returned a non-transaction") + return + + transId=Node.getTransId(trans) + if Utils.Debug: + status=Node.getTransStatus(trans) + blockNum=Node.getTransBlockNum(trans) + if transId in self.transCache.keys(): + replaceMsg="replacing previous trans=\n%s" % json.dumps(self.transCache[transId], indent=2, sort_keys=True) + else: + replaceMsg="" + Utils.Print(" cmd returned transaction id: %s, status: %s, (possible) block num: %s %s" % (transId, status, blockNum, replaceMsg)) + self.transCache[transId]=trans + def reportStatus(self): Utils.Print("Node State:") Utils.Print(" cmd : %s" % (self.cmd)) diff --git a/tests/TestHelper.py b/tests/TestHelper.py index 6e00645e9dcff111948e44dd341a3a33fc08e627..471c397beff1cb8911fb6df1b3da02e7ddf4bb86 100644 --- a/tests/TestHelper.py +++ b/tests/TestHelper.py @@ -26,6 +26,7 @@ class AppArgs: class TestHelper(object): LOCAL_HOST="localhost" DEFAULT_PORT=8888 + DEFAULT_WALLET_PORT=9899 @staticmethod # pylint: disable=too-many-branches @@ -70,6 +71,12 @@ class TestHelper(object): if "--port" in includeArgs: parser.add_argument("-p", "--port", type=int, help="%s host port" % Utils.EosServerName, default=TestHelper.DEFAULT_PORT) + if "--wallet-host" in includeArgs: + parser.add_argument("--wallet-host", type=str, help="%s host" % Utils.EosWalletName, + default=TestHelper.LOCAL_HOST) + if "--wallet-port" in includeArgs: + parser.add_argument("--wallet-port", type=int, help="%s port" % Utils.EosWalletName, + default=TestHelper.DEFAULT_WALLET_PORT) if "--prod-count" in includeArgs: parser.add_argument("-c", "--prod-count", type=int, help="Per node producer count", default=1) if "--defproducera_prvt_key" in includeArgs: diff --git a/tests/WalletMgr.py b/tests/WalletMgr.py index c46dd78d6fd0861f748b238c5f970a885d461bd3..8b7e4957277e36e9e6c84be9813d4a5b30f9ee61 100644 --- a/tests/WalletMgr.py +++ b/tests/WalletMgr.py @@ -12,8 +12,10 @@ from testUtils import Utils Wallet=namedtuple("Wallet", "name password host port") # pylint: disable=too-many-instance-attributes class WalletMgr(object): - __walletLogFile="test_keosd_output.log" + __walletLogOutFile="test_keosd_out.log" + __walletLogErrFile="test_keosd_err.log" __walletDataDir="test_wallet_0" + __MaxPort=9999 # pylint: disable=too-many-arguments # walletd [True|False] True=Launch wallet(keosd) process; False=Manage launch process externally. @@ -25,26 +27,76 @@ class WalletMgr(object): self.host=host self.wallets={} self.__walletPid=None - self.endpointArgs="--url http://%s:%d" % (self.nodeosHost, self.nodeosPort) - self.walletEndpointArgs="" - if self.walletd: - self.walletEndpointArgs += " --wallet-url http://%s:%d" % (self.host, self.port) - self.endpointArgs += self.walletEndpointArgs + + def getWalletEndpointArgs(self): + if not self.walletd or not self.isLaunched(): + return "" + + return " --wallet-url http://%s:%d" % (self.host, self.port) + + def getArgs(self): + return " --url http://%s:%d%s %s" % (self.nodeosHost, self.nodeosPort, self.getWalletEndpointArgs(), Utils.MiscEosClientArgs) + + def isLaunched(self): + return self.__walletPid is not None + + def isLocal(self): + return self.host=="localhost" or self.host=="127.0.0.1" + + def findAvailablePort(self): + for i in range(WalletMgr.__MaxPort): + port=self.port+i + if port > WalletMgr.__MaxPort: + port-=WalletMgr.__MaxPort + if Utils.arePortsAvailable(port): + return port + if Utils.Debug: Utils.Print("Port %d not available for %s" % (port, Utils.EosWalletPath)) + + Utils.errorExit("Failed to find free port to use for %s" % (Utils.EosWalletPath)) def launch(self): if not self.walletd: Utils.Print("ERROR: Wallet Manager wasn't configured to launch keosd") return False + if self.isLaunched(): + return True + + if self.isLocal(): + self.port=self.findAvailablePort() + + pgrepCmd=Utils.pgrepCmd(Utils.EosWalletName) + if Utils.Debug: + portTaken=False + if self.isLocal(): + if not Utils.arePortsAvailable(self.port): + portTaken=True + psOut=Utils.checkOutput(pgrepCmd.split(), ignoreError=True) + if psOut or portTaken: + statusMsg="" + if psOut: + statusMsg+=" %s - {%s}." % (pgrepCmd, psOut) + if portTaken: + statusMsg+=" port %d is NOT available." % (self.port) + Utils.Print("Launching %s, note similar processes running. %s" % (Utils.EosWalletName, statusMsg)) + cmd="%s --data-dir %s --config-dir %s --http-server-address=%s:%d --verbose-http-errors" % ( Utils.EosWalletPath, WalletMgr.__walletDataDir, WalletMgr.__walletDataDir, self.host, self.port) if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) - with open(WalletMgr.__walletLogFile, 'w') as sout, open(WalletMgr.__walletLogFile, 'w') as serr: + with open(WalletMgr.__walletLogOutFile, 'w') as sout, open(WalletMgr.__walletLogErrFile, 'w') as serr: popen=subprocess.Popen(cmd.split(), stdout=sout, stderr=serr) self.__walletPid=popen.pid # Give keosd time to warm up time.sleep(2) + + try: + if Utils.Debug: Utils.Print("Checking if %s launched. %s" % (Utils.EosWalletName, pgrepCmd)) + psOut=Utils.checkOutput(pgrepCmd.split()) + if Utils.Debug: Utils.Print("Launched %s. {%s}" % (Utils.EosWalletName, psOut)) + except subprocess.CalledProcessError as ex: + Utils.errorExit("Failed to launch the wallet manager") + return True def create(self, name, accounts=None, exitOnError=True): @@ -54,7 +106,7 @@ class WalletMgr(object): return wallet p = re.compile(r'\n\"(\w+)\"\n', re.MULTILINE) cmdDesc="wallet create" - cmd="%s %s %s --name %s --to-console" % (Utils.EosClientPath, self.endpointArgs, cmdDesc, name) + cmd="%s %s %s --name %s --to-console" % (Utils.EosClientPath, self.getArgs(), cmdDesc, name) if Utils.Debug: Utils.Print("cmd: %s" % (cmd)) retStr=None maxRetryCount=4 @@ -67,7 +119,15 @@ class WalletMgr(object): retryCount+=1 if retryCount # --keep-logs ############################################################### -args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run"}) +args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--wallet-port"}) Utils.Debug=args.v totalNodes=4 cluster=Cluster(walletd=True) @@ -62,17 +63,19 @@ dumpErrorDetails=args.dump_error_details keepLogs=args.keep_logs dontKill=args.leave_running killAll=args.clean_run +walletPort=args.wallet_port -walletMgr=WalletMgr(True) +walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False killEosInstances=not dontKill killWallet=not dontKill -WalletdName="keosd" +WalletdName=Utils.EosWalletName ClientName="cleos" try: TestHelper.printSystemInfo("BEGIN") + cluster.setWalletMgr(walletMgr) cluster.killall(allInstances=killAll) cluster.cleanup() @@ -82,7 +85,7 @@ try: maxRAMFlag="--chain-state-db-size-mb" maxRAMValue=1010 extraNodeosArgs=" %s %d %s %d " % (minRAMFlag, minRAMValue, maxRAMFlag, maxRAMValue) - if cluster.launch(onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs, useBiosBootFile=False) is False: + if cluster.launch(onlyBios=False, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes, extraNodeosArgs=extraNodeosArgs, useBiosBootFile=False) is False: Utils.cmdError("launcher") errorExit("Failed to stand up eos cluster.") @@ -96,12 +99,6 @@ try: testWalletName="test" Print("Creating wallet \"%s\"." % (testWalletName)) - walletMgr.killall(allInstances=killAll) - walletMgr.cleanup() - if walletMgr.launch() is False: - Utils.cmdError("%s" % (WalletdName)) - errorExit("Failed to stand up eos walletd.") - testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount]) for _, account in cluster.defProducerAccounts.items(): @@ -126,7 +123,7 @@ try: transferAmount="70000000.0000 {0}".format(CORE_SYMBOL) Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name)) nodes[0].transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer") - trans=nodes[0].delegatebw(account, 1000000.0000, 68000000.0000, exitOnError=True) + trans=nodes[0].delegatebw(account, 1000000.0000, 68000000.0000, waitForTransBlock=True, exitOnError=True) contractAccount=cluster.createAccountKeys(1)[0] contractAccount.name="contracttest" @@ -136,7 +133,7 @@ try: transferAmount="90000000.0000 {0}".format(CORE_SYMBOL) Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, contractAccount.name)) nodes[0].transferFunds(cluster.eosioAccount, contractAccount, transferAmount, "test transfer") - trans=nodes[0].delegatebw(contractAccount, 1000000.0000, 88000000.0000, exitOnError=True) + trans=nodes[0].delegatebw(contractAccount, 1000000.0000, 88000000.0000, waitForTransBlock=True, exitOnError=True) contractDir="contracts/integration_test" wasmFile="integration_test.wasm" @@ -155,6 +152,7 @@ try: count=0 while keepProcessing: numAmount+=1 + timeOutCount=0 for fromIndex in range(namedAccounts.numAccounts): count+=1 toIndex=fromIndex+1 @@ -167,8 +165,15 @@ try: try: trans=nodes[0].pushMessage(contract, action, data, opts) if trans is None or not trans[0]: + timeOutCount+=1 + if timeOutCount>=3: + Print("Failed to push create action to eosio contract for %d consecutive times, looks like nodeos already exited." % (timeOutCount)) + keepProcessing=False + break Print("Failed to push create action to eosio contract. sleep for 60 seconds") time.sleep(60) + else: + timeOutCount=0 time.sleep(1) except TypeError as ex: keepProcessing=False diff --git a/tests/nodeos_voting_test.py b/tests/nodeos_voting_test.py index b6f176af8c95a6e33879b5fb866832ffa1e04c19..d4781d0eefe8da27b41eaae6faa34586628d4ca6 100755 --- a/tests/nodeos_voting_test.py +++ b/tests/nodeos_voting_test.py @@ -140,7 +140,8 @@ errorExit=Utils.errorExit from core_symbol import CORE_SYMBOL -args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--p2p-plugin"}) +args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run", + "--p2p-plugin","--wallet-port"}) Utils.Debug=args.v totalNodes=4 cluster=Cluster(walletd=True) @@ -150,22 +151,24 @@ dontKill=args.leave_running prodCount=args.prod_count killAll=args.clean_run p2pPlugin=args.p2p_plugin +walletPort=args.wallet_port -walletMgr=WalletMgr(True) +walletMgr=WalletMgr(True, port=walletPort) testSuccessful=False killEosInstances=not dontKill killWallet=not dontKill -WalletdName="keosd" +WalletdName=Utils.EosWalletName ClientName="cleos" try: TestHelper.printSystemInfo("BEGIN") + cluster.setWalletMgr(walletMgr) cluster.killall(allInstances=killAll) cluster.cleanup() Print("Stand up cluster") - if cluster.launch(prodCount=prodCount, onlyBios=False, dontKill=dontKill, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes*21, p2pPlugin=p2pPlugin, useBiosBootFile=False) is False: + if cluster.launch(prodCount=prodCount, onlyBios=False, pnodes=totalNodes, totalNodes=totalNodes, totalProducers=totalNodes*21, p2pPlugin=p2pPlugin, useBiosBootFile=False) is False: Utils.cmdError("launcher") Utils.errorExit("Failed to stand up eos cluster.") @@ -184,12 +187,6 @@ try: testWalletName="test" Print("Creating wallet \"%s\"." % (testWalletName)) - walletMgr.killall(allInstances=killAll) - walletMgr.cleanup() - if walletMgr.launch() is False: - Utils.cmdError("%s" % (WalletdName)) - Utils.errorExit("Failed to stand up eos walletd.") - testWallet=walletMgr.create(testWalletName, [cluster.eosioAccount,accounts[0],accounts[1],accounts[2],accounts[3],accounts[4]]) for _, account in cluster.defProducerAccounts.items(): @@ -216,7 +213,7 @@ try: transferAmount="100000000.0000 {0}".format(CORE_SYMBOL) Print("Transfer funds %s from account %s to %s" % (transferAmount, cluster.eosioAccount.name, account.name)) node.transferFunds(cluster.eosioAccount, account, transferAmount, "test transfer") - trans=node.delegatebw(account, 20000000.0000, 20000000.0000, exitOnError=True) + trans=node.delegatebw(account, 20000000.0000, 20000000.0000, waitForTransBlock=True, exitOnError=True) # containers for tracking producers prodsActive={} @@ -229,7 +226,7 @@ try: #first account will vote for node0 producers, all others will vote for node1 producers node=node0 for account in accounts: - trans=node.vote(account, node.producers) + trans=node.vote(account, node.producers, waitForTransBlock=True) node=node1 setActiveProducers(prodsActive, node1.producers) @@ -240,7 +237,7 @@ try: # first account will vote for node2 producers, all others will vote for node3 producers node1 for account in accounts: - trans=node.vote(account, node.producers) + trans=node.vote(account, node.producers, waitForTransBlock=True) node=node2 setActiveProducers(prodsActive, node2.producers) diff --git a/tests/restart-scenarios-test.py b/tests/restart-scenarios-test.py index d3329c8b65f14a2a86c03011a5ffa455468c0526..a8ae17d78b71f427c6e0dae4a49c3f4dd7d87605 100755 --- a/tests/restart-scenarios-test.py +++ b/tests/restart-scenarios-test.py @@ -52,6 +52,7 @@ walletMgr=WalletMgr(True) try: TestHelper.printSystemInfo("BEGIN") + cluster.setWalletMgr(walletMgr) cluster.setChainStrategy(chainSyncStrategyStr) cluster.setWalletMgr(walletMgr) @@ -74,11 +75,6 @@ try: errorExit("Cluster never stabilized") Print("Stand up EOS wallet keosd") - walletMgr.killall(allInstances=killAll) - walletMgr.cleanup() - if walletMgr.launch() is False: - errorExit("Failed to stand up keosd.") - accountsCount=total_nodes walletName="MyWallet" Print("Creating wallet %s if one doesn't already exist." % walletName) diff --git a/tests/testUtils.py b/tests/testUtils.py index d2a692315135c1ba1e8ba8eef5feb1901e97b633..9c52fe3796a230cf77d6ba56158288e7b6c4bfe7 100755 --- a/tests/testUtils.py +++ b/tests/testUtils.py @@ -1,11 +1,14 @@ +import errno import subprocess import time import os +import platform from collections import deque from collections import namedtuple import inspect import json import shlex +import socket from sys import stdout from sys import exit import traceback @@ -16,6 +19,7 @@ class Utils: FNull = open(os.devnull, 'w') EosClientPath="programs/cleos/cleos" + MiscEosClientArgs="--no-auto-keosd" EosWalletName="keosd" EosWalletPath="programs/keosd/"+ EosWalletName @@ -75,12 +79,12 @@ class Utils: return chainSyncStrategies @staticmethod - def checkOutput(cmd): + def checkOutput(cmd, ignoreError=False): assert(isinstance(cmd, list)) popen=subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (output,error)=popen.communicate() Utils.CheckOutputDeque.append((output,error,cmd)) - if popen.returncode != 0: + if popen.returncode != 0 and not ignoreError: raise subprocess.CalledProcessError(returncode=popen.returncode, cmd=cmd, output=error) return output.decode("utf-8") @@ -171,6 +175,44 @@ class Utils: cmdArr=shlex.split(cmd) return Utils.runCmdArrReturnJson(cmdArr, trace=trace, silentErrors=silentErrors) + @staticmethod + def arePortsAvailable(ports): + """Check if specified port (as int) or ports (as set) is/are available for listening on.""" + assert(ports) + if isinstance(ports, int): + ports={ports} + assert(isinstance(ports, set)) + + for port in ports: + if Utils.Debug: Utils.Print("Checking if port %d is available." % (port)) + assert(isinstance(port, int)) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + try: + s.bind(("127.0.0.1", port)) + except socket.error as e: + if e.errno == errno.EADDRINUSE: + Utils.Print("ERROR: Port %d is already in use" % (port)) + else: + # something else raised the socket.error exception + Utils.Print("ERROR: Unknown exception while trying to listen on port %d" % (port)) + Utils.Print(e) + return False + finally: + s.close() + + return True + + @staticmethod + def pgrepCmd(serverName): + pgrepOpts="-fl" + # pylint: disable=deprecated-method + if platform.linux_distribution()[0] in ["Ubuntu", "LinuxMint", "Fedora","CentOS Linux","arch"]: + pgrepOpts="-a" + + return "pgrep %s %s" % (pgrepOpts, serverName) + ########################################################################################### class Account(object): diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt index 328b66a44621717657bfcdf0e619f23a38729fa0..c21c959731240ee2b8ce7afdff88180cc2701fb1 100644 --- a/unittests/CMakeLists.txt +++ b/unittests/CMakeLists.txt @@ -32,9 +32,6 @@ add_dependencies(unit_test asserter test_api test_api_mem test_api_db test_ram_l #Manually run unit_test for all supported runtimes #To run unit_test with all log from blockchain displayed, put --verbose after --, i.e. unit_test -- --verbose -add_test(NAME unit_test_binaryen COMMAND unit_test - -t \!wasm_tests/weighted_cpu_limit_tests - --report_level=detailed --color_output -- --binaryen) add_test(NAME unit_test_wavm COMMAND unit_test -t \!wasm_tests/weighted_cpu_limit_tests --report_level=detailed --color_output --catch_system_errors=no -- --wavm) @@ -59,7 +56,7 @@ if(ENABLE_COVERAGE_TESTING) endif() # NOT GENHTML_PATH # no spaces allowed within tests list - set(ctest_tests 'unit_test_binaryen|unit_test_wavm') + set(ctest_tests 'unit_test_wabt|unit_test_wavm') set(ctest_exclude_tests '') # Setup target diff --git a/unittests/abi_tests.cpp b/unittests/abi_tests.cpp index 44677b261c7d89aabdbda971dadc33ed4857d203..c3b46d3f8478f2a0958268ae4ee93c8609c2714c 100644 --- a/unittests/abi_tests.cpp +++ b/unittests/abi_tests.cpp @@ -3528,6 +3528,8 @@ BOOST_AUTO_TEST_CASE(abi_deep_structs_validate) BOOST_AUTO_TEST_CASE(variants) { + using eosio::testing::fc_exception_message_starts_with; + auto duplicate_variant_abi = R"({ "version": "eosio::abi/1.1", "variants": [ @@ -3575,13 +3577,18 @@ BOOST_AUTO_TEST_CASE(variants) BOOST_CHECK_THROW( abi_serializer( fc::json::from_string(variant_abi_invalid_type).as(), max_serialization_time ), invalid_type_inside_abi ); // expected array containing variant - BOOST_CHECK_THROW( abis.variant_to_binary("v1", fc::json::from_string(R"(9)"), max_serialization_time), abi_exception ); - BOOST_CHECK_THROW( abis.variant_to_binary("v1", fc::json::from_string(R"([4])"), max_serialization_time), abi_exception ); - BOOST_CHECK_THROW( abis.variant_to_binary("v1", fc::json::from_string(R"([4, 5])"), max_serialization_time), abi_exception ); - BOOST_CHECK_THROW( abis.variant_to_binary("v1", fc::json::from_string(R"(["4", 5, 6])"), max_serialization_time), abi_exception ); + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"(9)"), max_serialization_time), + pack_exception, fc_exception_message_starts_with("Expected input to be an array of two items while processing variant 'v1'") ); + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"([4])"), max_serialization_time), + pack_exception, fc_exception_message_starts_with("Expected input to be an array of two items while processing variant 'v1") ); + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"([4, 5])"), max_serialization_time), + pack_exception, fc_exception_message_starts_with("Encountered non-string as first item of input array while processing variant 'v1") ); + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"(["4", 5, 6])"), max_serialization_time), + pack_exception, fc_exception_message_starts_with("Expected input to be an array of two items while processing variant 'v1'") ); // type is not valid within this variant - BOOST_CHECK_THROW( abis.variant_to_binary("v1", fc::json::from_string(R"(["int9", 21])"), max_serialization_time), abi_exception ); + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"(["int9", 21])"), max_serialization_time), + pack_exception, fc_exception_message_starts_with("Specified type 'int9' in input array is not valid within the variant 'v1'") ); verify_round_trip_conversion(abis, "v1", R"(["int8",21])", "0015"); verify_round_trip_conversion(abis, "v1", R"(["string","abcd"])", "010461626364"); @@ -3594,6 +3601,8 @@ BOOST_AUTO_TEST_CASE(variants) BOOST_AUTO_TEST_CASE(extend) { + using eosio::testing::fc_exception_message_starts_with; + auto abi = R"({ "version": "eosio::abi/1.1", "structs": [ @@ -3603,18 +3612,27 @@ BOOST_AUTO_TEST_CASE(extend) {"name": "i2", "type": "int8$"}, {"name": "a", "type": "int8[]$"}, {"name": "o", "type": "int8?$"}, + ]}, + {"name": "s2", "base": "", "fields": [ + {"name": "i0", "type": "int8"}, + {"name": "i1", "type": "int8$"}, + {"name": "i2", "type": "int8"}, ]} ], })"; + // NOTE: Ideally this ABI would be rejected during validation for an improper definition for struct "s2". + // Such a check is not yet implemented during validation, but it can check during serialization. try { abi_serializer abis(fc::json::from_string(abi).as(), max_serialization_time ); // missing i1 - BOOST_CHECK_THROW( abis.variant_to_binary("s", fc::json::from_string(R"({"i0":5})"), max_serialization_time), abi_exception ); + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s", fc::json::from_string(R"({"i0":5})"), max_serialization_time), + pack_exception, fc_exception_message_starts_with("Missing field 'i1' in input object while processing struct") ); // Unexpected 'a' - BOOST_CHECK_THROW( abis.variant_to_binary("s", fc::json::from_string(R"({"i0":5,"i1":6,"a":[8,9,10]})"), max_serialization_time), pack_exception ); + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s", fc::json::from_string(R"({"i0":5,"i1":6,"a":[8,9,10]})"), max_serialization_time), + pack_exception, fc_exception_message_starts_with("Unexpected field 'a' found in input object while processing struct") ); verify_round_trip_conversion(abis, "s", R"({"i0":5,"i1":6})", "0506"); verify_round_trip_conversion(abis, "s", R"({"i0":5,"i1":6,"i2":7})", "050607"); @@ -3627,6 +3645,11 @@ BOOST_AUTO_TEST_CASE(extend) verify_round_trip_conversion(abis, "s", R"([5,6,7,[8,9,10]])", "0506070308090a", R"({"i0":5,"i1":6,"i2":7,"a":[8,9,10]})"); verify_round_trip_conversion(abis, "s", R"([5,6,7,[8,9,10],null])", "0506070308090a00", R"({"i0":5,"i1":6,"i2":7,"a":[8,9,10],"o":null})"); verify_round_trip_conversion(abis, "s", R"([5,6,7,[8,9,10],31])", "0506070308090a011f", R"({"i0":5,"i1":6,"i2":7,"a":[8,9,10],"o":31})"); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s2", fc::json::from_string(R"({"i0":1})"), max_serialization_time), + abi_exception, fc_exception_message_starts_with("Encountered field 'i2' without binary extension designation while processing struct") ); + + } FC_LOG_AND_RETHROW() } @@ -3660,10 +3683,10 @@ BOOST_AUTO_TEST_CASE(abi_serialize_incomplete_json_array) abi_serializer abis( fc::json::from_string(abi).as(), max_serialization_time ); BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s", fc::json::from_string(R"([])"), max_serialization_time), - pack_exception, fc_exception_message_starts_with("Early end to array specifying the fields of struct") ); + pack_exception, fc_exception_message_starts_with("Early end to input array specifying the fields of struct") ); BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s", fc::json::from_string(R"([1,2])"), max_serialization_time), - pack_exception, fc_exception_message_starts_with("Early end to array specifying the fields of struct") ); + pack_exception, fc_exception_message_starts_with("Early end to input array specifying the fields of struct") ); verify_round_trip_conversion(abis, "s", R"([1,2,3])", "010203", R"({"i0":1,"i1":2,"i2":3})"); @@ -3672,7 +3695,7 @@ BOOST_AUTO_TEST_CASE(abi_serialize_incomplete_json_array) BOOST_AUTO_TEST_CASE(abi_serialize_incomplete_json_object) { - using eosio::testing::fc_exception_message_is; + using eosio::testing::fc_exception_message_starts_with; auto abi = R"({ "version": "eosio::abi/1.0", @@ -3692,10 +3715,10 @@ BOOST_AUTO_TEST_CASE(abi_serialize_incomplete_json_object) abi_serializer abis( fc::json::from_string(abi).as(), max_serialization_time ); BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s2", fc::json::from_string(R"({})"), max_serialization_time), - pack_exception, fc_exception_message_is("Missing 'f0' in variant object") ); + pack_exception, fc_exception_message_starts_with("Missing field 'f0' in input object") ); BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s2", fc::json::from_string(R"({"f0":{"i0":1}})"), max_serialization_time), - pack_exception, fc_exception_message_is("Missing 'i1' in variant object") ); + pack_exception, fc_exception_message_starts_with("Missing field 'i1' in input object") ); verify_round_trip_conversion(abis, "s2", R"({"f0":{"i0":1,"i1":2},"i2":3})", "010203"); @@ -3723,12 +3746,249 @@ BOOST_AUTO_TEST_CASE(abi_serialize_json_mismatching_type) abi_serializer abis( fc::json::from_string(abi).as(), max_serialization_time ); BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s2", fc::json::from_string(R"({"f0":1,"i1":2})"), max_serialization_time), - pack_exception, fc_exception_message_is("Failed to serialize struct 's1' in variant object") ); + pack_exception, fc_exception_message_is("Unexpected input encountered while processing struct 's2.f0'") ); verify_round_trip_conversion(abis, "s2", R"({"f0":{"i0":1},"i1":2})", "0102"); } FC_LOG_AND_RETHROW() } +BOOST_AUTO_TEST_CASE(abi_serialize_detailed_error_messages) +{ + using eosio::testing::fc_exception_message_is; + + auto abi = R"({ + "version": "eosio::abi/1.1", + "types": [ + {"new_type_name": "foo", "type": "s2"}, + {"new_type_name": "bar", "type": "foo"}, + {"new_type_name": "s1array", "type": "s1[]"}, + {"new_type_name": "s1arrayarray", "type": "s1array[]"} + ], + "structs": [ + {"name": "s1", "base": "", "fields": [ + {"name": "i0", "type": "int8"}, + {"name": "i1", "type": "int8"} + ]}, + {"name": "s2", "base": "", "fields": [ + {"name": "f0", "type": "s1"}, + {"name": "i2", "type": "int8"} + ]}, + {"name": "s3", "base": "s1", "fields": [ + {"name": "i2", "type": "int8"}, + {"name": "f3", "type": "v2"}, + {"name": "f4", "type": "foo$"}, + {"name": "f5", "type": "s1$"} + ]}, + {"name": "s4", "base": "", "fields": [ + {"name": "f0", "type": "int8[]"}, + {"name": "f1", "type": "s1[]"} + ]}, + {"name": "s5", "base": "", "fields": [ + {"name": "f0", "type": "v2[]"}, + ]}, + ], + "variants": [ + {"name": "v1", "types": ["s3", "int8", "s4"]}, + {"name": "v2", "types": ["foo", "bar"]}, + ], + })"; + + try { + abi_serializer abis( fc::json::from_string(abi).as(), max_serialization_time ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("bar", fc::json::from_string(R"({"f0":{"i0":1},"i2":3})"), max_serialization_time), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's2.f0'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s3", fc::json::from_string(R"({"i0":1,"i2":3})"), max_serialization_time), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's3'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s3", fc::json::from_string(R"({"i0":1,"i1":2,"i2":3,"f3":["s2",{}]})"), max_serialization_time), + pack_exception, fc_exception_message_is("Specified type 's2' in input array is not valid within the variant 's3.f3'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s3", fc::json::from_string(R"({"i0":1,"i1":2,"i2":3,"f3":["bar",{"f0":{"i0":11},"i2":13}]})"), max_serialization_time), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's3.f3..f0'") ); + + verify_round_trip_conversion(abis, "s3", R"({"i0":1,"i1":2,"i2":3,"f3":["bar",{"f0":{"i0":11,"i1":12},"i2":13}]})", "010203010b0c0d"); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"(["s3",{"i0":1,"i1":2,"i2":3,"f3":["bar",{"f0":{"i0":11,"i1":12},"i2":13}],"f5":0}])"), max_serialization_time), + pack_exception, fc_exception_message_is("Unexpected field 'f5' found in input object while processing struct 'v1.'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"(["s4",{"f0":[0,1],"f1":[{"i0":2,"i1":3},{"i1":5}]}])"), max_serialization_time), + pack_exception, fc_exception_message_is("Missing field 'i0' in input object while processing struct 'v1..f1[1]'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s2[]", fc::json::from_string(R"([{"f0":{"i0":1,"i1":2},"i2":3},{"f0":{"i0":4},"i2":6}])"), max_serialization_time), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 'ARRAY[1].f0'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s5", fc::json::from_string(R"({"f0":[["bar",{"f0":{"i0":1,"i1":2},"i2":3}],["foo",{"f0":{"i0":4},"i2":6}]]})"), max_serialization_time), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's5.f0[1]..f0'") ); + + verify_round_trip_conversion( abis, "s1arrayarray", R"([[{"i0":1,"i1":2},{"i0":3,"i1":4}],[{"i0":5,"i1":6},{"i0":7,"i1":8},{"i0":9,"i1":10}]])", "0202010203040305060708090a"); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s1arrayarray", fc::json::from_string(R"([[{"i0":1,"i1":2},{"i0":3,"i1":4}],[{"i0":6,"i1":6},{"i0":7,"i1":8},{"i1":10}]])"), max_serialization_time), + pack_exception, fc_exception_message_is("Missing field 'i0' in input object while processing struct 'ARRAY[1][2]'") ); + } FC_LOG_AND_RETHROW() +} + +BOOST_AUTO_TEST_CASE(abi_serialize_short_error_messages) +{ + using eosio::testing::fc_exception_message_is; + + auto abi = R"({ + "version": "eosio::abi/1.1", + "types": [ + {"new_type_name": "foo", "type": "s2"}, + {"new_type_name": "bar", "type": "foo"}, + {"new_type_name": "s1array", "type": "s1[]"}, + {"new_type_name": "s1arrayarray", "type": "s1array[]"} + ], + "structs": [ + {"name": "s1", "base": "", "fields": [ + {"name": "i0", "type": "int8"}, + {"name": "i1", "type": "int8"} + ]}, + {"name": "s2", "base": "", "fields": [ + {"name": "f0", "type": "s1"}, + {"name": "i2", "type": "int8"} + ]}, + {"name": "very_very_very_very_very_very_very_very_very_very_long_struct_name_s3", "base": "s1", "fields": [ + {"name": "i2", "type": "int8"}, + {"name": "f3", "type": "v2"}, + {"name": "f4", "type": "foo$"}, + {"name": "very_very_very_very_very_very_very_very_very_very_long_field_name_f5", "type": "s1$"} + ]}, + {"name": "s4", "base": "", "fields": [ + {"name": "f0", "type": "int8[]"}, + {"name": "f1", "type": "s1[]"} + ]}, + {"name": "s5", "base": "", "fields": [ + {"name": "f0", "type": "v2[]"}, + ]}, + ], + "variants": [ + {"name": "v1", "types": ["very_very_very_very_very_very_very_very_very_very_long_struct_name_s3", "int8", "s4"]}, + {"name": "v2", "types": ["foo", "bar"]}, + ], + })"; + + try { + abi_serializer abis( fc::json::from_string(abi).as(), max_serialization_time ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("bar", fc::json::from_string(R"({"f0":{"i0":1},"i2":3})"), max_serialization_time, true), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's1'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary( "very_very_very_very_very_very_very_very_very_very_long_struct_name_s3", + fc::json::from_string(R"({"i0":1,"i2":3})"), max_serialization_time, true ), + pack_exception, + fc_exception_message_is("Missing field 'i1' in input object while processing struct 'very_very_very_very_very_very_very_very_very_very_long_...ame_s3'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary( "very_very_very_very_very_very_very_very_very_very_long_struct_name_s3", + fc::json::from_string(R"({"i0":1,"i1":2,"i2":3,"f3":["s2",{}]})"), max_serialization_time, true ), + pack_exception, fc_exception_message_is("Specified type 's2' in input array is not valid within the variant 'v2'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary( "very_very_very_very_very_very_very_very_very_very_long_struct_name_s3", + fc::json::from_string(R"({"i0":1,"i1":2,"i2":3,"f3":["bar",{"f0":{"i0":11},"i2":13}]})"), max_serialization_time, true ), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's1'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary( "v1", + fc::json::from_string(R"(["very_very_very_very_very_very_very_very_very_very_long_struct_name_s3",{"i0":1,"i1":2,"i2":3,"f3":["bar",{"f0":{"i0":11,"i1":12},"i2":13}],"very_very_very_very_very_very_very_very_very_very_long_field_name_f5":0}])"), + max_serialization_time, true ), + pack_exception, + fc_exception_message_is("Unexpected field 'very_very_very_very_very_very_very_very_very_very_long_...ame_f5' found in input object while processing struct 'very_very_very_very_very_very_very_very_very_very_long_...ame_s3'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("v1", fc::json::from_string(R"(["s4",{"f0":[0,1],"f1":[{"i0":2,"i1":3},{"i1":5}]}])"), max_serialization_time, true), + pack_exception, fc_exception_message_is("Missing field 'i0' in input object while processing struct 's1'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s2[]", fc::json::from_string(R"([{"f0":{"i0":1,"i1":2},"i2":3},{"f0":{"i0":4},"i2":6}])"), max_serialization_time, true), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's1'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s5", fc::json::from_string(R"({"f0":[["bar",{"f0":{"i0":1,"i1":2},"i2":3}],["foo",{"f0":{"i0":4},"i2":6}]]})"), max_serialization_time, true), + pack_exception, fc_exception_message_is("Missing field 'i1' in input object while processing struct 's1'") ); + + BOOST_CHECK_EXCEPTION( abis.variant_to_binary("s1arrayarray", fc::json::from_string(R"([[{"i0":1,"i1":2},{"i0":3,"i1":4}],[{"i0":6,"i1":6},{"i0":7,"i1":8},{"i1":10}]])"), max_serialization_time, true), + pack_exception, fc_exception_message_is("Missing field 'i0' in input object while processing struct 's1'") ); + } FC_LOG_AND_RETHROW() +} + +BOOST_AUTO_TEST_CASE(abi_deserialize_detailed_error_messages) +{ + using eosio::testing::fc_exception_message_is; + + auto abi = R"({ + "version": "eosio::abi/1.1", + "types": [ + {"new_type_name": "oint", "type": "int8?"}, + {"new_type_name": "os1", "type": "s1?"} + ], + "structs": [ + {"name": "s1", "base": "", "fields": [ + {"name": "i0", "type": "int8"}, + {"name": "i1", "type": "int8"} + ]}, + {"name": "s2", "base": "", "fields": [ + {"name": "f0", "type": "int8[]"}, + {"name": "f1", "type": "s1[]"} + ]}, + {"name": "s3", "base": "s1", "fields": [ + {"name": "i3", "type": "int8"}, + {"name": "i4", "type": "int8$"}, + {"name": "i5", "type": "int8"} + ]}, + {"name": "s4", "base": "", "fields": [ + {"name": "f0", "type": "oint[]"} + ]}, + {"name": "s5", "base": "", "fields": [ + {"name": "f0", "type": "os1[]"}, + {"name": "f1", "type": "v1[]"}, + ]}, + {"name": "s6", "base": "", "fields": [ + ]}, + ], + "variants": [ + {"name": "v1", "types": ["int8", "s1"]}, + ], + })"; + + try { + abi_serializer abis( fc::json::from_string(abi).as(), max_serialization_time ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s2", fc::variant("020102").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Stream unexpectedly ended; unable to unpack field 'f1' of struct 's2'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s2", fc::variant("0201020103").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Stream unexpectedly ended; unable to unpack field 'i1' of struct 's2.f1[0]'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s2", fc::variant("020102ff").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Unable to unpack size of array 's2.f1'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s3", fc::variant("010203").as(), max_serialization_time), + abi_exception, fc_exception_message_is("Encountered field 'i5' without binary extension designation while processing struct 's3'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s3", fc::variant("02010304").as(), max_serialization_time), + abi_exception, fc_exception_message_is("Encountered field 'i5' without binary extension designation while processing struct 's3'") ); + + // This check actually points to a problem with the current abi_serializer. + // An array of optionals (which is unfortunately not rejected in validation) leads to an unpack_exception here because one of the optional elements is not present. + // However, abis.binary_to_variant("s4", fc::variant("03010101020103").as(), max_serialization_time) would work just fine! + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s4", fc::variant("030101000103").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Invalid packed array 's4.f0[1]'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s4", fc::variant("020101").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Unable to unpack optional of built-in type 'int8' while processing 's4.f0[1]'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s5", fc::variant("02010102").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Unable to unpack presence flag of optional 's5.f0[1]'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s5", fc::variant("0001").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Unable to unpack tag of variant 's5.f1[0]'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s5", fc::variant("00010501").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Unpacked invalid tag (5) for variant 's5.f1[0]'") ); + + BOOST_CHECK_EXCEPTION( abis.binary_to_variant("s5", fc::variant("00010101").as(), max_serialization_time), + unpack_exception, fc_exception_message_is("Stream unexpectedly ended; unable to unpack field 'i1' of struct 's5.f1[0].'") ); + + } FC_LOG_AND_RETHROW() +} BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index c9811adf07111202fd858ae4fa7189445ed5ee32..60bde1edd99d439f2c615c5e30bf519ac273b01d 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -1964,23 +1964,9 @@ BOOST_FIXTURE_TEST_CASE(new_api_feature_tests, TESTER) { try { }); // change privilege - { - chainbase::database &db = control->db(); - const account_object &account = db.get(N(testapi)); - db.modify(account, [&](account_object &v) { - v.privileged = true; - }); - } - -#ifndef NON_VALIDATING_TEST - { - chainbase::database &db = validating_node->db(); - const account_object &account = db.get(N(testapi)); - db.modify(account, [&](account_object &v) { - v.privileged = true; - }); - } -#endif + push_action(config::system_account_name, N(setpriv), config::system_account_name, mutable_variant_object() + ("account", "testapi") + ("is_priv", 1)); CALL_TEST_FUNCTION( *this, "test_transaction", "new_feature", {} ); diff --git a/unittests/auth_tests.cpp b/unittests/auth_tests.cpp index bec8ab67b18e4b10010bb33627fee63ba736a6ec..e54964b87b451ab356116765815626e647834c81 100644 --- a/unittests/auth_tests.cpp +++ b/unittests/auth_tests.cpp @@ -383,7 +383,7 @@ try { chain.create_account(acc1a); chain.produce_block(); - chainbase::database &db = chain.control->db(); + const chainbase::database &db = chain.control->db(); using resource_usage_object = eosio::chain::resource_limits::resource_usage_object; using by_owner = eosio::chain::resource_limits::by_owner; diff --git a/unittests/database_tests.cpp b/unittests/database_tests.cpp index d2192d980f272fe402a8d455b3c5465754d717d0..ac97f6c21a6f4017d213a2ec695fa59061a2f844 100644 --- a/unittests/database_tests.cpp +++ b/unittests/database_tests.cpp @@ -26,7 +26,9 @@ BOOST_AUTO_TEST_SUITE(database_tests) BOOST_AUTO_TEST_CASE(undo_test) { try { TESTER test; - auto &db = test.control->db(); + + // Bypass read-only restriction on state DB access for this unit test which really needs to mutate the DB to properly conduct its test. + eosio::chain::database& db = const_cast( test.control->db() ); auto ses = db.start_undo_session(true); diff --git a/unittests/delay_tests.cpp b/unittests/delay_tests.cpp index 6d2d0ea63ccc541dc25da80933d4974dbcd35286..095ba93dfbee2810cb4e2c9610589dc6fd82efa9 100644 --- a/unittests/delay_tests.cpp +++ b/unittests/delay_tests.cpp @@ -2317,16 +2317,10 @@ BOOST_AUTO_TEST_CASE( max_transaction_delay_execute ) { try { chain.produce_blocks(); //change max_transaction_delay to 60 sec - chain.control->db().modify( chain.control->get_global_properties(), - [&]( auto& gprops ) { - gprops.configuration.max_transaction_delay = 60; - }); -#ifndef NON_VALIDATING_TEST - chain.validating_node->db().modify( chain.validating_node->get_global_properties(), - [&]( auto& gprops ) { - gprops.configuration.max_transaction_delay = 60; - }); -#endif + auto params = chain.control->get_global_properties().configuration; + params.max_transaction_delay = 60; + chain.push_action( config::system_account_name, N(setparams), config::system_account_name, mutable_variant_object() + ("params", params) ); chain.produce_blocks(); //should be able to create transaction with delay 60 sec, despite permission delay being 30 days, because max_transaction_delay is 60 sec diff --git a/unittests/special_accounts_tests.cpp b/unittests/special_accounts_tests.cpp index 9bdbc588e3a76c5b9d493465294924646bfc286e..5f5ebe198e87cba7793f14dc2c02fe25eb0b505d 100644 --- a/unittests/special_accounts_tests.cpp +++ b/unittests/special_accounts_tests.cpp @@ -35,7 +35,7 @@ BOOST_FIXTURE_TEST_CASE(accounts_exists, tester) tester test; chain::controller *control = test.control.get(); - chain::database &chain1_db = control->db(); + const chain::database& chain1_db = control->db(); auto nobody = chain1_db.find(config::null_account_name); BOOST_CHECK(nobody != nullptr); diff --git a/unittests/whitelist_blacklist_tests.cpp b/unittests/whitelist_blacklist_tests.cpp index 1621e2ef9164b7df902d78ed5d0196fdf2d134fb..9957b4ccccc6d3cdc179cf3eeb3b2e1067633295 100644 --- a/unittests/whitelist_blacklist_tests.cpp +++ b/unittests/whitelist_blacklist_tests.cpp @@ -41,14 +41,10 @@ class whitelist_blacklist_tester { cfg.genesis.initial_key = base_tester::get_public_key( config::system_account_name, "active" ); for(int i = 0; i < boost::unit_test::framework::master_test_suite().argc; ++i) { - if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--binaryen")) - cfg.wasm_runtime = chain::wasm_interface::vm_type::binaryen; - else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wavm")) + if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wavm")) cfg.wasm_runtime = chain::wasm_interface::vm_type::wavm; else if(boost::unit_test::framework::master_test_suite().argv[i] == std::string("--wabt")) cfg.wasm_runtime = chain::wasm_interface::vm_type::wabt; - else - cfg.wasm_runtime = chain::wasm_interface::vm_type::binaryen; } return cfg;